From 5f47aa24d1b1372b44fd6e8c8f9644125b0a7284 Mon Sep 17 00:00:00 2001 From: Daniel Dale Date: Fri, 10 Jun 2022 15:24:53 -0700 Subject: [PATCH 1/6] update fts link, add advanced feature ref, cleanup depth logging --- .gitignore | 4 ++++ lightning_examples/finetuning-scheduler/.meta.yml | 6 +++--- .../finetuning-scheduler/finetuning-scheduler.py | 14 +++++++++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index b97131ebb..c9008398a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# project +lightning_logs/ +.vscode/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/lightning_examples/finetuning-scheduler/.meta.yml b/lightning_examples/finetuning-scheduler/.meta.yml index 3cae7999e..3e6daa6ce 100644 --- a/lightning_examples/finetuning-scheduler/.meta.yml +++ b/lightning_examples/finetuning-scheduler/.meta.yml @@ -1,11 +1,11 @@ title: Finetuning Scheduler author: "[Dan Dale](https://github.com/speediedan)" created: 2021-11-29 -updated: 2022-05-10 +updated: 2022-XX-XX license: CC BY-SA -build: 3 +build: 4 tags: - - finetuning + - Finetuning description: | This notebook introduces the [Finetuning Scheduler](https://finetuning-scheduler.readthedocs.io/en/stable/index.html) extension and demonstrates the use of it to finetune a small foundational model on the diff --git a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py index e276fe677..f2aa4a231 100644 --- a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py +++ b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py @@ -59,7 +59,7 @@ # %% [markdown] # ## The Default Finetuning Schedule # -# Schedule definition is facilitated via the [gen_ft_schedule](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts_supporters.html#finetuning_scheduler.fts_supporters.SchedulingMixin.gen_ft_schedule) method which dumps a default finetuning schedule (by default using a naive, 2-parameters per level heuristic) which can be adjusted as +# Schedule definition is facilitated via the [gen_ft_schedule](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts_supporters.html#finetuning_scheduler.fts_supporters.ScheduleImplMixin.gen_ft_schedule) method which dumps a default finetuning schedule (by default using a naive, 2-parameters per level heuristic) which can be adjusted as # desired by the user and/or subsequently passed to the callback. Using the default/implicitly generated schedule will likely be less computationally efficient than a user-defined finetuning schedule but is useful for exploring a model's finetuning behavior and can serve as a good baseline for subsequent explicit schedule refinement. # While the current version of [FinetuningScheduler](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts.html#finetuning_scheduler.fts.FinetuningScheduler) only supports single optimizer and (optional) lr_scheduler configurations, per-phase maximum learning rates can be set as demonstrated in the next section. @@ -153,6 +153,9 @@ # - ``DDP_SHARDED`` # - ``DDP_SHARDED_SPAWN`` # +# Custom or officially unsupported strategies can be used by setting [FinetuningScheduler.allow_untested](https://finetuning-scheduler.readthedocs.io/en/latest/api/finetuning_scheduler.fts.html?highlight=allow_untested#finetuning_scheduler.fts.FinetuningScheduler.params.allow_untested) to ``True``. +# Note that most currently unsupported strategies are so because they require varying degrees of modification to be compatible (e.g. ``deepspeed`` requires an ``add_param_group`` method, ``tpu_spawn`` an override of the current broadcast method to include python objects) +# # # %% [markdown] @@ -387,9 +390,12 @@ def training_step(self, batch, batch_idx): self.log("train_loss", loss) return loss - def training_epoch_end(self, outputs: List[Any]) -> None: + def on_train_epoch_start(self) -> None: if self.finetuningscheduler_callback: - self.log("finetuning_schedule_depth", float(self.finetuningscheduler_callback.curr_depth)) + self.logger.log_metrics( + metrics={"finetuning_schedule_depth": float(self.finetuningscheduler_callback.curr_depth)}, + step=self.global_step, + ) def validation_step(self, batch, batch_idx, dataloader_idx=0): outputs = self(**batch) @@ -524,6 +530,8 @@ def configure_optimizers(self): # used in other pytorch-lightning tutorials) also work with FinetuningScheduler. Though the LR scheduler is theoretically # justified [(Loshchilov & Hutter, 2016)](#f4), the particular values provided here are primarily empircally driven. # +# [FinetuningScheduler](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts.html#finetuning_scheduler.fts.FinetuningScheduler) also supports LR scheduler +# reinitialization in both explicit and implicit finetuning schedule modes. See the [advanced usage documentation](https://finetuning-scheduler.readthedocs.io/en/stable/advanced/lr_scheduler_reinitialization.html) for explanations and demonstration of the extensions' support for more complex requirements. # From da4749a36f7d37b8a9d1abcce19066826961cdac Mon Sep 17 00:00:00 2001 From: Daniel Dale Date: Fri, 10 Jun 2022 15:25:59 -0700 Subject: [PATCH 2/6] fix sphinx language warning --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 8e90952d1..efadb440b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -99,7 +99,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From ff60373b21bbda708f33e1b7a094b593229afb9d Mon Sep 17 00:00:00 2001 From: Daniel Dale Date: Fri, 10 Jun 2022 15:34:06 -0700 Subject: [PATCH 3/6] revert inadvertent .gitignore update, address stale ffmpeg package references on nbval job, add reference to new fts release param --- .github/workflows/ci_testing.yml | 1 + .gitignore | 4 ---- .../finetuning-scheduler/finetuning-scheduler.py | 3 +-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_testing.yml b/.github/workflows/ci_testing.yml index 5f116f1eb..23ab72f6d 100644 --- a/.github/workflows/ci_testing.yml +++ b/.github/workflows/ci_testing.yml @@ -39,6 +39,7 @@ jobs: - name: Install dependencies run: | + sudo apt-get update --fix-missing sudo apt install -y ffmpeg pip --version pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html diff --git a/.gitignore b/.gitignore index c9008398a..b97131ebb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,3 @@ -# project -lightning_logs/ -.vscode/ - # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py index f2aa4a231..6e46bedbb 100644 --- a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py +++ b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py @@ -153,9 +153,8 @@ # - ``DDP_SHARDED`` # - ``DDP_SHARDED_SPAWN`` # -# Custom or officially unsupported strategies can be used by setting [FinetuningScheduler.allow_untested](https://finetuning-scheduler.readthedocs.io/en/latest/api/finetuning_scheduler.fts.html?highlight=allow_untested#finetuning_scheduler.fts.FinetuningScheduler.params.allow_untested) to ``True``. +# Custom or officially unsupported strategies can be used by setting [FinetuningScheduler.allow_untested](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts.html?highlight=allow_untested#finetuning_scheduler.fts.FinetuningScheduler.params.allow_untested) to ``True``. # Note that most currently unsupported strategies are so because they require varying degrees of modification to be compatible (e.g. ``deepspeed`` requires an ``add_param_group`` method, ``tpu_spawn`` an override of the current broadcast method to include python objects) -# # # %% [markdown] From ec97b7c643de9141b5d0425ab7beb89adcdf113d Mon Sep 17 00:00:00 2001 From: Daniel Dale Date: Fri, 10 Jun 2022 19:15:55 -0700 Subject: [PATCH 4/6] recheck with updated fts deps From a269bbbe877cbf58423288c59ea1c08db02afc57 Mon Sep 17 00:00:00 2001 From: Daniel Dale Date: Fri, 10 Jun 2022 20:00:41 -0700 Subject: [PATCH 5/6] add hydra version requirement until a future finetuning-scheduler release due to bug in omegaconf 2.0.6 that requires omegaconfig >=2.1.0 which in turns requires hydra-core>=1.1.0 if it's installed (fts does not require hydra) --- lightning_examples/finetuning-scheduler/.meta.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lightning_examples/finetuning-scheduler/.meta.yml b/lightning_examples/finetuning-scheduler/.meta.yml index 3e6daa6ce..92808323e 100644 --- a/lightning_examples/finetuning-scheduler/.meta.yml +++ b/lightning_examples/finetuning-scheduler/.meta.yml @@ -1,7 +1,7 @@ title: Finetuning Scheduler author: "[Dan Dale](https://github.com/speediedan)" created: 2021-11-29 -updated: 2022-XX-XX +updated: 2022-06-10 license: CC BY-SA build: 4 tags: @@ -15,5 +15,6 @@ description: | and foundational model weights. The required dependencies are installed via the finetuning-scheduler ``[examples]`` extra. requirements: - finetuning-scheduler[examples] + - hydra-core>=1.1.0 accelerator: - GPU From 1a6e1bd721cb075dae167c581e89e332e16ab23d Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 13 Jun 2022 13:57:33 +0200 Subject: [PATCH 6/6] Apply suggestions from code review Co-authored-by: Rohit Gupta --- lightning_examples/finetuning-scheduler/.meta.yml | 2 +- lightning_examples/finetuning-scheduler/finetuning-scheduler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lightning_examples/finetuning-scheduler/.meta.yml b/lightning_examples/finetuning-scheduler/.meta.yml index 92808323e..9cfa148fe 100644 --- a/lightning_examples/finetuning-scheduler/.meta.yml +++ b/lightning_examples/finetuning-scheduler/.meta.yml @@ -3,7 +3,7 @@ author: "[Dan Dale](https://github.com/speediedan)" created: 2021-11-29 updated: 2022-06-10 license: CC BY-SA -build: 4 +build: 0 tags: - Finetuning description: | diff --git a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py index 6e46bedbb..6458cacec 100644 --- a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py +++ b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py @@ -530,7 +530,7 @@ def configure_optimizers(self): # justified [(Loshchilov & Hutter, 2016)](#f4), the particular values provided here are primarily empircally driven. # # [FinetuningScheduler](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts.html#finetuning_scheduler.fts.FinetuningScheduler) also supports LR scheduler -# reinitialization in both explicit and implicit finetuning schedule modes. See the [advanced usage documentation](https://finetuning-scheduler.readthedocs.io/en/stable/advanced/lr_scheduler_reinitialization.html) for explanations and demonstration of the extensions' support for more complex requirements. +# reinitialization in both explicit and implicit finetuning schedule modes. See the [advanced usage documentation](https://finetuning-scheduler.readthedocs.io/en/stable/advanced/lr_scheduler_reinitialization.html) for explanations and demonstration of the extension's support for more complex requirements. #