From 376c6a6f48d4130e5a4a45aea1ffce46f7ba7a7e Mon Sep 17 00:00:00 2001 From: Swati Allabadi Date: Tue, 18 Mar 2025 11:09:20 +0000 Subject: [PATCH] 1. In case of finetuning on qaic, torch_qaic gradScaler will be used. 2. Moving back to lora_dropout = 0.05 on ML Framework team's ask. Signed-off-by: Swati Allabadi --- QEfficient/finetune/configs/peft_config.py | 2 +- QEfficient/finetune/utils/train_utils.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/QEfficient/finetune/configs/peft_config.py b/QEfficient/finetune/configs/peft_config.py index ae9b1c8c3..e2d018f05 100644 --- a/QEfficient/finetune/configs/peft_config.py +++ b/QEfficient/finetune/configs/peft_config.py @@ -19,7 +19,7 @@ class lora_config: target_modules: List[str] = field(default_factory=lambda: ["q_proj", "v_proj"]) bias = "none" task_type: str = "CAUSAL_LM" - lora_dropout: float = 0.0 + lora_dropout: float = 0.05 inference_mode: bool = False # should be False for finetuning diff --git a/QEfficient/finetune/utils/train_utils.py b/QEfficient/finetune/utils/train_utils.py index d8e2799f4..241c5ebaa 100644 --- a/QEfficient/finetune/utils/train_utils.py +++ b/QEfficient/finetune/utils/train_utils.py @@ -24,6 +24,7 @@ import torch_qaic.debug as qaic_debug # noqa: F401 import torch_qaic.profile as qaic_profile # noqa: F401 import torch_qaic.utils as qaic_utils # noqa: F401 + from torch.qaic.amp import GradScaler as QAicGradScaler except ImportError as e: print(f"Warning: {e}. Moving ahead without these qaic modules.") @@ -60,7 +61,6 @@ def train( Returns: results dictionary containing average training and validation perplexity and loss """ - train_prep = [] train_loss = [] val_prep = [] @@ -92,7 +92,10 @@ def train( tensorboard_updates = SummaryWriter() if train_config.grad_scaler: - scaler = GradScaler() + if device.startswith("qaic"): + scaler = QAicGradScaler() + else: + scaler = GradScaler() loss_0_counter = torch.tensor([0]).to(device)