Upgrade transformers to 4.48.0 for llama2 #24302
Conversation
![github-advanced-security[bot]](https://avatars.githubusercontent.com/in/57789?s=60&v=4){kind=small}
| @@ -7,6 +7,7 @@ | |||
|
|
|||
| import numpy as np | |||
| import torch | |||
| import transformers | |||
Check notice
Code scanning / CodeQL
Module is imported with 'import' and 'import from' Note
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix
AI 10 days ago
To fix the problem, we should remove the from transformers import AutoConfig, AutoTokenizer statement and access these components through the transformers module directly. This approach maintains a single import style and avoids potential namespace conflicts.
- Remove the
from transformers import AutoConfig, AutoTokenizerstatement. - Update the code to access
AutoConfigandAutoTokenizerthrough thetransformersmodule.
| @@ -10,3 +10,2 @@ | ||
| import transformers | ||
| from transformers import AutoConfig, AutoTokenizer | ||
|
|
||
| @@ -32,3 +31,3 @@ | ||
| def get_sample_inputs( | ||
| config: AutoConfig, | ||
| config: transformers.AutoConfig, | ||
| device: torch.device, | ||
| @@ -67,3 +66,3 @@ | ||
| def get_sample_with_past_kv_inputs( | ||
| config: AutoConfig, | ||
| config: transformers.AutoConfig, | ||
| device: torch.device, |
| import torch | ||
| import transformers |
Check notice
Code scanning / CodeQL
Module is imported with 'import' and 'import from' Note
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix
AI 10 days ago
To fix the problem, we should remove the from transformers import AutoConfig statement and use transformers.AutoConfig instead. This will ensure that the module is only imported once, reducing confusion and potential for errors.
- Remove the
from transformers import AutoConfigstatement. - Replace all instances of
AutoConfigwithtransformers.AutoConfig.
| @@ -28,3 +28,2 @@ | ||
| from models.torch_export_patches.cache_helper import make_dynamic_cache | ||
| from transformers import AutoConfig | ||
|
|
||
| @@ -35,3 +34,3 @@ | ||
|
|
||
| def get_sequence_lengths(args: argparse.Namespace, config: AutoConfig): | ||
| def get_sequence_lengths(args: argparse.Namespace, config: transformers.AutoConfig): | ||
| past_sequence_length, curr_sequence_length = (8, 1) if args.use_past_kv else (0, 8) | ||
| @@ -41,3 +40,3 @@ | ||
|
|
||
| def get_inputs(args: argparse.Namespace, config: AutoConfig): | ||
| def get_inputs(args: argparse.Namespace, config: transformers.AutoConfig): | ||
| # Dummy values for parity | ||
| @@ -104,3 +103,3 @@ | ||
| pytorch_model: None | torch.nn.Module = None, | ||
| config: None | AutoConfig = None, | ||
| config: None | transformers.AutoConfig = None, | ||
| ): |
| def _catch_produce_guards_and_solve_constraints( | ||
| previous_function: Callable, | ||
| fake_mode: "FakeTensorMode", | ||
| gm: "torch.fx.GraphModule", | ||
| dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None, | ||
| equalities_inputs: "EqualityConstraint", # noqa: F821 | ||
| original_signature: inspect.Signature, | ||
| _is_torch_jit_trace: bool = False, | ||
| verbose: int = 0, | ||
| ): |
Check notice
Code scanning / CodeQL
Explicit returns mixed with implicit (fall through) returns Note
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix
AI 10 days ago
To fix the problem, we need to add explicit return statements at the end of the functions _catch_produce_guards_and_solve_constraints and patch__check_input_constraints_for_graph. This will ensure that the functions always return a value explicitly, making the code easier to read and understand.
- For
_catch_produce_guards_and_solve_constraints, we will addreturn Noneat the end of the function. - For
patch__check_input_constraints_for_graph, we will also addreturn Noneat the end of the function.
| @@ -43,3 +43,3 @@ | ||
| ) | ||
|
|
||
| return None | ||
|
|
||
| @@ -66,3 +66,3 @@ | ||
| ) | ||
|
|
||
| return None | ||
|
|
| def patch__check_input_constraints_for_graph( | ||
| previous_function: Callable, | ||
| input_placeholders: list[torch.fx.Node], | ||
| flat_args_with_path, | ||
| range_constraints, | ||
| verbose: int = 0, | ||
| ) -> None: |
Check notice
Code scanning / CodeQL
Explicit returns mixed with implicit (fall through) returns Note
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix
AI 10 days ago
To fix the problem, we need to add an explicit return statement at the end of the function patch__check_input_constraints_for_graph. This ensures that the function consistently returns None when no exception is raised, making the code easier to read and understand.
- Add an explicit
return Nonestatement at the end of the functionpatch__check_input_constraints_for_graph. - This change should be made in the file
onnxruntime/python/tools/transformers/models/torch_export_patches/patches/patch_torch.py.
| @@ -66,3 +66,3 @@ | ||
| ) | ||
|
|
||
| return None | ||
|
|
| # if config.print_specializations: | ||
| # self.log.warning( | ||
| # "Specializing %s to %s", self.var_to_sources[a][0].name(), tgt |
Check notice
Code scanning / CodeQL
Commented-out code Note
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix
AI 10 days ago
To fix the problem, we should remove the commented-out code entirely. This will clean up the code and eliminate any potential confusion for future developers. If the logging functionality is still required, it should be properly implemented with a configuration check.
- Remove the commented-out code on lines 304-308.
- Ensure that the removal does not affect the existing functionality of the code.
| @@ -303,7 +303,7 @@ | ||
|
|
||
| # if config.print_specializations: | ||
| # self.log.warning( | ||
| # "Specializing %s to %s", self.var_to_sources[a][0].name(), tgt | ||
| # ) | ||
| # self.log.debug("SPECIALIZATION", stack_info=True) | ||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
| assert msg != "range_refined_to_singleton", ( |
| # if input_ids.shape[1] == 0: | ||
| # inputs_embeds = inputs_embeds[:, -cache_position.shape[0] :] | ||
| # else: | ||
| # if cache_position[-1] >= input_ids.shape[1]: | ||
| # input_ids = input_ids[:, -cache_position.shape[0] :] | ||
| # else: | ||
| # if input_ids.shape[1] != cache_position.shape[0]: | ||
| # input_ids = input_ids[:, cache_position] |
Check notice
Code scanning / CodeQL
Commented-out code Note
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix
AI 10 days ago
To fix the problem, we should remove the commented-out code entirely. This will make the code cleaner and less confusing for future developers. The functionality of the code will remain unchanged as the commented-out code is not executed.
| @@ -279,11 +279,3 @@ | ||
| else: | ||
| # This is the code we need to implemented with torch.cond. | ||
| # if input_ids.shape[1] == 0: | ||
| # inputs_embeds = inputs_embeds[:, -cache_position.shape[0] :] | ||
| # else: | ||
| # if cache_position[-1] >= input_ids.shape[1]: | ||
| # input_ids = input_ids[:, -cache_position.shape[0] :] | ||
| # else: | ||
| # if input_ids.shape[1] != cache_position.shape[0]: | ||
| # input_ids = input_ids[:, cache_position] | ||
|
|
||
| def branch_1(inputs_embeds, cache_position): |
|
@microsoft-github-policy-service agree company="Microsoft" |
selenayang888
changed the title
Description
Upgrade Transformers to 4.48.0 for llama2, this version deprecated the old format of past_key_value, the current format is DynamicCache. So, we need to add patches to dynamo exporter in llama2.
Thanks to @xadupre who made the changes to add the patches to dynamo exporter, and implements patches to transformers 4.48.0 which don't export and convert dynamic_axes into dynamic shapes.