diff --git a/codeflash/code_utils/code_utils.py b/codeflash/code_utils/code_utils.py index 13a844015..507e79f74 100644 --- a/codeflash/code_utils/code_utils.py +++ b/codeflash/code_utils/code_utils.py @@ -13,7 +13,7 @@ def encoded_tokens_len(s: str) -> int: '''Function for returning the approximate length of the encoded tokens It's an approximation of BPE encoding (https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf)''' - return len(s)//2 + return int(len(s)*0.25) def get_qualified_name(module_name: str, full_qualified_name: str) -> str: if not full_qualified_name: