From 91c653bac2c8400b0bbcbba079fc379d1d944e29 Mon Sep 17 00:00:00 2001 From: drbh Date: Wed, 17 Apr 2024 20:59:16 +0000 Subject: [PATCH] fix: default use_qk_norm false in cohere --- .../models/custom_modeling/flash_cohere_modeling.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py b/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py index 56d9a966429..1d4620c71be 100644 --- a/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py @@ -216,7 +216,9 @@ def __init__( self.query_key_value = load_attention(config, prefix, weights) - self.use_qk_norm = config.use_qk_norm + self.use_qk_norm = ( + config.use_qk_norm if hasattr(config, "use_qk_norm") else False + ) if self.use_qk_norm: self.q_norm = CohereLayerNorm( prefix=f"{prefix}.q_norm",