fix few gguf bugs of hf-name and checkpoint name

auto_round/compressors/entry.py

@@ -362,7 +362,8 @@ def __new__(
         # Model-free routing is now supported directly by the new entry path.
         model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)
         model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)
-        if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_kwargs):
+        route_decision_kwargs = dict(route_kwargs, format=format)
+        if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs):
             from auto_round.compressors.model_free import ModelFreeCompressor
 
             if not isinstance(model, str):

auto_round/compressors/model_free.py

@@ -1910,6 +1910,11 @@ def __init__(
         # Start from the remaining user kwargs and explicitly set/override
         # known compressor init parameters for clarity.
         fallback_init = dict(fallback_kwargs)
+        # Route-control kwargs are only meaningful for the initial entry
+        # selection. Strip them so fallback always re-enters the regular flow
+        # with a single explicit disable_model_free=True override.
+        fallback_init.pop("model_free", None)
+        fallback_init.pop("disable_model_free", None)
         fallback_init.update(
             model=model_name_or_path,
             iters=0,

auto_round/compressors/utils.py

@@ -611,6 +611,7 @@ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str
         model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT
         layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)
 
+    _apply_gguf_shape_fallback(layer_config, model)
     dispatch_layer_config(layer_config)
     return layer_config, has_qlayer_outside_block, regex_config
 
@@ -681,6 +682,48 @@ def get_gguf_qtype_by_layer_config(layer_config):
     raise ValueError("Unknown layer config")
 
 
+def _apply_gguf_shape_fallback(layer_config, model):
+    from auto_round.utils.model import get_module
+
+    for layer_name, config in layer_config.items():
+        if not check_to_quantized(config):
+            continue
+        layer = get_module(model, layer_name)
+        if layer is None or not hasattr(layer, "weight"):
+            continue
+        try:
+            qtype = get_gguf_qtype_by_layer_config(config)
+        except ValueError:
+            continue
+        if qtype is None:
+            continue
+
+        gguf_type = f"gguf:{qtype.name.lower()}"
+        block_size = GGML_QUANT_SIZES[gguf_type.split(":")[-1]][0]
+        input_features = (
+            layer.weight.shape[0] if type(layer) == transformers.pytorch_utils.Conv1D else layer.weight.shape[-1]
+        )
+        if input_features % block_size == 0:
+            continue
+
+        fallback_type = _gguf_type_fallback(gguf_type)
+        fallback_block_size = GGML_QUANT_SIZES[fallback_type.split(":")[-1]][0]
+        if input_features % fallback_block_size != 0:
+            fallback_type = "gguf:bf16"
+
+        preserved = {key: config[key] for key in ("fixed_by_user", "scale_dtype") if key in config}
+        config.update(GGUF_INNER_CONFIG[fallback_type])
+        config.update(preserved)
+        logger.warning(
+            "fallback %s to %s before quantization, because input_features(%s) is not divisible by %s block_size(%s)",
+            layer_name,
+            fallback_type,
+            input_features,
+            gguf_type,
+            block_size,
+        )
+
+
 def _get_digital_in_layer_name(layer_name):
     pattern = re.compile(r"([a-zA-Z]+\.){1,}(\d+)")
     res = re.search(pattern, layer_name)

auto_round/eval/evaluation.py

@@ -184,6 +184,28 @@ def evaluate_diffusion_model(args, autoround=None, model=None, pipe=None):
         diffusion_eval(pipe, args.prompt_file, metrics, args.image_save_dir, 1, gen_kwargs, args.limit)
 
 
+def select_gguf_eval_file(eval_folder, formats):
+    """Select the text GGUF file used for HF-based evaluation."""
+    gguf_format = None
+    for format in formats:
+        if format.startswith("gguf"):
+            gguf_format = format.split(":")[-1].upper()
+            break
+
+    if gguf_format is None:
+        return None, []
+
+    gguf_files = sorted(
+        file for file in os.listdir(eval_folder) if file.endswith(".gguf") and file != "mmproj-model.gguf"
+    )
+    matched_files = [file for file in gguf_files if gguf_format in file]
+    if matched_files:
+        return matched_files[0], gguf_files
+    if len(gguf_files) == 1:
+        return gguf_files[0], gguf_files
+    return None, gguf_files
+
+
 def load_gguf_model_for_eval(eval_folder, formats, args):
     """
     Load GGUF model for evaluation.
@@ -202,26 +224,13 @@ def load_gguf_model_for_eval(eval_folder, formats, args):
 
     from auto_round.utils import get_model_dtype, logger
 
-    # Find corresponding GGUF format
-    gguf_format = None
-    for format in formats:
-        if format.startswith("gguf"):
-            gguf_format = format.split(":")[-1].upper()
-            break
-
-    if gguf_format is None:
+    gguf_file, gguf_files = select_gguf_eval_file(eval_folder, formats)
+    if not any(format.startswith("gguf") for format in formats):
         logger.error("No valid gguf format found in formats. Please check the input.")
         sys.exit(-1)
 
-    # Find matching GGUF file
-    gguf_file = None
-    for file in os.listdir(eval_folder):
-        if gguf_format in file:
-            gguf_file = file
-            break
-
     if gguf_file is None:
-        logger.error("Cannot find correct gguf file for evaluation, please check.")
+        logger.error("Cannot find correct gguf file for evaluation, candidates=%s", gguf_files)
         sys.exit(-1)
 
     # Load model and tokenizer
@@ -482,6 +491,8 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):
         # Load or prepare model instance
         if eval_gguf_model:
             model, tokenizer = load_gguf_model_for_eval(eval_folder, formats, args)
+            if model is None:
+                return
         else:
             eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
             model = prepare_model_for_eval(model, args.device_map, eval_model_dtype)