[MVP Task 04] Support hf/transformers 'config.json' for EXL3, FP8 and BF16 models

[mratsim]

Overview

Our MVP will focus on state-of-the-art EXL3 quantization hence we need to support exllamav3 customized config.json: https://github.com/turboderp-org/exllamav3/blob/v0.0.20/exllamav3/model/config.py

Also since AI labs routinely use BF16 and FP8 weights, we should also support the default ones that huggingface/transformers use.

Examples

Exllamav3

https://huggingface.co/mratsim/GLM-4.7-EXL3/blob/3.84bpw-tuned/config.json

{
    "architectures": [
        "Glm4MoeForCausalLM"
    ],
    "attention_bias": true,
    "attention_dropout": 0.0,
    "pad_token_id": 151329,
    "eos_token_id": [
        151329,
        151336,
        151338
    ],
    "head_dim": 128,
    "hidden_act": "silu",
    "hidden_size": 5120,
    "partial_rotary_factor": 0.5,
    "initializer_range": 0.02,
    "intermediate_size": 12288,
    "max_position_embeddings": 202752,
    "model_type": "glm4_moe",
    "moe_intermediate_size": 1536,
    "norm_topk_prob": true,
    "num_attention_heads": 96,
    "n_group": 1,
    "topk_group": 1,
    "n_routed_experts": 160,
    "n_shared_experts": 1,
    "routed_scaling_factor": 2.5,
    "num_experts_per_tok": 8,
    "first_k_dense_replace": 3,
    "num_hidden_layers": 92,
    "num_key_value_heads": 8,
    "rms_norm_eps": 1e-05,
    "rope_scaling": null,
    "rope_theta": 1000000,
    "num_nextn_predict_layers": 1,
    "tie_word_embeddings": false,
    "torch_dtype": "bfloat16",
    "transformers_version": "4.54.0",
    "use_cache": true,
    "use_qk_norm": true,
    "vocab_size": 151552,
    "quantization_config": {
        "quant_method": "exl3",
        "version": "0.0.14",
        "bits": 3.84,
        "head_bits": 8,
        "calibration": {
            "rows": 250,
            "cols": 2048
        },
        "out_scales": "auto",
        "codebook": "mcg"
    }
}

FP8

https://huggingface.co/MiniMaxAI/MiniMax-M2.1/blob/main/config.json

{
  "architectures": [
    "MiniMaxM2ForCausalLM"
  ],
  "attn_type_list": [
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1,
    1
  ],
  "auto_map": {
    "AutoConfig": "configuration_minimax_m2.MiniMaxM2Config",
    "AutoModelForCausalLM": "modeling_minimax_m2.MiniMaxM2ForCausalLM"
  },
  "head_dim": 128,
  "hidden_act": "silu",
  "hidden_size": 3072,
  "intermediate_size": 1536,
  "max_position_embeddings": 196608,
  "model_type": "minimax_m2",
  "mtp_transformer_layers": 1,
  "num_attention_heads": 48,
  "num_experts_per_tok": 8,
  "num_hidden_layers": 62,
  "num_key_value_heads": 8,
  "num_local_experts": 256,
  "num_mtp_modules": 3,
  "qk_norm_type": "per_layer",
  "quantization_config": {
    "activation_scheme": "dynamic",
    "fmt": "float8_e4m3fn",
    "quant_method": "fp8",
    "weight_block_size": [
      128,
      128
    ],
    "modules_to_not_convert": [
      "gate",
      "e_score_correction_bias",
      "lm_head"
    ]
  },
  "rms_norm_eps": 1e-06,
  "rope_theta": 5000000,
  "rotary_dim": 64,
  "scoring_func": "sigmoid",
  "shared_intermediate_size": 0,
  "tie_word_embeddings": false,
  "transformers_version": "4.46.1",
  "use_cache": true,
  "use_mtp": true,
  "use_qk_norm": true,
  "use_routing_bias": true,
  "vocab_size": 200064
}

References

• https://github.com/turboderp-org/exllamav3/blob/v0.0.20/exllamav3/model/config.py