From 7f4d9b9ca6910896443ae6f06b0539adfcd6802f Mon Sep 17 00:00:00 2001 From: praneshnikhar Date: Sun, 9 Aug 2026 23:37:14 +0530 Subject: [PATCH] feat: write hidden_activation='relu2' for BitNet models + bump submodule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BitNet b1.58 architecture uses squared ReLU (relu²) in the FFN, but the graph builder was hardcoded to SiLU and the GGUF files didn't include the hidden_activation key. This caused ~6x perplexity regression on every backend. Changes: - Update HF/MS converters to write hidden_activation='relu2' for BitNet models - Bump llama.cpp submodule to include config-based activation support (reads .hidden_activation, defaults to SiLU for backward compat, uses ReLU² when key is present) When models are re-converted with these changes, the runtime will automatically use the correct ReLU² activation. Existing GGUF files without the key continue to work (behavior unchanged). Fixes microsoft/BitNet#602 Submodule PR: https://github.com/praneshnikhar/llama.cpp/pull/2 --- 3rdparty/llama.cpp | 2 +- utils/convert-hf-to-gguf-bitnet.py | 2 ++ utils/convert-ms-to-gguf-bitnet.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/3rdparty/llama.cpp b/3rdparty/llama.cpp index 390c30775..1d75155fc 160000 --- a/3rdparty/llama.cpp +++ b/3rdparty/llama.cpp @@ -1 +1 @@ -Subproject commit 390c307752ab78fd8189f359d6954c9ba1be74af +Subproject commit 1d75155fcc8e882c9585f733704153784cddae04 diff --git a/utils/convert-hf-to-gguf-bitnet.py b/utils/convert-hf-to-gguf-bitnet.py index b11e831b9..dd3a9c7f3 100644 --- a/utils/convert-hf-to-gguf-bitnet.py +++ b/utils/convert-hf-to-gguf-bitnet.py @@ -1045,6 +1045,8 @@ def set_gguf_parameters(self): self.gguf_writer.add_vocab_size(self.hparams["vocab_size"]) + self.gguf_writer.add_hidden_act("relu2") + # rope dimension count (required for correct positional encoding) if "head_dim" in self.hparams: rope_dim = self.hparams["head_dim"] diff --git a/utils/convert-ms-to-gguf-bitnet.py b/utils/convert-ms-to-gguf-bitnet.py index edf702788..755729bb0 100644 --- a/utils/convert-ms-to-gguf-bitnet.py +++ b/utils/convert-ms-to-gguf-bitnet.py @@ -1178,6 +1178,8 @@ def add_meta_arch(self, params: Params) -> None: else: raise ValueError('f_norm_eps is None') + self.gguf.add_hidden_act("relu2") + if params.f_rope_freq_base is not None: self.gguf.add_rope_freq_base(params.f_rope_freq_base)