diff --git a/gallery/index.yaml b/gallery/index.yaml index e46620e729c1..f09d3224163b 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1630,6 +1630,9 @@ variants: - model: qwen3.8-27b-q4-mtp - model: qwen3.8-27b-ridge + - model: qwen3.8-27b-gsq-rco-iq2-xs + - model: qwen3.8-27b-gsq-rco-iq2-s + - model: qwen3.8-27b-gsq-rco-iq3-xxs - model: qwen3.8-27b-q8 url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: @@ -1845,6 +1848,129 @@ - filename: llama-cpp/mmproj/qwen3.8-27b-ridge/mmproj-Qwen3.8-27B-BF16.gguf uri: huggingface://empero-ai/Qwen3.8-27B-Ridge-GGUF/mmproj-Qwen3.8-27B-BF16.gguf sha256: 52228402ce4823f10705d901813cd43ced71859524cf2d8bf83305ad6b7dcbc2 +- !!merge <<: *qwen3-8-27b + name: "qwen3.8-27b-gsq-rco-iq2-xs" + variants: [] + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF + description: | + Qwen3.8-27B with ISTA DASLab's smallest GSQ-RCO mixed quantization. The + 2.50-bit IQ2_XS model uses 8.4 GB for the weights and retains the shared + BF16 vision projector. + last_checked: "2026-08-30" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/qwen3.8-27b-gsq-rco/mmproj-Qwen3.8-27B-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/qwen3.8-27b-gsq-rco/Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf + presence_penalty: 0 + repeat_penalty: 1 + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-27b-gsq-rco/Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf + uri: huggingface://ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf + sha256: f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c + - filename: llama-cpp/mmproj/qwen3.8-27b-gsq-rco/mmproj-Qwen3.8-27B-BF16.gguf + uri: huggingface://ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/mmproj-Qwen3.8-27B-BF16.gguf + sha256: 13cb7bebccbd04afc8f4090cb949ecf8937cdf7377c5799b1a0c594e7c0d3e16 +- !!merge <<: *qwen3-8-27b + name: "qwen3.8-27b-gsq-rco-iq2-s" + variants: [] + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF + description: | + Qwen3.8-27B with ISTA DASLab's 2.75-bit GSQ-RCO IQ2_S mixed quantization. + The 9.3 GB model targets higher fidelity than IQ2_XS and retains the shared + BF16 vision projector. + last_checked: "2026-08-30" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/qwen3.8-27b-gsq-rco/mmproj-Qwen3.8-27B-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/qwen3.8-27b-gsq-rco/Qwen3.8-27B-GSQ-RCO-IQ2_S.gguf + presence_penalty: 0 + repeat_penalty: 1 + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-27b-gsq-rco/Qwen3.8-27B-GSQ-RCO-IQ2_S.gguf + uri: huggingface://ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/Qwen3.8-27B-GSQ-RCO-IQ2_S.gguf + sha256: 16c9802111aa9ef3acde465188d6d601f8db128ee3d828ad983a5caca4135ecb + - filename: llama-cpp/mmproj/qwen3.8-27b-gsq-rco/mmproj-Qwen3.8-27B-BF16.gguf + uri: huggingface://ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/mmproj-Qwen3.8-27B-BF16.gguf + sha256: 13cb7bebccbd04afc8f4090cb949ecf8937cdf7377c5799b1a0c594e7c0d3e16 +- !!merge <<: *qwen3-8-27b + name: "qwen3.8-27b-gsq-rco-iq3-xxs" + variants: [] + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF + description: | + Qwen3.8-27B with ISTA DASLab's recommended 3.00-bit GSQ-RCO IQ3_XXS mixed + quantization. The 10.1 GB model provides the highest quality in this set + and retains the shared BF16 vision projector. + last_checked: "2026-08-30" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/qwen3.8-27b-gsq-rco/mmproj-Qwen3.8-27B-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/qwen3.8-27b-gsq-rco/Qwen3.8-27B-GSQ-RCO-IQ3_XXS.gguf + presence_penalty: 0 + repeat_penalty: 1 + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-27b-gsq-rco/Qwen3.8-27B-GSQ-RCO-IQ3_XXS.gguf + uri: huggingface://ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/Qwen3.8-27B-GSQ-RCO-IQ3_XXS.gguf + sha256: fdfcb6a29b11188956dfbfd904223588a6c1b77eb250c3e8a36e1bd269df91f7 + - filename: llama-cpp/mmproj/qwen3.8-27b-gsq-rco/mmproj-Qwen3.8-27B-BF16.gguf + uri: huggingface://ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/mmproj-Qwen3.8-27B-BF16.gguf + sha256: 13cb7bebccbd04afc8f4090cb949ecf8937cdf7377c5799b1a0c594e7c0d3e16 - &qwen3-8-9b name: "qwen3.8-9b-q4" variants: