Skip to content

fix(chat): enable combining --mtp with --stream-experts #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
asher merged 1 commit into from
Aug 2, 2026
Merged

fix(chat): enable combining --mtp with --stream-experts #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

### Fixed

- `gmlx chat` no longer rejects `--stream-experts` on an MTP/speculative
base.
- Chat markdown rendering no longer reverts a block to raw text when it
grows taller than the terminal: the rendered top scrolls into scrollback
and only the last screenful stays live-repainted, so long fenced code
Expand Down
12 changes: 10 additions & 2 deletions gmlx/chat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2378,9 +2378,12 @@ def _backend_mtp_text(args, kv_kwargs) -> _ChatBackend:
from mlx_vlm.models.cache import make_prompt_cache as _mtp_make_cache

from . import loadlog
from .cli import _apply_placement
from .loader import preset_native_fp_wire_env
from .mtp_load import load_mtp_model

b = _ChatBackend()
preset_native_fp_wire_env(args)
with loadlog.load_ui(args.verbose, args.gguf):
b.model, b.drafter, b.config, b.tok = load_mtp_model(
args.gguf,
Expand All @@ -2389,7 +2392,11 @@ def _backend_mtp_text(args, kv_kwargs) -> _ChatBackend:
chat_template=args.chat_template,
zero_copy=not args.no_zero_copy,
verbose=args.verbose,
wire=not getattr(args, "stream_experts", False),
)
# Streaming placement applies to the target trunk only (the drafter
# stays resident); same composition as the run/bench entry points.
_apply_placement(args, getattr(b.model, "language_model", b.model))
_require_chat_template(b.tok)

# --kv-bits on the MTP path: same pooled-cache packing as the plain
Expand Down Expand Up @@ -2748,8 +2755,10 @@ def cmd_chat(argv: list[str] | None = None, prog: str = "gmlx chat") -> int:
# that already holds media) fall back to the plain VLM path. (resolve_speculative
# treats --mmproj as a hard MTP blocker, so the VLM x MTP decision is its own.)
vlm_mtp = bool(args.mmproj and _vlm_mtp_drafter_available(args))
# --stream-experts is absent: streaming composes with MTP here the
# same way as run/bench (placement after load_mtp_model).
if speculative and not vlm_mtp and (
args.mmproj or args.adapter or args.stream_experts or args.stream_cpu
args.mmproj or args.adapter or args.stream_cpu
or args.moe_experts is not None or args.moe_expert_mass is not None
or args.moe_expert_probe or args.moe_miss_shed is not None
or args.moe_layer_shed is not None
Expand All @@ -2759,7 +2768,6 @@ def cmd_chat(argv: list[str] | None = None, prog: str = "gmlx chat") -> int:
for name, on in (
("--mmproj", args.mmproj),
("--adapter", args.adapter),
("--stream-experts", args.stream_experts),
("--stream-cpu", args.stream_cpu),
("--moe-experts", args.moe_experts is not None),
("--moe-expert-mass", args.moe_expert_mass is not None),
Expand Down