From e5e85464937db03eacd7c77ac99c28e2a84ebd05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=AE=E6=A2=A6?= <46097299+frozenleaves@users.noreply.github.com> Date: Tue, 31 Mar 2026 23:04:45 +0800 Subject: [PATCH] [misc] fix moe (#10334) Co-authored-by: frozenleaves --- src/llamafactory/model/model_utils/moe.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/llamafactory/model/model_utils/moe.py b/src/llamafactory/model/model_utils/moe.py index d89019865..592e7e397 100644 --- a/src/llamafactory/model/model_utils/moe.py +++ b/src/llamafactory/model/model_utils/moe.py @@ -147,6 +147,11 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None: _set_z3_leaf_modules(model, [Qwen3NextSparseMoeBlock]) + if model_type == "qwen3_5_moe": + from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeSparseMoeBlock + + _set_z3_leaf_modules(model, [Qwen3_5MoeSparseMoeBlock]) + def configure_moe(config: "PretrainedConfig", model_args: "ModelArguments", is_trainable: bool) -> None: if not is_trainable or not model_args.moe_aux_loss_coef: