Skip to content

Warn when LoRA skips the dense and shared expert layers of MoE models on transformers v5 with peft 0.19 to 0.21.2 #7771

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
albertvillanova wants to merge 2 commits into
base: main
Choose a base branch
from
+319 −3
Open

Warn when LoRA skips the dense and shared expert layers of MoE models on transformers v5 with peft 0.19 to 0.21.2 #7771

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1801502
Raise when LoRA would skip the routed experts of MoE models on transf…
albertvillanova Oct 16, 2026
7273cb2
Warn when LoRA skips the dense and shared expert layers of MoE models…
albertvillanova Oct 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 47 additions & 1 deletion trl/trainer/distillation_trainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@


if is_peft_available():
from peft import PeftConfig, PeftModel, PromptLearningConfig, get_peft_model
import peft
from peft import LoraConfig, PeftConfig, PeftModel, PromptLearningConfig, get_peft_model
from peft.tuners.tuners_utils import BaseTunerLayer


Expand Down Expand Up @@ -534,6 +535,51 @@ def __init__(
"tokens are never prepended and the loss is computed on the wrong sequence. Use a weight-based "
"adapter such as LoRA instead."
)
# With transformers v5, peft<0.19 doesn't apply `target_modules` to the fused MoE routed experts (#7765)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version(peft.__version__) < Version("0.19.0")
and (Version(peft.__version__) < Version("0.17.0") or not peft_config.target_parameters)
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
raise ValueError(
"With transformers v5, LoRA `target_modules` only reach the routed experts of MoE models from "
"`peft>=0.19.0`. Upgrade peft, or also target the expert parameters with `target_parameters` "
"(e.g. `target_parameters=['mlp.experts.gate_up_proj', 'mlp.experts.down_proj']`). See "
"https://github.com/huggingface/trl/issues/7765."
)
# With transformers v5, peft 0.19 to 0.21.2 skip the MoE dense and shared layers (#7767)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version("0.19.0") <= Version(peft.__version__) < Version("0.21.3.dev0")
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and any(
isinstance(module, torch.nn.Linear)
and name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name, module in model.named_modules()
)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
logger.warning(
"With transformers v5 and peft>=0.19.0,<0.21.3, LoRA `target_modules` skip the dense and shared "
"expert layers of MoE models, so only the routed experts are trained. Install peft from main to "
"train them too. See https://github.com/huggingface/trl/issues/7767."
)
model = get_peft_model(model, peft_config, **get_peft_model_kwargs)

# PEFT + DeepSpeed ZeRO-3 requires reentrant checkpointing. For more details, see
Expand Down
45 changes: 45 additions & 0 deletions trl/trainer/dpo_trainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -661,6 +661,51 @@ def __init__(
"peft, or load the model with `device_map=None`. See "
"https://github.com/huggingface/peft/issues/2821."
)
# With transformers v5, peft<0.19 doesn't apply `target_modules` to the fused MoE routed experts (#7765)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version(peft.__version__) < Version("0.19.0")
and (Version(peft.__version__) < Version("0.17.0") or not peft_config.target_parameters)
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
raise ValueError(
"With transformers v5, LoRA `target_modules` only reach the routed experts of MoE models from "
"`peft>=0.19.0`. Upgrade peft, or also target the expert parameters with `target_parameters` "
"(e.g. `target_parameters=['mlp.experts.gate_up_proj', 'mlp.experts.down_proj']`). See "
"https://github.com/huggingface/trl/issues/7765."
)
# With transformers v5, peft 0.19 to 0.21.2 skip the MoE dense and shared layers (#7767)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version("0.19.0") <= Version(peft.__version__) < Version("0.21.3.dev0")
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and any(
isinstance(module, torch.nn.Linear)
and name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name, module in model.named_modules()
)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
logger.warning(
"With transformers v5 and peft>=0.19.0,<0.21.3, LoRA `target_modules` skip the dense and shared "
"expert layers of MoE models, so only the routed experts are trained. Install peft from main to "
"train them too. See https://github.com/huggingface/trl/issues/7767."
)
model = get_peft_model(model, peft_config, **get_peft_model_kwargs)

elif is_peft_model(model) and ref_model is None:
Expand Down
45 changes: 45 additions & 0 deletions trl/trainer/grpo_trainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,51 @@ def __init__(
"peft, or load the model with `device_map=None`. See "
"https://github.com/huggingface/peft/issues/2821."
)
# With transformers v5, peft<0.19 doesn't apply `target_modules` to the fused MoE routed experts (#7765)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version(peft.__version__) < Version("0.19.0")
and (Version(peft.__version__) < Version("0.17.0") or not peft_config.target_parameters)
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
raise ValueError(
"With transformers v5, LoRA `target_modules` only reach the routed experts of MoE models from "
"`peft>=0.19.0`. Upgrade peft, or also target the expert parameters with `target_parameters` "
"(e.g. `target_parameters=['mlp.experts.gate_up_proj', 'mlp.experts.down_proj']`). See "
"https://github.com/huggingface/trl/issues/7765."
)
# With transformers v5, peft 0.19 to 0.21.2 skip the MoE dense and shared layers (#7767)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version("0.19.0") <= Version(peft.__version__) < Version("0.21.3.dev0")
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and any(
isinstance(module, torch.nn.Linear)
and name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name, module in model.named_modules()
)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
logger.warning(
"With transformers v5 and peft>=0.19.0,<0.21.3, LoRA `target_modules` skip the dense and shared "
"expert layers of MoE models, so only the routed experts are trained. Install peft from main to "
"train them too. See https://github.com/huggingface/trl/issues/7767."
)
model = get_peft_model(model, peft_config, **get_peft_model_kwargs)

elif is_peft_model(model) and args.beta != 0.0:
Expand Down
45 changes: 45 additions & 0 deletions trl/trainer/kto_trainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -712,6 +712,51 @@ def __init__(
"peft, or load the model with `device_map=None`. See "
"https://github.com/huggingface/peft/issues/2821."
)
# With transformers v5, peft<0.19 doesn't apply `target_modules` to the fused MoE routed experts (#7765)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version(peft.__version__) < Version("0.19.0")
and (Version(peft.__version__) < Version("0.17.0") or not peft_config.target_parameters)
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
raise ValueError(
"With transformers v5, LoRA `target_modules` only reach the routed experts of MoE models from "
"`peft>=0.19.0`. Upgrade peft, or also target the expert parameters with `target_parameters` "
"(e.g. `target_parameters=['mlp.experts.gate_up_proj', 'mlp.experts.down_proj']`). See "
"https://github.com/huggingface/trl/issues/7765."
)
# With transformers v5, peft 0.19 to 0.21.2 skip the MoE dense and shared layers (#7767)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version("0.19.0") <= Version(peft.__version__) < Version("0.21.3.dev0")
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and any(
isinstance(module, torch.nn.Linear)
and name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name, module in model.named_modules()
)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
logger.warning(
"With transformers v5 and peft>=0.19.0,<0.21.3, LoRA `target_modules` skip the dense and shared "
"expert layers of MoE models, so only the routed experts are trained. Install peft from main to "
"train them too. See https://github.com/huggingface/trl/issues/7767."
)
model = get_peft_model(model, peft_config, **get_peft_model_kwargs)

elif is_peft_model(model) and ref_model is None:
Expand Down
47 changes: 46 additions & 1 deletion trl/trainer/reward_trainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@

if is_peft_available():
import peft
from peft import PeftConfig, PeftModel, PrefixTuningConfig, get_peft_model
from peft import LoraConfig, PeftConfig, PeftModel, PrefixTuningConfig, get_peft_model


logger = get_logger(__name__)
Expand Down Expand Up @@ -516,6 +516,51 @@ def __init__(
"peft, or load the model with `device_map=None`. See "
"https://github.com/huggingface/peft/issues/2821."
)
# With transformers v5, peft<0.19 doesn't apply `target_modules` to the fused MoE routed experts (#7765)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version(peft.__version__) < Version("0.19.0")
and (Version(peft.__version__) < Version("0.17.0") or not peft_config.target_parameters)
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
raise ValueError(
"With transformers v5, LoRA `target_modules` only reach the routed experts of MoE models from "
"`peft>=0.19.0`. Upgrade peft, or also target the expert parameters with `target_parameters` "
"(e.g. `target_parameters=['mlp.experts.gate_up_proj', 'mlp.experts.down_proj']`). See "
"https://github.com/huggingface/trl/issues/7765."
)
# With transformers v5, peft 0.19 to 0.21.2 skip the MoE dense and shared layers (#7767)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version("0.19.0") <= Version(peft.__version__) < Version("0.21.3.dev0")
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and any(
isinstance(module, torch.nn.Linear)
and name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name, module in model.named_modules()
)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
logger.warning(
"With transformers v5 and peft>=0.19.0,<0.21.3, LoRA `target_modules` skip the dense and shared "
"expert layers of MoE models, so only the routed experts are trained. Install peft from main to "
"train them too. See https://github.com/huggingface/trl/issues/7767."
)
model = get_peft_model(model, peft_config, **get_peft_model_kwargs)

# PEFT + DeepSpeed ZeRO-3 requires reentrant checkpointing. For more details, see
Expand Down
45 changes: 45 additions & 0 deletions trl/trainer/rloo_trainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,51 @@ def __init__(
"peft, or load the model with `device_map=None`. See "
"https://github.com/huggingface/peft/issues/2821."
)
# With transformers v5, peft<0.19 doesn't apply `target_modules` to the fused MoE routed experts (#7765)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version(peft.__version__) < Version("0.19.0")
and (Version(peft.__version__) < Version("0.17.0") or not peft_config.target_parameters)
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
raise ValueError(
"With transformers v5, LoRA `target_modules` only reach the routed experts of MoE models from "
"`peft>=0.19.0`. Upgrade peft, or also target the expert parameters with `target_parameters` "
"(e.g. `target_parameters=['mlp.experts.gate_up_proj', 'mlp.experts.down_proj']`). See "
"https://github.com/huggingface/trl/issues/7765."
)
# With transformers v5, peft 0.19 to 0.21.2 skip the MoE dense and shared layers (#7767)
if (
isinstance(peft_config, LoraConfig)
and Version(transformers.__version__) >= Version("5.0.0")
and Version("0.19.0") <= Version(peft.__version__) < Version("0.21.3.dev0")
and any(param.ndim == 3 for name, param in model.named_parameters() if ".experts." in name)
and any(
isinstance(module, torch.nn.Linear)
and name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name, module in model.named_modules()
)
and (
isinstance(peft_config.target_modules, str)
or any(
name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"}
for name in peft_config.target_modules or []
)
)
):
logger.warning(
"With transformers v5 and peft>=0.19.0,<0.21.3, LoRA `target_modules` skip the dense and shared "
"expert layers of MoE models, so only the routed experts are trained. Install peft from main to "
"train them too. See https://github.com/huggingface/trl/issues/7767."
)
model = get_peft_model(model, peft_config, **get_peft_model_kwargs)

elif is_peft_model(model):
Expand Down
Loading
Loading