diff --git a/model_qwen4b/check_perlayer_mse.py b/model_qwen4b/check_perlayer_mse.py index 1af2b53..b95f982 100644 --- a/model_qwen4b/check_perlayer_mse.py +++ b/model_qwen4b/check_perlayer_mse.py @@ -40,6 +40,13 @@ def pre(mod, args): def post(mod, args, output): x = captured.pop("x") + # Qwen3 MoE: experts that receive 0 routed tokens in this step get + # an empty input. F.mse_loss handles numel()==0 cleanly, but + # `.abs().max()` and `.abs().mean()` blow up on empty tensors + # ("Expected reduction dim to be specified for input.numel() == 0"). + # Skip — there is no SC error to record for an unused expert call. + if x.numel() == 0: + return with torch.no_grad(): ref = F.linear(x.to(weight.dtype), weight, bias).float() out = output.float()