[Bug] Megatron MoE LoRA with merge_lora=False fails the weight sync on Qwen3.6-35B MoE #1852

diff --git a/skyrl/backends/skyrl_train/workers/megatron/megatron_worker.py b/skyrl/backends/skyrl_train/workers/megatron/megatron_worker.py
index 75261d47..a7be56a1 100644
--- a/skyrl/backends/skyrl_train/workers/megatron/megatron_worker.py
+++ b/skyrl/backends/skyrl_train/workers/megatron/megatron_worker.py
@@ -86,6 +86,49 @@ from skyrl.backends.skyrl_train.workers.megatron.model_bridges import (
 )
 
 
+def _convert_moe_experts_lora_to_vllm(
+    adapter_state: Dict[str, "torch.Tensor"],
+) -> Dict[str, "torch.Tensor"]:
+    """Rewrite fused-MoE expert LoRA tensors into the layout vLLM expects.
+
+    Megatron-Bridge exports the fused experts as 3D (num_experts-leading)
+    tensors under two keys per layer::
+
+        ...mlp.experts.gate_up_proj.lora_{A,B}.weight   # w13
+        ...mlp.experts.down_proj.lora_{A,B}.weight       # w2
+
+    with shapes ``lora_A=(E, rank, in)`` and ``lora_B=(E, out, rank)``.
+
+    vLLM's fused-MoE LoRA loader for a 3D-MoE model
+    (``FusedMoE3DWithLoRA`` / ``LoRAModelManager._stack_moe_lora_weights``)
+    instead consumes the flat PEFT layout, keyed by the FusedMoE module name
+    itself for ``down_proj`` (w2) and ``<module>.base_layer`` for the packed
+    ``gate_up_proj`` (w13), with shapes ``lora_A=(rank*E, in)`` and
+    ``lora_B=(out, rank*E)``.  It then reconstructs the per-expert tensors via
+    ``reshape(E, -1, in)`` (lora_A) and ``reshape(out, -1, E).permute(2,0,1)``
+    (lora_B); this transform is the exact inverse, verified to round-trip.
+
+    Non-expert tensors (attention, shared_expert) are passed through unchanged.
+    """
+    converted: Dict[str, "torch.Tensor"] = {}
+    for key, tensor in adapter_state.items():
+        is_gate_up = ".mlp.experts.gate_up_proj." in key
+        is_down = ".mlp.experts.down_proj." in key
+        if (is_gate_up or is_down) and tensor.ndim == 3:
+            if key.endswith(".lora_A.weight"):
+                # (E, rank, in) -> (rank*E [expert-major], in)
+                tensor = tensor.reshape(-1, tensor.shape[-1]).contiguous()
+            elif key.endswith(".lora_B.weight"):
+                # (E, out, rank) -> (out, rank*E [expert-minor])
+                tensor = tensor.permute(1, 2, 0).contiguous().reshape(tensor.shape[1], -1)
+            if is_gate_up:
+                key = key.replace(".mlp.experts.gate_up_proj.", ".mlp.experts.base_layer.")
+            else:
+                key = key.replace(".mlp.experts.down_proj.", ".mlp.experts.")
+        converted[key] = tensor
+    return converted
+
+
 class MegatronWeightExtractor(WeightExtractor):
     """Extracts weights from Megatron model-parallel models.
 
@@ -1238,7 +1281,21 @@ class MegatronPolicyWorkerBase(MegatronWorker, PolicyWorkerBase):
         if torch.distributed.get_rank() == 0:
             os.makedirs(lora_sync_path, exist_ok=True)
 
-            target_modules = infer_target_modules_from_adapter_weights(adapter_state.keys())
+            # The bridge exports the fused-MoE expert LoRA as 3D
+            # (num_experts-leading) tensors keyed `...mlp.experts.gate_up_proj`
+            # / `...mlp.experts.down_proj`. vLLM's fused-MoE LoRA loader
+            # (FusedMoE3DWithLoRA / _stack_moe_lora_weights) instead expects the
+            # flat PEFT layout keyed `...mlp.experts.base_layer` (gate_up_proj /
+            # w13) and `...mlp.experts` (down_proj / w2), with lora_A shaped
+            # (rank*num_experts, in) and lora_B shaped (out, rank*num_experts).
+            # Rewrite the expert tensors so merge_lora=False on-policy sync is
+            # accepted (otherwise load_lora_adapter rejects `experts.down_proj`).
+            adapter_state = _convert_moe_experts_lora_to_vllm(adapter_state)
+
+            target_modules = sorted(
+                set(infer_target_modules_from_adapter_weights(adapter_state.keys()))
+                - {"base_layer"}
+            )
             base_model_name_or_path = str(
                 getattr(self.bridge.hf_pretrained, "model_name_or_path", "")
                 or getattr(self.bridge.hf_pretrained, "name_or_path", "")