# LoRA fine-tuning config for Qwen3-MoE.
#
# `Qwen3MoeExperts` stores fused `gate_up_proj` (3D, [num_experts, 2*intermediate, hidden])
# and `down_proj` (3D, [num_experts, hidden, intermediate]).
#
# `lora_config` schema (`veomni/trainer/base.py::_setup_lora`):
#   - `lora_modules`         — LoRA targets by module name. `q/k/v/o` etc. install a dense
#                              `LoraLinear`; the fused-MoE expert names `gate_proj` /
#                              `up_proj` / `down_proj` are auto-mapped to the experts'
#                              fused `target_parameters` and wrapped by VeOmni's MoE-LoRA.
#   - `target_parameters`    — (advanced) explicit glob patterns for the 3D expert params;
#                              equivalent to listing the semantic names above.
#   - `share_expert_lora`    — when True, expert LoRA uses `LoraSharedExperts` (single 2D
#                              LoRA pair shared across all experts in each layer, Mode 2)
#                              instead of the per-expert `LoraIndependentExperts` (Mode 1).
#
# Switch to Mode 1 (independent LoRA per expert) by setting `share_expert_lora: false`.

model:
  model_path: Qwen3-30B-A3B-merge
  ops_implementation:
    attn_implementation: flash_attention_2
    moe_implementation: eager
    cross_entropy_loss_implementation: eager
    rms_norm_implementation: eager
    swiglu_mlp_implementation: eager
    rotary_pos_emb_implementation: eager
  lora_config:
    rank: 16
    alpha: 32
    use_rslora: false
    lora_modules:
      - q_proj
      - k_proj
      - v_proj
      - o_proj
      - gate_proj
      - up_proj
      - down_proj
    share_expert_lora: true

data:
  train_path: tulu-3-sft-mixture
  data_type: conversation
  chat_template: chatml
  max_seq_len: 2048
  train_size: 750000
  text_keys: messages
  dyn_bsz_buffer_size: 200

train:
  accelerator:
    ulysses_size: 1
    fsdp_config:
      fsdp_mode: fsdp2
  gradient_checkpointing:
    enable: true
  global_batch_size: 8
  micro_batch_size: 1
  bsz_warmup_ratio: 0.007
  optimizer:
    type: adamw
    lr: 1.0e-4
    lr_decay_style: cosine
    max_grad_norm: 1.0
  num_train_epochs: 1
  init_device: meta
  enable_full_determinism: false
  empty_cache_steps: 500
  checkpoint:
    output_dir: Qwen3-30B-A3B-lora-tulu
    manager: dcp
    save_steps: 0
    save_epochs: 1
    save_hf_weights: true
  wandb:
    project: VeOmni
    name: Qwen3-30B-A3B-lora-tulu
