# Qwen3.5-4B DFlash draft, offline (features captured to disk) — AMD ROCm.
# Trainer runs on ROCm unchanged (flex_attention). Capture the hidden states with
# scripts/prepare_hidden_states.py using the AITER + --sglang-disable-radix-cache
# flags (see docs/basic_usage/AMD/amd_rocm.md, Section 3).
model:
  target_model_path: "Qwen/Qwen3.5-4B"
  draft_model_config: "configs/qwen3.5-4b-dflash.json"
  target_backend: "sglang"
  trust_remote_code: true
  embedding_key: "model.language_model.embed_tokens.weight"
  torch_dtype: "bfloat16"
data:
  hidden_states_path: "./cache/hidden_states/qwen3.5-4b-dflash-sharegpt"
  max_length: 2048
  chat_template: "qwen3.5"
  cache_dir: "./cache"
training:
  strategy: "dflash"
  num_epochs: 10
  max_steps: 10000
  batch_size: 2
  accumulation_steps: 4
  learning_rate: 6.0e-4
  warmup_ratio: 0.04
  max_grad_norm: 1
  attention_backend: "flex_attention"
  num_anchors: 512
  loss_decay_gamma: 7
  save_interval: 1000
  log_interval: 50
  dist_timeout: 30
  seed: 42
tracking:
  report_to: "tensorboard"
run_id: "qwen3.5-4b-dflash-offline"
output_dir: "./outputs/qwen3.5-4b-dflash-offline"

deployment:
  mode: local_colocated
  trainer:
    nnodes: 1
    nproc_per_node: 1