Add online-live mode: train drafters from production serving traffic

examples/configs/README.md

@@ -47,6 +47,13 @@ The `qwen3-8b-dflash-1server-dp7-disaggregated.yaml`,
 full-stack examples. Their typed `managed_local` blocks own Mooncake, one or
 two patched SGLang capture servers, and the trainer GPU allocation; the same
 `specforge train -c ...` command starts and cleans up each complete stack.
+The `qwen3-4b-dspark-live.yaml` recipe is the online-live example: external
+serving traffic drives capture (`deployment.disaggregated.live`). With
+`live.mooncake` set, one `specforge train` command supervises Mooncake +
+producer + consumer; `scripts/online_live/launch_capture_server.py` launches
+the live-patched SGLang server with flags derived from the same config, and
+`scripts/online_live/live_traffic_mimic.py` stands in for real users.
+
 Disaggregated recipes without `managed_local` keep Mooncake and SGLang external
 for scheduler- or service-managed deployments.
 
@@ -298,6 +305,11 @@ For `deployment.mode: disaggregated`, also write:
 | `deployment.disaggregated.producer_hold_s` | `null` | Optional positive offline producer retention timeout. Unset is unbounded; expiration fails the attempt. |
 | `deployment.disaggregated.shutdown_grace_s` | `30.0` | SIGTERM-to-SIGKILL window for a plain supervisor teardown; must cover worker cleanup (Mooncake drains, checkpoint flush, failure sentinels). `managed_local` stacks use `managed_local.shutdown_grace_s`. |
 | `deployment.disaggregated.managed_local` | `null` | Optional owned single-node Mooncake + capture-server stack described below. |
+| `deployment.disaggregated.live` | `null` | Online-live mode: external serving-traffic capture servers push records to a producer-hosted intake endpoint. Mutually exclusive with `managed_local` and `server_urls`; requires `training.max_steps` (or `total_steps`) and no data source. |
+| `deployment.disaggregated.live.host` | `0.0.0.0` | Intake bind host on the producer. |
+| `deployment.disaggregated.live.port` | required | Intake bind port; capture servers point `--spec-capture-intake-url` at it. |
+| `deployment.disaggregated.live.mooncake` | `null` | Optional supervisor-owned loopback Mooncake master (same shape as `managed_local.mooncake`): one `specforge train` command launches Mooncake + producer + consumer. Set together with `trainer_cuda_visible_devices`. |
+| `deployment.disaggregated.live.trainer_cuda_visible_devices` | `null` | Consumer CUDA devices for the supervised live launch; count must equal `deployment.trainer.nproc_per_node`. |
 
 The four path fields have different ownership:
 

examples/configs/qwen3-4b-dspark-disaggregated.yaml

@@ -12,6 +12,9 @@ training:
   strategy: dspark
   num_epochs: 6
   max_steps: 10000
+  # Data-parallel over 7 ranks (global batch 7, accumulation 1). max_steps
+  # binds long before the epoch cap, so runtime = 10000 x per-step time; the
+  # extra ranks cut per-step time rather than adding sequential work.
   batch_size: 1
   learning_rate: 0.0006
   warmup_ratio: 0.04
@@ -20,21 +23,46 @@ training:
   loss_decay_gamma: 4.0
   objective_chunk_blocks: 128
   save_interval: 1000
+  log_interval: 10
   dist_timeout: 30
   seed: 42
+tracking:
+  # WANDB_API_KEY is supplied via the process environment, never YAML.
+  report_to: wandb
+  wandb_project: specforge
+  wandb_name: qwen3-4b-dspark-disaggregated
 run_id: qwen3-4b-dspark-disaggregated
 output_dir: outputs/qwen3-4b-dspark-disaggregated
 
 deployment:
   mode: disaggregated
   trainer:
     nnodes: 1
-    nproc_per_node: 1
+    nproc_per_node: 7
   disaggregated:
     control_dir: outputs/qwen3-4b-dspark-disaggregated/control
     backend: mooncake
-    server_urls:
-      - http://127.0.0.1:30000
-    mooncake_metadata_server: http://127.0.0.1:35880/metadata
-    mooncake_master_server_addr: 127.0.0.1:35551
-    mooncake_protocol: tcp
+    managed_local:
+      trainer_cuda_visible_devices:
+        - "1"
+        - "2"
+        - "3"
+        - "4"
+        - "5"
+        - "6"
+        - "7"
+      mooncake:
+        # Loopback TCP; RDMA loopback is broken in this KVM guest (transport
+        # retry counter exceeded even intra-HCA) and NVLink does not apply to
+        # the host-RAM feature buffer.
+        protocol: tcp
+        global_segment_size_bytes: 34359738368
+        local_buffer_size_bytes: 1073741824
+      capture_servers:
+        # A 4B target generates samples far faster than the trainer consumes
+        # them, so one capture GPU is enough; the other 7 GPUs go to training.
+        - port: 30000
+          cuda_visible_devices:
+            - "0"
+          tp_size: 1
+          mem_fraction_static: 0.7

examples/configs/qwen3-4b-dspark-live.yaml

@@ -0,0 +1,72 @@
+# Online-live DSpark training from real serving traffic.
+#
+# Unlike the driven qwen3-4b-dspark-disaggregated recipe, the serving engine
+# and its traffic stay external: users (or the mimic script) send ordinary
+# /generate requests, the live-patched server captures prefill + generated
+# tokens, and SpecForge trains on them. live.mooncake makes the launcher a
+# supervisor that owns Mooncake master + producer + consumer, so the whole
+# workflow is three commands (one terminal each):
+#
+#   1. specforge train --config examples/configs/qwen3-4b-dspark-live.yaml
+#   2. python scripts/online_live/launch_capture_server.py \
+#        --config examples/configs/qwen3-4b-dspark-live.yaml --cuda 0
+#   3. python scripts/online_live/live_traffic_mimic.py \
+#        --server-url http://127.0.0.1:30000 \
+#        --config examples/configs/qwen3-4b-dspark-live.yaml
+model:
+  target_model_path: Qwen/Qwen3-4B
+  draft_model_config: configs/qwen3-4b-dspark.json
+  target_backend: sglang
+data:
+  # No training data source: samples come from serving traffic. max_length is
+  # the intake token cap (prompt + generated tokens); longer captures are shed.
+  max_length: 3072
+  chat_template: qwen
+  cache_dir: cache
+training:
+  strategy: dspark
+  max_steps: 10000
+  batch_size: 1
+  learning_rate: 0.0006
+  warmup_ratio: 0.04
+  max_grad_norm: 1.0
+  num_anchors: 512
+  loss_decay_gamma: 4.0
+  objective_chunk_blocks: 128
+  save_interval: 1000
+  log_interval: 10
+  dist_timeout: 30
+  seed: 42
+tracking:
+  # WANDB_API_KEY is supplied via the process environment, never YAML.
+  report_to: wandb
+  wandb_project: specforge
+  wandb_name: qwen3-4b-dspark-live
+run_id: qwen3-4b-dspark-live
+output_dir: outputs/qwen3-4b-dspark-live
+
+deployment:
+  mode: disaggregated
+  trainer:
+    nnodes: 1
+    nproc_per_node: 7
+  disaggregated:
+    control_dir: outputs/qwen3-4b-dspark-live/control
+    backend: mooncake
+    live:
+      host: 0.0.0.0
+      port: 8600
+      # Supervisor-owned loopback Mooncake master; the pool size is the live
+      # capture buffer (write-until-full, trainer acks free space).
+      mooncake:
+        protocol: tcp
+        global_segment_size_bytes: 34359738368
+        local_buffer_size_bytes: 1073741824
+      trainer_cuda_visible_devices:
+        - "1"
+        - "2"
+        - "3"
+        - "4"
+        - "5"
+        - "6"
+        - "7"

examples/configs/qwen3.6-27b-dspark-disaggregated.yaml

@@ -15,9 +15,15 @@ data:
 training:
   strategy: dspark
   num_epochs: 10
-  # Portable one-rank equivalent of the source recipe's global batch 512.
+  # Global batch ~512 as in the source recipe: 6 trainer ranks x batch 1 x
+  # accumulation 85 = 510. Measured balance: one capture server sustains
+  # ~7 samples/s while 7 trainer ranks consumed ~23 samples/s of compute,
+  # so a GPU moved from the trainer to a second capture server.
   batch_size: 1
-  accumulation_steps: 512
+  accumulation_steps: 85
+  # One optimizer step is ~minutes here; log every step so WandB shows
+  # progress immediately (default 50 would mean hours to the first point).
+  log_interval: 1
   learning_rate: 0.0006
   warmup_ratio: 0.04
   max_grad_norm: 1.0
@@ -27,21 +33,42 @@ training:
   save_interval: 125
   dist_timeout: 30
   seed: 42
+runtime:
+  # The consumer dispatches one full optimizer window at a time
+  # (nnodes * nproc_per_node * batch_size * accumulation_steps = 510),
+  # so both watermarks must be at least that quantum.
+  in_flight_high_watermark: 576
+  in_flight_low_watermark: 512
+tracking:
+  # WANDB_API_KEY is supplied via the process environment, never YAML.
+  report_to: wandb
+  wandb_project: specforge
+  wandb_name: qwen3.6-27b-dspark-disaggregated
 run_id: qwen3.6-27b-dspark-disaggregated
 output_dir: outputs/qwen3.6-27b-dspark-disaggregated
 
 deployment:
   mode: disaggregated
   trainer:
     nnodes: 1
-    nproc_per_node: 1
+    nproc_per_node: 6
   disaggregated:
     control_dir: outputs/qwen3.6-27b-dspark-disaggregated/control
     backend: mooncake
     managed_local:
       trainer_cuda_visible_devices:
-        - "1"
+        - "2"
+        - "3"
+        - "4"
+        - "5"
+        - "6"
+        - "7"
       mooncake:
+        # RDMA was tried here (mlx5_4/mlx5_5) but this KVM guest's IB fabric
+        # rejects same-host loopback: every slice dies with "transport retry
+        # counter exceeded", even intra-HCA. Loopback TCP (pinned via
+        # MC_TCP_BIND_ADDRESS in launch_plan) is the working transport for
+        # single-node runs; use RDMA only for true multi-node deployments.
         protocol: tcp
         global_segment_size_bytes: 68719476736
         local_buffer_size_bytes: 1073741824
@@ -51,3 +78,8 @@ deployment:
             - "0"
           tp_size: 1
           mem_fraction_static: 0.7
+        - port: 30001
+          cuda_visible_devices:
+            - "1"
+          tp_size: 1
+          mem_fraction_static: 0.7