feat: add AIPerf multimodal benchmark integration

.gitignore

@@ -111,6 +111,8 @@ uv.lock
 .mypy_cache/
 
 # custom
+/artifacts/
+/benchmarks/aiperf/
 /models
 /data
 .vscode
@@ -142,6 +144,11 @@ docs/**/modelzoo.md
 *.MP3
 *.mp3
 
+# Source-controlled AIPerf benchmark profiles use JSON despite the global rule.
+!benchmarks/telefuser_aiperf/configs/*.json
+!benchmarks/telefuser_aiperf/data/stream_lingbot_controls.json
+!benchmarks/baseline/sglang_lingbot_stream/configs/*.json
+
 project
 output
 work_dirs

CLAUDE.md

@@ -60,7 +60,7 @@ telefuser/
 ├── feature_cache/    # Feature caching: AdaTaylorCache
 ├── cache/            # General cache management
 ├── offload/          # CPU offload strategies
-├── metrics/          # Metrics collection and monitoring
+├── metrics/          # Metrics collection, monitoring, and raw runtime facts
 ├── orchestrator/     # Request orchestration and actor-based streaming scheduler
 ├── worker/           # Distributed worker management
 ├── entrypoints/      # CLI entry points
@@ -124,6 +124,13 @@ TeleFuser's model follows a strict layered architecture for operations:
    - May use `torch.library.custom_op` for torch.compile compatibility
    - Only used by ops/ layer, never directly by models/
 
+### Benchmark Metrics Boundary
+
+- `telefuser/metrics/runtime.py` measures synchronized target-side phase duration and allocator peaks.
+- Target services emit only raw, bounded phase, chunk, and runtime facts. AIPerf owns warmup exclusion,
+  aggregation, semantic mapping, artifacts, and visualization.
+- Client delivery and target compute metrics remain distinct (`stream_fps` versus `chunk_compute_fps`).
+
 ## Code Style
 
 - PEP8 with ruff (line length: 120)
@@ -135,6 +142,7 @@ TeleFuser's model follows a strict layered architecture for operations:
 
 | Topic | English | Chinese |
 |-------|---------|---------|
+| AIPerf Benchmark | [docs/en/benchmark_aiperf.md](docs/en/benchmark_aiperf.md) | [docs/zh/benchmark_aiperf.md](docs/zh/benchmark_aiperf.md) |
 | Adding New Example | [docs/en/adding_new_example.md](docs/en/adding_new_example.md) | [docs/zh/adding_new_example.md](docs/zh/adding_new_example.md) |
 | Adding New Model | [docs/en/adding_new_model.md](docs/en/adding_new_model.md) | [docs/zh/adding_new_model.md](docs/zh/adding_new_model.md) |
 | Adding New Stage | [docs/en/adding_new_stage.md](docs/en/adding_new_stage.md) | [docs/zh/adding_new_stage.md](docs/zh/adding_new_stage.md) |

benchmarks/baseline/sglang_lingbot_stream/README.md

@@ -0,0 +1,53 @@
+# SGLang-Diffusion LingBot Stream Baseline
+
+This target compares TeleFuser LingBot streaming with the diffusion runtime in
+`sgl-project/sglang` (`sglang.multimodal_gen`). It uses WebSocket + MessagePack while
+TeleFuser uses WebRTC + DataChannel; AIPerf normalizes both into the same session and
+control timeline.
+
+## Requirements
+
+- a version-pinned SGLang checkout that provides `LingBotWorldCausalDMDPipeline`;
+- `robbyant/lingbot-world-fast-diffusers` or an equivalent local model path;
+- the AIPerf checkout prepared by `scripts/setup_aiperf_repo.sh`.
+
+The launcher does not monkeypatch SGLang internals. Missing dependencies or incompatible
+CUDA kernels must be fixed in the SGLang environment or recorded as a failed
+qualification, not hidden behind an unversioned shim.
+
+## Start the target
+
+```bash
+bash benchmarks/baseline/sglang_lingbot_stream/scripts/run_service.sh
+```
+
+Common overrides:
+
+```bash
+SGLANG_PYTHON=/path/to/venv/bin/python \
+SGLANG_LINGBOT_MODEL_PATH=/path/to/model \
+SGLANG_LINGBOT_NUM_GPUS=1 \
+SGLANG_LINGBOT_ULYSSES_DEGREE=1 \
+  bash benchmarks/baseline/sglang_lingbot_stream/scripts/run_service.sh
+```
+
+The default address is `http://127.0.0.1:30000`; readiness is checked at `/health`.
+
+## Run AIPerf
+
+```bash
+bash benchmarks/baseline/sglang_lingbot_stream/scripts/run_stream_bench.sh
+
+bash benchmarks/baseline/sglang_lingbot_stream/scripts/run_stream_bench.sh \
+  benchmarks/baseline/sglang_lingbot_stream/configs/stream_lingbot_world_fast_compare.json
+```
+
+The baseline reuses
+`benchmarks/telefuser_aiperf/data/stream_lingbot_controls.json`. AIPerf maps the shared
+directional controls to SGLang camera actions and keeps implementation-specific fields
+as raw evidence.
+
+For a valid performance comparison, use the same accelerator count, prompt, first
+frame, FPS, session window, control trace, dtype, attention/cache geometry, and offload
+policy. Record the exact SGLang and model revisions. Mock, native fallback, CPU offload,
+and layerwise offload runs require separate qualifications.

benchmarks/baseline/sglang_lingbot_stream/benchmark_contract.yaml

@@ -0,0 +1,104 @@
+# Benchmark contract example for a WebSocket stream-world baseline.
+# Enum comments use "one-of". List comments use "one-or-more".
+contract_version: v1  # one-of: v1. Bump only when the contract schema changes.
+name: sglang_lingbot_world_stream  # Stable benchmark target id used in reports and automation.
+mode: stream_world  # one-of: batch_video, stream_world.
+implementation: sglang_diffusion  # Example values: telefuser, diffusers, sglang_diffusion.
+model_family: lingbot_world_fast  # Example values: wan_video, lingbot_world_fast, hunyuan_video, ltx_video.
+model: robbyant/lingbot-world-fast-diffusers  # Concrete model or service profile under test.
+supported_tasks:  # one-or-more: t2v, i2v, ti2v, bidirectional.
+  - bidirectional
+transport: websocket  # one-of: http, http_polling, websocket, sse, webrtc.
+adapter: sglang_websocket  # Built-in AIPerf adapter; target repository owns no transport implementation.
+endpoint:
+  health_path: /health  # Service readiness path.
+  metadata_path: /v1/models  # Optional SGLang model and pipeline identity snapshot.
+  websocket_path: /v1/realtime_video/generate  # WebSocket path for realtime video sessions.
+  models_path: /v1/models  # Optional model metadata path.
+request_encoding:
+  message_format: msgpack  # one-of for this baseline: msgpack.
+  init_required_fields:  # one-or-more. Required fields in the first WebSocket message.
+    - type
+    - prompt
+    - first_frame
+    - size
+    - fps
+    - num_frames
+  init_parameters:  # Mapping from benchmark parameter names to SGLang init payload fields.
+    model: model
+    prompt: prompt
+    image_path: first_frame
+    size: size
+    fps: fps
+    num_frames: num_frames
+  control_channel:
+    transport: websocket_message  # one-of for WebSocket: websocket_message.
+    message_type: event  # Runtime control messages use type=event.
+    kind: camera_actions  # one-of for LingBot realtime controls: camera_actions, prompt.
+    payload_mode: state  # one-of for camera_actions: script, state.
+    action_tokens:  # one-or-more. SGLang LingBot camera action tokens.
+      - w
+      - a
+      - s
+      - d
+      - i
+      - j
+      - k
+      - l
+    key_mapping:
+      ArrowUp: w
+      ArrowDown: s
+      ArrowLeft: a
+      ArrowRight: d
+result_delivery:
+  media: websocket_frame_batch  # one-of for this baseline: websocket_frame_batch.
+  metadata: websocket_chunk_stats  # one-of for this baseline: websocket_chunk_stats.
+  session_log: sessions.jsonl  # Per-session result records.
+  event_log: events/{phase}_{logical_session_index}_{session_id}.jsonl  # Per-session event trace template.
+workload:
+  mode: bidirectional  # one-of: server_push, bidirectional.
+  task: bidirectional  # one-of for this service: bidirectional.
+  size: 832x480
+  fps: 16
+  session_count: 1
+  warmup_sessions: 1
+  session_duration_s: 90.0
+  control_trace: benchmarks/telefuser_aiperf/data/stream_lingbot_controls.json  # Timed control-event workload.
+  request_extra:  # Service-specific request config passed through to SGLang.
+    num_frames: 9
+    num_inference_steps: 4
+    guidance_scale: 1.0
+    realtime_causal_sink_size: 6
+    realtime_causal_kv_cache_num_frames: 9
+    realtime_output_format: webp  # one-of: webp, jpeg, raw.
+    output_compression: 95
+    max_chunks: 8
+metrics:  # one-or-more. Choose all metrics emitted by this benchmark mode.
+  - connected_latency_ms
+  - first_frame_latency_ms
+  - first_metadata_latency_ms
+  - stream_fps
+  - session_runtime_s
+  - frames_received
+  - control_ack_latency_ms
+  - control_to_next_frame_latency_ms
+  - chunk_request_prepare_seconds
+  - chunk_compute_seconds
+  - chunk_encode_seconds
+  - chunk_output_pacing_seconds
+  - chunk_output_header_write_seconds
+  - chunk_output_payload_write_seconds
+  - chunk_output_write_seconds
+  - chunk_total_seconds
+  - chunk_compute_fps
+  - chunk_raw_output_bytes
+  - chunk_wire_output_bytes
+  - chunk_output_batches
+  - chunk_peak_reserved_bytes
+  - success_rate
+limits:
+  active_sessions: 1  # Current SGLang realtime endpoint accepts one active session for this target.
+artifacts:  # Paths consumed by automation and documentation.
+  config: benchmarks/baseline/sglang_lingbot_stream/configs/stream_lingbot_world_fast_compare.json
+  control_trace: benchmarks/telefuser_aiperf/data/stream_lingbot_controls.json
+  runner: aiperf profile --stream-config

benchmarks/baseline/sglang_lingbot_stream/configs/stream_lingbot_world_fast_compare.json

@@ -0,0 +1,34 @@
+{
+  "contract": "benchmarks/baseline/sglang_lingbot_stream/benchmark_contract.yaml",
+  "server_url": "http://127.0.0.1:30000",
+  "mode": "bidirectional",
+  "task": "bidirectional",
+  "prompt": "walk forward through the scene",
+  "image_path": "examples/data/1.png",
+  "fps": 16,
+  "session_count": 1,
+  "warmup_sessions": 1,
+  "warmup_chunks": 1,
+  "session_duration_s": 90.0,
+  "stagger_s": 0.0,
+  "control_trace_path": "benchmarks/telefuser_aiperf/data/stream_lingbot_controls.json",
+  "request_extra": {
+    "num_frames": 9,
+    "num_inference_steps": 4,
+    "guidance_scale": 1.0,
+    "realtime_causal_sink_size": 6,
+    "realtime_causal_kv_cache_num_frames": 9,
+    "realtime_output_format": "webp",
+    "output_compression": 95,
+    "max_chunks": 8,
+    "realtime_output_pacing": false
+  },
+  "transport": {
+    "connect_timeout_s": 60.0,
+    "message_timeout_s": 180.0
+  },
+  "server_metrics": {
+    "enabled": false
+  },
+  "artifacts_dir": "artifacts/sglang_lingbot_stream/stream_lingbot_compare"
+}

benchmarks/baseline/sglang_lingbot_stream/configs/stream_lingbot_world_fast_quick.json

@@ -0,0 +1,33 @@
+{
+  "contract": "benchmarks/baseline/sglang_lingbot_stream/benchmark_contract.yaml",
+  "server_url": "http://127.0.0.1:30000",
+  "mode": "bidirectional",
+  "task": "bidirectional",
+  "prompt": "walk forward through the scene",
+  "image_path": "examples/data/1.png",
+  "fps": 16,
+  "session_count": 1,
+  "warmup_sessions": 0,
+  "session_duration_s": 12.0,
+  "stagger_s": 0.0,
+  "control_trace_path": "benchmarks/telefuser_aiperf/data/stream_lingbot_controls.json",
+  "request_extra": {
+    "num_frames": 9,
+    "num_inference_steps": 4,
+    "guidance_scale": 1.0,
+    "realtime_causal_sink_size": 6,
+    "realtime_causal_kv_cache_num_frames": 9,
+    "realtime_output_format": "webp",
+    "output_compression": 95,
+    "max_chunks": 8,
+    "realtime_output_pacing": false
+  },
+  "transport": {
+    "connect_timeout_s": 30.0,
+    "message_timeout_s": 120.0
+  },
+  "server_metrics": {
+    "enabled": false
+  },
+  "artifacts_dir": "artifacts/sglang_lingbot_stream/stream_lingbot_quick"
+}

benchmarks/baseline/sglang_lingbot_stream/scripts/run_service.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
+cd "${ROOT_DIR}"
+
+if [[ -z "${CUDA_HOME:-}" && -d /usr/local/cuda ]]; then
+    export CUDA_HOME=/usr/local/cuda
+fi
+
+if [[ -n "${SGLANG_EXTRA_PYTHONPATH:-}" ]]; then
+    export PYTHONPATH="${SGLANG_EXTRA_PYTHONPATH}${PYTHONPATH:+:${PYTHONPATH}}"
+fi
+
+SGLANG_BIN="${SGLANG_BIN:-sglang}"
+SGLANG_PYTHON="${SGLANG_PYTHON:-}"
+SERVICE_PORT="${SGLANG_LINGBOT_PORT:-30000}"
+MODEL_PATH="${SGLANG_LINGBOT_MODEL_PATH:-robbyant/lingbot-world-fast-diffusers}"
+MODEL_ID="${SGLANG_LINGBOT_MODEL_ID:-lingbot-world-fast-diffusers}"
+MODEL_TYPE="${SGLANG_LINGBOT_MODEL_TYPE:-diffusion}"
+PIPELINE_CLASS="${SGLANG_LINGBOT_PIPELINE_CLASS:-LingBotWorldCausalDMDPipeline}"
+PERFORMANCE_MODE="${SGLANG_LINGBOT_PERFORMANCE_MODE:-speed}"
+ATTENTION_BACKEND_CONFIG="${SGLANG_LINGBOT_ATTENTION_BACKEND_CONFIG:-VSA_sparsity=0.0}"
+NUM_GPUS="${SGLANG_LINGBOT_NUM_GPUS:-1}"
+ULYSSES_DEGREE="${SGLANG_LINGBOT_ULYSSES_DEGREE:-1}"
+DIT_CPU_OFFLOAD="${SGLANG_LINGBOT_DIT_CPU_OFFLOAD:-false}"
+TEXT_ENCODER_CPU_OFFLOAD="${SGLANG_LINGBOT_TEXT_ENCODER_CPU_OFFLOAD:-false}"
+
+if [[ -n "${SGLANG_PYTHON}" ]]; then
+    SGLANG_CMD=("${SGLANG_PYTHON}" -c "from sglang.cli.main import main; main()")
+else
+    read -r -a SGLANG_CMD <<< "${SGLANG_BIN}"
+fi
+
+exec "${SGLANG_CMD[@]}" serve \
+    --model-type "${MODEL_TYPE}" \
+    --model-path "${MODEL_PATH}" \
+    --model-id "${MODEL_ID}" \
+    --pipeline-class-name "${PIPELINE_CLASS}" \
+    --performance-mode "${PERFORMANCE_MODE}" \
+    --attention-backend-config "${ATTENTION_BACKEND_CONFIG}" \
+    --port "${SERVICE_PORT}" \
+    --num-gpus "${NUM_GPUS}" \
+    --ulysses-degree "${ULYSSES_DEGREE}" \
+    --dit-cpu-offload "${DIT_CPU_OFFLOAD}" \
+    --text-encoder-cpu-offload "${TEXT_ENCODER_CPU_OFFLOAD}" \
+    "$@"

benchmarks/baseline/sglang_lingbot_stream/scripts/run_stream_bench.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
+cd "${ROOT_DIR}"
+
+AIPERF_DIR="${ROOT_DIR}/benchmarks/aiperf"
+UV_BIN="${AIPERF_UV_BIN:-uv}"
+CONFIG_PATH="${1:-benchmarks/baseline/sglang_lingbot_stream/configs/stream_lingbot_world_fast_quick.json}"
+if [[ $# -gt 0 ]]; then
+    shift
+fi
+
+if [[ ! -f "${AIPERF_DIR}/pyproject.toml" ]]; then
+    echo "AIPerf checkout not found. Run: bash scripts/setup_aiperf_repo.sh" >&2
+    exit 1
+fi
+if ! command -v "${UV_BIN}" >/dev/null 2>&1; then
+    echo "uv is required: https://docs.astral.sh/uv/getting-started/installation/" >&2
+    exit 1
+fi
+
+SERVER_URL="${SGLANG_STREAM_BENCH_URL:-http://127.0.0.1:30000}"
+SERVER_ARGS=(--stream-server-url "${SERVER_URL}")
+for argument in "$@"; do
+    if [[ "${argument}" == "--stream-server-url" || "${argument}" == --stream-server-url=* ]]; then
+        SERVER_ARGS=()
+        break
+    fi
+done
+
+RESOURCE_ARGS=()
+RESOURCE_HISTORY_URL="${AIPERF_HISTORY_URL:-}"
+RESOURCE_TARGET_PID="${AIPERF_RESOURCE_TARGET_PID:-${SGLANG_STREAM_BENCH_PID:-}}"
+if [[ -n "${RESOURCE_HISTORY_URL}" || -n "${RESOURCE_TARGET_PID}" ]]; then
+    if [[ -z "${RESOURCE_HISTORY_URL}" || -z "${RESOURCE_TARGET_PID}" ]]; then
+        echo "AIPERF_HISTORY_URL and AIPERF_RESOURCE_TARGET_PID must be set together" >&2
+        exit 2
+    fi
+    RESOURCE_ARGS+=(--stream-resource-history-url "${RESOURCE_HISTORY_URL}")
+    RESOURCE_ARGS+=(--stream-resource-target-pid "${RESOURCE_TARGET_PID}")
+fi
+
+exec "${UV_BIN}" run --frozen --no-dev --project "${AIPERF_DIR}" \
+    aiperf profile \
+    --stream-config "${CONFIG_PATH}" \
+    "${SERVER_ARGS[@]}" \
+    "${RESOURCE_ARGS[@]}" \
+    "$@"