From 991d5b8bb4ba380ac78d07085e6404bba82d26be Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 07:55:19 +0000 Subject: [PATCH 01/35] feat: add llama_server loader (Stage B1 of the parser-migration roadmap) Drives llama-server as a subprocess over its native OpenAI-compatible HTTP API instead of binding llama.cpp in-process, unlocking real concurrent request handling that the llama_cpp loader's per-instance lock couldn't offer. Reuses LlamaCppPreflight's GGUF/RAM-budget sizing, adjusted for llama-server's --parallel slot splitting. --- modelship/infer/infer_config.py | 27 +- .../infer/llama_server/llama_server_infer.py | 467 ++++++++++++++++++ modelship/infer/model_deployment.py | 4 + modelship/preflight/base.py | 7 + modelship/preflight/llama_cpp.py | 39 ++ pyproject.toml | 1 + tests/test_config.py | 61 +++ tests/test_integration.py | 251 ++++++++++ tests/test_llama_server_infer.py | 316 ++++++++++++ tests/test_preflight_llama_server.py | 110 +++++ 10 files changed, 1280 insertions(+), 3 deletions(-) create mode 100644 modelship/infer/llama_server/llama_server_infer.py create mode 100644 tests/test_llama_server_infer.py create mode 100644 tests/test_preflight_llama_server.py diff --git a/modelship/infer/infer_config.py b/modelship/infer/infer_config.py index fab456b..a02d4cf 100644 --- a/modelship/infer/infer_config.py +++ b/modelship/infer/infer_config.py @@ -44,6 +44,7 @@ class ModelLoader(StrEnum): transformers = "transformers" diffusers = "diffusers" llama_cpp = "llama_cpp" + llama_server = "llama_server" stable_diffusion_cpp = "stable_diffusion_cpp" custom = "custom" @@ -163,6 +164,24 @@ class LlamaCppConfig(BaseModel): cache: LlamaCppCacheConfig | None = None +class LlamaServerConfig(BaseModel): + """Tunables for the ``llama_server`` loader, which drives a `llama-server` + subprocess over its native OpenAI-compatible HTTP API rather than binding + llama.cpp in-process (that's `llama_cpp`/`LlamaCppConfig`).""" + + n_ctx: int = 2048 + n_batch: int = 512 + n_gpu_layers: int = -1 + # Concurrent request slots. llama-server splits its total context (`-c`) + # across slots, so the process is launched with `n_ctx * parallel`. + parallel: int = 1 + # Built-in template name (e.g. "chatml") or a path to a Jinja file; + # None lets llama-server use the GGUF's embedded chat template. + chat_template: str | None = None + # Escape hatch for launch flags not otherwise surfaced, appended verbatim. + extra_args: list[str] = Field(default_factory=list) + + class StableDiffusionCppConfig(BaseModel): """Tunables for the CPU-only `stable_diffusion_cpp` image loader (stable-diffusion.cpp via stable-diffusion-cpp-python). `sample_steps` and @@ -258,6 +277,7 @@ class ModelshipModelConfig(BaseModel): transformers_config: TransformersConfig | None = None diffusers_config: DiffusersConfig | None = None llama_cpp_config: LlamaCppConfig | None = None + llama_server_config: LlamaServerConfig | None = None stable_diffusion_cpp_config: StableDiffusionCppConfig | None = None plugin_config: dict[str, Any] | None = None # plugin devs parse this themselves # Extra variables forwarded verbatim into the chat-template Jinja render on @@ -332,10 +352,11 @@ def check_disk_cache_single_replica(self): @model_validator(mode="after") def validate_llama_cpp_num_gpus(self): # llama.cpp has no VRAM-fraction knob, so a fractional GPU share can't be - # honored — require whole GPUs (or 0 for CPU). - if self.loader == ModelLoader.llama_cpp and self.num_gpus != int(self.num_gpus): + # honored — require whole GPUs (or 0 for CPU). Applies equally to + # llama_server, which drives the same llama.cpp engine out of process. + if self.loader in (ModelLoader.llama_cpp, ModelLoader.llama_server) and self.num_gpus != int(self.num_gpus): raise ValueError( - f"num_gpus={self.num_gpus!r} is not allowed for the llama_cpp loader: " + f"num_gpus={self.num_gpus!r} is not allowed for the {self.loader.value} loader: " f"use an integer number of whole GPUs, or 0 for CPU. Fractional GPU " f"sharing isn't supported (llama.cpp has no GPU-memory fraction control)." ) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py new file mode 100644 index 0000000..252d29f --- /dev/null +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -0,0 +1,467 @@ +import asyncio +import contextlib +import json +import os +import secrets +import socket +import subprocess +import time +from collections import deque +from collections.abc import AsyncGenerator +from typing import Any + +import httpx + +from modelship.infer.base_infer import BaseInfer +from modelship.infer.infer_config import LlamaServerConfig, ModelshipModelConfig, RawRequestProxy +from modelship.logging import TRACE, get_logger +from modelship.openai.chat_utils import UnsupportedContentError, normalize_chat_messages +from modelship.openai.protocol import ( + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResponseChoice, + ChatCompletionResponseStreamChoice, + ChatCompletionStreamResponse, + ChatMessage, + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, + ErrorResponse, + FunctionCall, + ToolCall, + UsageInfo, + create_error_response, +) +from modelship.preflight import discover_hardware, merge_with_user_overrides, run_preflight +from modelship.utils import base_request_id, random_uuid + +logger = get_logger("infer.llama_server") + +_HEALTH_POLL_INTERVAL_S = 0.5 +_STARTUP_TIMEOUT_S = float(os.environ.get("MSHIP_LLAMA_SERVER_STARTUP_TIMEOUT", "900")) +# A child that dies this soon after spawn is presumed to be the free-port +# TOCTOU race (port stolen between our probe close() and its bind()), not a +# real load failure — retry the whole launch with a fresh port. +_EARLY_CRASH_WINDOW_S = 3.0 +_LAUNCH_RETRY_LIMIT = 5 +_RECENT_LOG_LINES = 50 + + +def _free_port() -> int: + """Bind an ephemeral port and release it immediately so the child can bind + it. Racy (TOCTOU) by nature: something else may grab the port before the + child does. The caller retries the whole launch when that happens.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class _EarlyCrashError(RuntimeError): + """The llama-server child exited within `_EARLY_CRASH_WINDOW_S` of spawn — + treated as a transient bind race and retried with a fresh port.""" + + +class LlamaServerInfer(BaseInfer): + """Drives a `llama-server` subprocess over its native OpenAI-compatible + HTTP API. Unlike `llama_cpp` (in-process `Llama` bindings + modelship's + own tool-call/reasoning parsers), llama-server does its own chat + templating, tool-call, and reasoning parsing — this loader is a thin, + concurrency-safe proxy that projects its responses onto modelship's + protocol models (never relaying its JSON/SSE verbatim, which would leak + llama.cpp-only extension fields like `timings` to clients).""" + + def __init__(self, model_config: ModelshipModelConfig): + super().__init__(model_config) + user_config = model_config.llama_server_config or LlamaServerConfig() + user_overrides = user_config.model_dump(exclude_unset=True) + + recommendation = run_preflight(model_config, discover_hardware()) + if recommendation: + logger.info("preflight recommendation for '%s': %s", model_config.name, recommendation) + else: + logger.info("preflight recommendation for '%s': none", model_config.name) + merged = merge_with_user_overrides(recommendation, user_overrides, model_name=model_config.name) + self.config = user_config.model_copy(update=merged) + + self._proc: subprocess.Popen | None = None + self._port: int | None = None + self._api_key: str = secrets.token_hex(32) + self._client: httpx.AsyncClient | None = None + self._recent_log_lines: deque[str] = deque(maxlen=_RECENT_LOG_LINES) + self._log_tasks: list[asyncio.Future] = [] + + def shutdown(self) -> None: + if self._proc is not None and self._proc.poll() is None: + logger.info("Shutting down llama-server for %s", self.model_config.name) + self._proc.terminate() + try: + self._proc.wait(timeout=10) + except subprocess.TimeoutExpired: + self._proc.kill() + self._proc = None + for task in self._log_tasks: + task.cancel() + self._log_tasks = [] + self._client = None + + def __del__(self): + self.shutdown() + + async def start(self) -> None: + binary = os.environ.get("MSHIP_LLAMA_SERVER_BIN") + if not binary or not os.path.isfile(binary): + raise ValueError( + f"llama_server deployment '{self.model_config.name}' requires MSHIP_LLAMA_SERVER_BIN " + f"to point at a llama-server executable; got {binary!r}. See docs/development.md." + ) + + model_path = self.model_config._resolved_path + if not model_path: + raise ValueError( + f"LlamaServer deployment '{self.model_config.name}' is missing a resolved model path. " + f"Check driver logs for resolution errors." + ) + + logger.info("Starting llama-server for model: %s", self.model_config.name) + last_error: Exception | None = None + for attempt in range(1, _LAUNCH_RETRY_LIMIT + 1): + try: + await self._launch(binary, model_path) + await self._wait_healthy() + break + except _EarlyCrashError as e: + last_error = e + logger.warning( + "llama-server for '%s' exited immediately on attempt %d/%d (likely a port race); retrying: %s", + self.model_config.name, + attempt, + _LAUNCH_RETRY_LIMIT, + e, + ) + else: + raise RuntimeError( + f"llama-server for '{self.model_config.name}' failed to start after " + f"{_LAUNCH_RETRY_LIMIT} attempts: {last_error}" + ) + + assert self._port is not None + self._client = httpx.AsyncClient( + base_url=f"http://127.0.0.1:{self._port}", + headers={"Authorization": f"Bearer {self._api_key}"}, + timeout=httpx.Timeout(timeout=None, connect=10.0), + ) + self._set_max_context_length(self.config.n_ctx) + + async def _launch(self, binary: str, model_path: str) -> None: + loop = asyncio.get_event_loop() + port = await loop.run_in_executor(None, _free_port) + self._port = port + + args = [ + binary, + "--host", + "127.0.0.1", + "--port", + str(port), + "-m", + model_path, + "-c", + str(self.config.n_ctx * self.config.parallel), + "-b", + str(self.config.n_batch), + "--parallel", + str(self.config.parallel), + "--jinja", + "--reasoning-format", + "auto", + "--no-webui", + "--api-key", + self._api_key, + ] + if self.config.chat_template: + flag = "--chat-template-file" if os.path.isfile(self.config.chat_template) else "--chat-template" + args += [flag, self.config.chat_template] + args += list(self.config.extra_args) + + logger.info("llama-server launch args for '%s': %s", self.model_config.name, _redact(args)) + self._proc = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + self._recent_log_lines.clear() + assert self._proc.stdout is not None + assert self._proc.stderr is not None + self._log_tasks = [ + loop.run_in_executor(None, self._drain_stream, self._proc.stdout, "stdout"), + loop.run_in_executor(None, self._drain_stream, self._proc.stderr, "stderr"), + ] + + def _drain_stream(self, stream: Any, tag: str) -> None: + """Consume a pipe to TRACE-level logs so the child never blocks on a + full pipe buffer during chatty load-time output. Runs in a worker + thread; `stream.readline()` returns '' at EOF (including when the + child dies and we close the pipe from the main thread).""" + try: + for raw_line in iter(stream.readline, ""): + line = raw_line.rstrip("\n") + logger.log(TRACE, "[%s:%s] %s", self.model_config.name, tag, line) + self._recent_log_lines.append(line) + except ValueError: + pass # stream closed from the main thread while we were reading + finally: + with contextlib.suppress(Exception): + stream.close() + + async def _wait_healthy(self) -> None: + assert self._proc is not None + assert self._port is not None + spawned_at = time.monotonic() + deadline = spawned_at + _STARTUP_TIMEOUT_S + + async with httpx.AsyncClient( + base_url=f"http://127.0.0.1:{self._port}", + headers={"Authorization": f"Bearer {self._api_key}"}, + ) as probe: + while True: + rc = self._proc.poll() + if rc is not None: + tail = "\n".join(self._recent_log_lines) + message = f"llama-server for '{self.model_config.name}' exited (rc={rc}) during startup: {tail}" + if time.monotonic() - spawned_at < _EARLY_CRASH_WINDOW_S: + raise _EarlyCrashError(message) + raise RuntimeError(message) + + if time.monotonic() > deadline: + raise RuntimeError( + f"llama-server for '{self.model_config.name}' did not become healthy within " + f"{_STARTUP_TIMEOUT_S}s" + ) + + try: + resp = await probe.get("/health", timeout=2.0) + if resp.status_code == 200: + logger.info("llama-server healthy for '%s' on port %d", self.model_config.name, self._port) + return + except httpx.TransportError: + pass + await asyncio.sleep(_HEALTH_POLL_INTERVAL_S) + + async def warmup(self) -> None: + logger.info("Warming up llama-server chat model: %s", self.model_config.name) + request = ChatCompletionRequest( + model=self.model_config.name, + messages=[{"role": "user", "content": "hi"}], + max_tokens=1, + ) + result = await self.create_chat_completion(request, RawRequestProxy(None, {})) + if isinstance(result, ErrorResponse): + logger.warning("warmup chat failed for '%s': %s", self.model_config.name, result.error.message) + else: + logger.info("warmup chat done for %s", self.model_config.name) + + async def create_chat_completion( + self, request: ChatCompletionRequest, raw_request: RawRequestProxy + ) -> ErrorResponse | ChatCompletionResponse | AsyncGenerator[str, None]: + if self._client is None: + return await super().create_chat_completion(request, raw_request) + + request_id = f"chat-{base_request_id(raw_request)}" + logger.info("chat completion request %s: stream=%s", request_id, request.stream) + logger.log( + TRACE, + "chat request %s: messages=%s tools=%s tool_choice=%s", + request_id, + request.messages, + request.tools, + request.tool_choice, + ) + + try: + messages = normalize_chat_messages(request.messages, supports_image=False, supports_audio=False) + except UnsupportedContentError as e: + logger.warning("chat request %s rejected: %s", request_id, e) + return create_error_response(e) + + payload = _build_payload(request, messages, model_name=self.model_config.name) + + if request.stream: + return self._stream_chat_completion(payload, request_id) + + try: + resp = await self._client.post("/v1/chat/completions", json=payload) + resp.raise_for_status() + except httpx.HTTPStatusError as e: + detail = _extract_error_detail(e.response) + logger.warning("chat request %s failed: %s", request_id, detail) + return create_error_response(detail, status_code=e.response.status_code) + except httpx.TransportError as e: + logger.warning("chat request %s failed: %s", request_id, e) + return create_error_response(f"llama-server request failed: {e}", status_code=502) + + data = resp.json() + logger.log(TRACE, "chat response %s: %s", request_id, data) + return _project_chat_response(data, model_name=self.model_config.name) + + async def _stream_chat_completion(self, payload: dict[str, Any], request_id: str) -> AsyncGenerator[str, None]: + assert self._client is not None + buffered: list[str] = [] + try: + async with self._client.stream("POST", "/v1/chat/completions", json=payload) as resp: + try: + resp.raise_for_status() + except httpx.HTTPStatusError as e: + await e.response.aread() + detail = _extract_error_detail(e.response) + logger.warning("chat request %s failed: %s", request_id, detail) + yield _encode_error(detail) + yield "data: [DONE]\n\n" + return + + async for line in resp.aiter_lines(): + if not line.startswith("data:"): + continue + data_str = line[len("data:") :].strip() + if data_str == "[DONE]": + break + try: + data = json.loads(data_str) + except json.JSONDecodeError: + logger.warning("chat request %s: unparseable stream chunk: %r", request_id, data_str) + continue + chunk = _project_stream_chunk(data, model_name=self.model_config.name) + for choice in chunk.choices: + if choice.delta.content: + buffered.append(choice.delta.content) + yield _encode_chunk(chunk) + yield "data: [DONE]\n\n" + finally: + logger.log(TRACE, "chat response %s (stream): %r", request_id, "".join(buffered)) + + +# --------------------------------------------------------------------------- +# Request / response projection — never relay llama-server's JSON verbatim, +# it's `extra="allow"`-shaped and would leak extension fields (e.g. `timings`). +# --------------------------------------------------------------------------- + + +def _redact(args: list[str]) -> list[str]: + redacted = list(args) + for i, arg in enumerate(redacted): + if arg == "--api-key" and i + 1 < len(redacted): + redacted[i + 1] = "***" + return redacted + + +def _build_payload(request: ChatCompletionRequest, messages: list[dict], *, model_name: str) -> dict[str, Any]: + # logprobs/top_logprobs aren't wired into the response projection yet (B3 + # adds that). ChatCompletionRequest defaults top_logprobs to 0 (not None), + # so exclude_none alone would still forward it — and llama-server rejects + # top_logprobs being present at all unless logprobs=true, even at 0. + payload = request.model_dump(exclude_none=True, exclude={"messages", "model", "logprobs", "top_logprobs"}) + payload["messages"] = messages + payload["model"] = model_name + return payload + + +def _extract_error_detail(response: httpx.Response) -> str: + try: + data = response.json() + error = data.get("error") + if isinstance(error, dict) and error.get("message"): + return str(error["message"]) + if isinstance(error, str): + return error + except Exception: + pass + return response.text or f"HTTP {response.status_code}" + + +def _project_tool_calls(raw_tool_calls: list[dict] | None) -> list[ToolCall]: + tool_calls = [] + for tc in raw_tool_calls or []: + function = tc.get("function") or {} + tool_calls.append( + ToolCall( + id=tc.get("id") or f"chatcmpl-tool-{random_uuid()}", + type="function", + function=FunctionCall(name=function.get("name", ""), arguments=function.get("arguments", "")), + ) + ) + return tool_calls + + +def _project_usage(raw_usage: dict | None) -> UsageInfo: + usage = raw_usage or {} + return UsageInfo( + prompt_tokens=usage.get("prompt_tokens", 0) or 0, + completion_tokens=usage.get("completion_tokens", 0) or 0, + total_tokens=usage.get("total_tokens", 0) or 0, + ) + + +def _project_chat_response(data: dict, *, model_name: str) -> ChatCompletionResponse: + choices = [] + for choice in data.get("choices", []): + message = choice.get("message") or {} + choices.append( + ChatCompletionResponseChoice( + index=choice.get("index", 0), + message=ChatMessage( + role=message.get("role", "assistant"), + content=message.get("content"), + reasoning=message.get("reasoning_content"), + tool_calls=_project_tool_calls(message.get("tool_calls")), + ), + finish_reason=choice.get("finish_reason") or "stop", + ) + ) + return ChatCompletionResponse( + model=model_name, + choices=choices, + usage=_project_usage(data.get("usage")), + ) + + +def _project_stream_chunk(data: dict, *, model_name: str) -> ChatCompletionStreamResponse: + choices = [] + for choice in data.get("choices", []): + delta = choice.get("delta") or {} + tool_calls = [] + for i, tc in enumerate(delta.get("tool_calls") or []): + function = tc.get("function") or {} + tool_calls.append( + DeltaToolCall( + index=tc.get("index", i), + id=tc.get("id"), + type="function" if tc.get("type", "function") == "function" else None, + function=DeltaFunctionCall(name=function.get("name"), arguments=function.get("arguments")), + ) + ) + choices.append( + ChatCompletionResponseStreamChoice( + index=choice.get("index", 0), + delta=DeltaMessage( + role=delta.get("role"), + content=delta.get("content"), + reasoning=delta.get("reasoning_content"), + tool_calls=tool_calls, + ), + finish_reason=choice.get("finish_reason"), + ) + ) + return ChatCompletionStreamResponse( + model=model_name, + choices=choices, + usage=_project_usage(data["usage"]) if data.get("usage") else None, + ) + + +def _encode_chunk(chunk: ChatCompletionStreamResponse) -> str: + return f"data: {json.dumps(chunk.model_dump(mode='json'))}\n\n" + + +def _encode_error(detail: str) -> str: + return f"data: {json.dumps({'error': {'message': detail, 'type': 'api_error'}})}\n\n" diff --git a/modelship/infer/model_deployment.py b/modelship/infer/model_deployment.py index 9af3b71..fa3382e 100644 --- a/modelship/infer/model_deployment.py +++ b/modelship/infer/model_deployment.py @@ -168,6 +168,10 @@ async def __init__(self, config: ModelshipModelConfig): from modelship.infer.llama_cpp.llama_cpp_infer import LlamaCppInfer self.infer = LlamaCppInfer(config) + elif config.loader == ModelLoader.llama_server: + from modelship.infer.llama_server.llama_server_infer import LlamaServerInfer + + self.infer = LlamaServerInfer(config) elif config.loader == ModelLoader.stable_diffusion_cpp: from modelship.infer.stable_diffusion_cpp.stable_diffusion_cpp_infer import StableDiffusionCppInfer diff --git a/modelship/preflight/base.py b/modelship/preflight/base.py index 7c7148b..a3aa89b 100644 --- a/modelship/preflight/base.py +++ b/modelship/preflight/base.py @@ -387,6 +387,13 @@ def _ensure_registered() -> None: register(ModelLoader.llama_cpp, LlamaCppPreflight()) except Exception: logger.debug("preflight: LlamaCppPreflight registration skipped", exc_info=True) + if ModelLoader.llama_server not in _REGISTRY: + try: + from modelship.preflight.llama_cpp import LlamaServerPreflight + + register(ModelLoader.llama_server, LlamaServerPreflight()) + except Exception: + logger.debug("preflight: LlamaServerPreflight registration skipped", exc_info=True) if ModelLoader.stable_diffusion_cpp not in _REGISTRY: try: from modelship.preflight.stable_diffusion_cpp import StableDiffusionCppPreflight diff --git a/modelship/preflight/llama_cpp.py b/modelship/preflight/llama_cpp.py index aff886f..744f0ee 100644 --- a/modelship/preflight/llama_cpp.py +++ b/modelship/preflight/llama_cpp.py @@ -130,6 +130,45 @@ def recommend(self, config: ModelshipModelConfig, hw: HardwareProfile) -> dict[s return {"n_ctx": suggested} +class LlamaServerPreflight: + """Reuses `LlamaCppPreflight`'s GGUF/RAM-budget math for the `llama_server` + loader. That math sizes a single context to the RAM budget; llama-server + instead splits its total context (`-c`) across `parallel` slots, so the + per-slot `n_ctx` LlamaServerConfig expects is the total budget divided by + the slot count (the loader's launch command re-multiplies by `parallel` + to reconstruct the RAM-safe total).""" + + def recommend(self, config: ModelshipModelConfig, hw: HardwareProfile) -> dict[str, Any]: + rec = LlamaCppPreflight().recommend(config, hw) + if "n_ctx" not in rec: + return rec + + server_config = config.llama_server_config + parallel = server_config.parallel if server_config else 1 + if parallel <= 1: + return rec + + per_slot = (rec["n_ctx"] // parallel // _NCTX_ALIGNMENT) * _NCTX_ALIGNMENT + if per_slot < _MIN_NCTX: + logger.warning( + "preflight '%s': RAM budget yields n_ctx=%d across %d parallel slots (< %d per slot); " + "skipping recommendation", + config.name, + per_slot, + parallel, + _MIN_NCTX, + ) + return {} + logger.info( + "preflight llama_server '%s': dividing total n_ctx budget %d across parallel=%d -> n_ctx=%d", + config.name, + rec["n_ctx"], + parallel, + per_slot, + ) + return {"n_ctx": per_slot} + + class _GGUFMeta: __slots__ = ("block_count", "context_length", "head_count_kv", "head_dim") diff --git a/pyproject.toml b/pyproject.toml index 54bfcd8..15601af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,6 +165,7 @@ known-first-party = ["modelship"] markers = [ "integration: tests that require a running Ray cluster and external models", "llama_cpp: integration tests that exercise the llama_cpp loader", + "llama_server: integration tests that exercise the llama_server loader", "vllm: integration tests that exercise the vllm loader", "diffusers: integration tests that exercise the diffusers loader", "stable_diffusion_cpp: integration tests that exercise the stable_diffusion_cpp loader", diff --git a/tests/test_config.py b/tests/test_config.py index 4583d5b..45a9d9c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,6 +6,7 @@ from modelship.infer.infer_config import ( AutoscalingConfig, LlamaCppConfig, + LlamaServerConfig, ModelLoader, ModelshipConfig, ModelshipModelConfig, @@ -170,6 +171,66 @@ def test_llama_cpp_model_config(self): assert config.model == "meta-llama/Llama-3-8B-Instruct-GGUF:*Q4_K_M.gguf" +class TestLlamaServerConfig: + def test_defaults(self): + config = LlamaServerConfig() + assert config.n_ctx == 2048 + assert config.n_batch == 512 + assert config.n_gpu_layers == -1 + assert config.parallel == 1 + assert config.chat_template is None + assert config.extra_args == [] + + def test_custom_values(self): + config = LlamaServerConfig( + n_ctx=4096, + n_batch=1024, + n_gpu_layers=33, + parallel=4, + chat_template="chatml", + extra_args=["--flash-attn"], + ) + assert config.n_ctx == 4096 + assert config.n_batch == 1024 + assert config.n_gpu_layers == 33 + assert config.parallel == 4 + assert config.chat_template == "chatml" + assert config.extra_args == ["--flash-attn"] + + def _num_gpus_model(self, num_gpus: float) -> ModelshipModelConfig: + return ModelshipModelConfig( + name="test-model", + model="repo/Qwen-GGUF:*Q4_K_M.gguf", + usecase=ModelUsecase.generate, + loader=ModelLoader.llama_server, + num_gpus=num_gpus, + ) + + def test_num_gpus_integer_allowed(self): + config = self._num_gpus_model(1) + assert config.num_gpus == 1 + + def test_num_gpus_zero_allowed(self): + config = self._num_gpus_model(0) + assert config.num_gpus == 0 + + def test_num_gpus_fractional_rejected(self): + with pytest.raises(ValidationError, match="not allowed for the llama_server loader"): + self._num_gpus_model(0.5) + + def test_llama_server_model_config(self): + config = ModelshipModelConfig( + name="llama-3", + model="meta-llama/Llama-3-8B-Instruct-GGUF:*Q4_K_M.gguf", + usecase=ModelUsecase.generate, + loader=ModelLoader.llama_server, + llama_server_config=LlamaServerConfig(parallel=4), + ) + assert config.loader == ModelLoader.llama_server + assert config.llama_server_config is not None + assert config.llama_server_config.parallel == 4 + + class TestModelshipModelConfig: def test_minimal_vllm_model(self): config = ModelshipModelConfig( diff --git a/tests/test_integration.py b/tests/test_integration.py index 9c105e2..2297072 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -131,6 +131,25 @@ "n_ctx": 4096, }, }, + "chat-llama-server": { + "name": "chat-llama-server", + # Same Qwen3-0.6B GGUF as `chat-llama-reasoning`, but through the new + # llama_server loader: a llama-server subprocess doing its own chat + # templating, tool-call, and reasoning parsing (`--jinja + # --reasoning-format auto`) instead of modelship's ChatOutputStreamer. + # `parallel: 4` exercises the loader's headline capability over + # llama_cpp: true multi-slot concurrency instead of one asyncio.Lock + # serializing every request. n_ctx is per-slot (the loader launches + # with `-c n_ctx*parallel`), bumped for reasoning headroom. + "model": "lmstudio-community/Qwen3-0.6B-GGUF:*Q4_K_M.gguf", + "usecase": "generate", + "loader": "llama_server", + "num_cpus": 2, + "llama_server_config": { + "n_ctx": 4096, + "parallel": 4, + }, + }, "chat-transformers": { "name": "chat-transformers", "model": "Qwen/Qwen2.5-0.5B-Instruct", @@ -1273,6 +1292,238 @@ def test_response_format_with_reasoning_deployment_rejected(self): assert "reasoning" in response.text.lower() +@pytest.mark.integration +@pytest.mark.llama_server +class TestChatLlamaServer: + """End-to-end chat, tool calling, reasoning, and concurrency through the + new `llama_server` loader (a `llama-server` subprocess proxied over its + native OpenAI-compatible HTTP API, rather than in-process llama-cpp-python + bindings + modelship's own parsers). + + Same Qwen3-0.6B-GGUF weights as `TestChatLlamaCppReasoning`'s + `chat-llama-reasoning` deployment, so these tests double as a parity check + between the two loaders on identical model output — except reasoning and + tool-call parsing is llama-server's own (`--jinja --reasoning-format + auto`), not modelship's `ChatOutputStreamer`. + """ + + @pytest.fixture(autouse=True, scope="class") + def _deploy(self, model_deployer): + model_deployer.deploy("chat-llama-server") + + def test_chat_completion(self, client): + # This deployment is Qwen3-0.6B (reasoning-capable), unlike + # `chat-llama-mship`'s plain Qwen2.5 — it always emits a `...` + # preamble before content, so the token budget needs headroom for + # reasoning to finish, not just the answer itself. + completion = client.chat.completions.create( + model="chat-llama-server", + messages=[{"role": "user", "content": "What is the capital of France?"}], + max_tokens=256, + ) + content = completion.choices[0].message.content + assert content + assert "Paris" in content + + def test_tool_calling_llama_server_loader(self, client): + """Round-trip a tool call through llama-server's own hermes-style + parser (`--jinja`, auto-detected from the GGUF's chat template).""" + completion = client.chat.completions.create( + model="chat-llama-server", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + tool_choice="auto", + max_tokens=128, + ) + tool_calls = completion.choices[0].message.tool_calls + assert tool_calls, f"expected a tool call, got content={completion.choices[0].message.content!r}" + assert tool_calls[0].function.name == "get_weather" + assert "Paris" in tool_calls[0].function.arguments + assert completion.choices[0].finish_reason == "tool_calls" + + def test_tool_calling_streaming_llama_server_loader(self, client): + """Stream a tool call through llama-server and verify the delta + sequence matches the OpenAI streaming contract, same shape as the + llama_cpp/transformers loader streaming tests.""" + stream = client.chat.completions.create( + model="chat-llama-server", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + tool_choice="auto", + max_tokens=128, + stream=True, + ) + + collected = _collect_streaming_tool_call(stream) + + assert collected["tool_calls"], ( + f"expected at least one streamed tool call; got content={collected['content']!r}" + ) + call_0 = collected["tool_calls"][0] + assert call_0["name"] == "get_weather" + assert collected["args_deltas"] >= 1 + parsed_args = json.loads(call_0["arguments"]) + assert parsed_args.get("city") + assert "Paris" in parsed_args["city"] + assert collected["finish_reason"] == "tool_calls" + + def test_reasoning_completion_llama_server(self): + """Non-streaming: llama-server's own `--reasoning-format auto` routes + the `...` block to `message.reasoning`, with the final + answer in `message.content` and no marker leakage into either.""" + response = httpx.post( + f"{OPENAI_API_BASE}/chat/completions", + json={ + "model": "chat-llama-server", + "messages": [{"role": "user", "content": "Briefly: what is 7 times 8?"}], + "max_tokens": 1024, + }, + timeout=300, + ) + assert response.status_code == 200, response.text + message = response.json()["choices"][0]["message"] + assert message.get("reasoning"), f"expected reasoning content, got {message!r}" + assert "" not in (message.get("content") or "") + assert "" not in (message.get("content") or "") + assert "" not in message["reasoning"] + assert "" not in message["reasoning"] + + def test_reasoning_streaming_llama_server(self): + """Streaming: at least one delta carries `reasoning`; concatenated + reasoning is non-empty; markers never leak into either field.""" + with httpx.stream( + "POST", + f"{OPENAI_API_BASE}/chat/completions", + json={ + "model": "chat-llama-server", + "messages": [{"role": "user", "content": "Briefly: what is 7 times 8?"}], + "max_tokens": 1024, + "stream": True, + }, + timeout=300, + ) as response: + assert response.status_code == 200 + reasoning_parts: list[str] = [] + content_parts: list[str] = [] + reasoning_deltas = 0 + for line in response.iter_lines(): + if not line.startswith("data: "): + continue + payload = line[len("data: ") :] + if payload == "[DONE]": + break + chunk = json.loads(payload) + delta = chunk["choices"][0].get("delta") or {} + if delta.get("reasoning"): + reasoning_parts.append(delta["reasoning"]) + reasoning_deltas += 1 + if delta.get("content"): + content_parts.append(delta["content"]) + + assert reasoning_deltas >= 1, "expected at least one reasoning delta" + assert "".join(reasoning_parts).strip(), "expected non-empty reasoning content" + assert "" not in "".join(reasoning_parts) + assert "" not in "".join(reasoning_parts) + assert "" not in "".join(content_parts) + assert "" not in "".join(content_parts) + + def test_reasoning_with_tools_llama_server(self, client): + """Reasoning + tool calling in one round-trip: llama-server populates + both `message.reasoning` and `message.tool_calls`, with + `finish_reason="tool_calls"`.""" + completion = client.chat.completions.create( + model="chat-llama-server", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + tool_choice="auto", + max_tokens=1024, + ) + message = completion.choices[0].message + # The OpenAI Python SDK exposes unknown fields via `model_extra`. + reasoning = getattr(message, "reasoning", None) or message.model_extra.get("reasoning") + assert reasoning, f"expected reasoning, got message={message!r}" + assert "" not in reasoning + tool_calls = message.tool_calls + assert tool_calls, f"expected a tool call, got content={message.content!r}, reasoning={reasoning!r}" + assert tool_calls[0].function.name == "get_weather" + assert "Paris" in tool_calls[0].function.arguments + assert completion.choices[0].finish_reason == "tool_calls" + + def test_concurrent_requests_are_not_serialized(self, client): + """The loader's headline win over llama_cpp: llama-server's + `--parallel` slots let several requests run concurrently instead of + being serialized behind a single `asyncio.Lock` (as `LlamaCppInfer` + is). Time one request, then several at once, and assert the + concurrent batch finishes well under what full serialization would take. + """ + prompt = { + "model": "chat-llama-server", + "messages": [{"role": "user", "content": "Count from 1 to 50, one number per line."}], + "max_tokens": 200, + } + + start = time.monotonic() + client.chat.completions.create(**prompt) + baseline = time.monotonic() - start + + concurrency = 3 + start = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = [pool.submit(client.chat.completions.create, **prompt) for _ in range(concurrency)] + for future in concurrent.futures.as_completed(futures): + future.result() + concurrent_elapsed = time.monotonic() - start + + # Full serialization (llama_cpp's single-lock behavior) would take + # roughly concurrency * baseline; llama-server's parallel slots should + # keep this well under that. + assert concurrent_elapsed < baseline * (concurrency - 0.5), ( + f"expected concurrent requests to overlap via llama-server's parallel slots " + f"(baseline={baseline:.1f}s, {concurrency} concurrent took {concurrent_elapsed:.1f}s)" + ) + + +@pytest.mark.integration +@pytest.mark.llama_server +class TestResponsesLlamaServer: + """The /v1/responses adapter is loader-agnostic: same smoke test as + `TestResponsesLlamaCpp`, run over the llama_server loader.""" + + @pytest.fixture(autouse=True, scope="class") + def _deploy(self, model_deployer): + model_deployer.deploy("chat-llama-server") + + def test_basic_response_through_llama_server(self, client): + # `chat-llama-server` is Qwen3-0.6B (reasoning-capable) — it always + # emits a `...` preamble first, so the token budget needs + # headroom for reasoning to finish, not just the one-word answer. + resp = client.responses.create( + model="chat-llama-server", + input="Say hello in one word.", + max_output_tokens=256, + ) + assert resp.status in {"completed", "incomplete"} + assert resp.output_text.strip() + + def test_streaming_response_through_llama_server(self, client): + stream = client.responses.create( + model="chat-llama-server", + input="Say hello in one word.", + max_output_tokens=256, + stream=True, + ) + text_deltas: list[str] = [] + completed = None + for event in stream: + if event.type == "response.output_text.delta": + text_deltas.append(event.delta) + elif event.type == "response.completed": + completed = event.response + assert "".join(text_deltas).strip() + assert completed is not None + assert completed.status in {"completed", "incomplete"} + + @pytest.mark.integration class TestChatTransformersLlama3Json: """End-to-end llama3_json tool calling through the transformers loader. diff --git a/tests/test_llama_server_infer.py b/tests/test_llama_server_infer.py new file mode 100644 index 0000000..4d90ada --- /dev/null +++ b/tests/test_llama_server_infer.py @@ -0,0 +1,316 @@ +"""Tests for the llama_server loader: subprocess lifecycle (fake executable, +no real llama-server binary in CI) and HTTP request/response projection +(httpx mocked, no real socket).""" + +from __future__ import annotations + +import stat +import sys +import textwrap +from unittest.mock import patch + +import httpx +import pytest + +from modelship.infer.infer_config import ( + LlamaServerConfig, + ModelLoader, + ModelshipModelConfig, + ModelUsecase, + RawRequestProxy, +) +from modelship.infer.llama_server.llama_server_infer import LlamaServerInfer +from modelship.openai.protocol import ChatCompletionRequest, ErrorResponse + +# --------------------------------------------------------------------------- +# Fake llama-server executables (plain scripts; no real binary needed in CI) +# --------------------------------------------------------------------------- + +_FAKE_HEALTHY_SERVER = textwrap.dedent( + """ + import http.server + import socketserver + import sys + + def _port(): + for i, a in enumerate(sys.argv): + if a == "--port": + return int(sys.argv[i + 1]) + raise SystemExit("no --port passed") + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/health": + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + else: + self.send_response(404) + self.end_headers() + + def log_message(self, fmt, *args): + pass + + socketserver.TCPServer.allow_reuse_address = True + with socketserver.TCPServer(("127.0.0.1", _port()), Handler) as httpd: + httpd.serve_forever() + """ +) + +_FAKE_CRASHING_SERVER = "import sys\nsys.exit(1)\n" + + +def _write_fake_executable(tmp_path, source: str, name: str = "fake-llama-server") -> str: + script = tmp_path / name + script.write_text(f"#!{sys.executable}\n{source}") + script.chmod(script.stat().st_mode | stat.S_IEXEC) + return str(script) + + +def _make_config(**llama_server_kwargs) -> ModelshipModelConfig: + cfg = ModelshipModelConfig( + name="test-model", + model="org/test-model", + usecase=ModelUsecase.generate, + loader=ModelLoader.llama_server, + llama_server_config=LlamaServerConfig(**llama_server_kwargs), + ) + cfg._resolved_path = "/fake/model.gguf" + return cfg + + +class TestSubprocessLifecycle: + @pytest.mark.asyncio + async def test_start_waits_for_health_then_shutdown_terminates(self, tmp_path, monkeypatch): + binary = _write_fake_executable(tmp_path, _FAKE_HEALTHY_SERVER) + monkeypatch.setenv("MSHIP_LLAMA_SERVER_BIN", binary) + + infer = LlamaServerInfer(_make_config()) + await infer.start() + try: + assert infer._proc is not None + assert infer._proc.poll() is None + assert infer.max_context_length == infer.config.n_ctx + assert infer._client is not None + finally: + infer.shutdown() + + assert infer._proc is None + + @pytest.mark.asyncio + async def test_missing_binary_raises(self, monkeypatch): + monkeypatch.delenv("MSHIP_LLAMA_SERVER_BIN", raising=False) + infer = LlamaServerInfer(_make_config()) + with pytest.raises(ValueError, match="MSHIP_LLAMA_SERVER_BIN"): + await infer.start() + + @pytest.mark.asyncio + async def test_missing_resolved_path_raises(self, tmp_path, monkeypatch): + binary = _write_fake_executable(tmp_path, _FAKE_HEALTHY_SERVER) + monkeypatch.setenv("MSHIP_LLAMA_SERVER_BIN", binary) + config = _make_config() + config._resolved_path = None + infer = LlamaServerInfer(config) + with pytest.raises(ValueError, match="resolved model path"): + await infer.start() + + @pytest.mark.asyncio + async def test_immediate_crash_retries_then_raises(self, tmp_path, monkeypatch): + binary = _write_fake_executable(tmp_path, _FAKE_CRASHING_SERVER) + monkeypatch.setenv("MSHIP_LLAMA_SERVER_BIN", binary) + infer = LlamaServerInfer(_make_config()) + with ( + patch("modelship.infer.llama_server.llama_server_infer._LAUNCH_RETRY_LIMIT", 2), + pytest.raises(RuntimeError, match="failed to start after"), + ): + await infer.start() + + +# --------------------------------------------------------------------------- +# Request/response projection over a mocked httpx transport +# --------------------------------------------------------------------------- + + +def _infer_with_client(handler) -> LlamaServerInfer: + infer = LlamaServerInfer(_make_config()) + infer._client = httpx.AsyncClient(transport=httpx.MockTransport(handler), base_url="http://test") + return infer + + +def _request(**kwargs) -> ChatCompletionRequest: + return ChatCompletionRequest(model="test-model", messages=[{"role": "user", "content": "hi"}], **kwargs) + + +class TestNonStreamingProjection: + @pytest.mark.asyncio + async def test_strips_extension_fields_and_maps_reasoning_and_tools(self): + raw = { + "id": "chatcmpl-xyz", + "object": "chat.completion", + "created": 123, + "model": "some-gguf", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "reasoning_content": "thinking...", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14}, + "timings": {"predicted_ms": 42.0}, # llama.cpp extension — must not leak + } + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/chat/completions" + return httpx.Response(200, json=raw) + + infer = _infer_with_client(handler) + result = await infer.create_chat_completion(_request(), RawRequestProxy(None, {})) + + assert not isinstance(result, ErrorResponse) + assert "timings" not in result.model_dump() + choice = result.choices[0] + assert choice.message.reasoning == "thinking..." + assert choice.message.tool_calls[0].function.name == "get_weather" + assert choice.finish_reason == "tool_calls" + assert result.usage.prompt_tokens == 10 + assert result.usage.total_tokens == 14 + + @pytest.mark.asyncio + async def test_forwards_normalized_messages_and_model_name(self): + captured = {} + + def handler(request: httpx.Request) -> httpx.Response: + import json + + captured["payload"] = json.loads(request.content) + return httpx.Response( + 200, + json={"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}}], "usage": {}}, + ) + + infer = _infer_with_client(handler) + await infer.create_chat_completion( + _request(tools=[{"type": "function", "function": {"name": "f"}}]), RawRequestProxy(None, {}) + ) + + assert captured["payload"]["model"] == "test-model" + assert captured["payload"]["stream"] is False + assert captured["payload"]["tools"][0]["function"]["name"] == "f" + + @pytest.mark.asyncio + async def test_drops_logprobs_fields_not_yet_supported(self): + # Regression test: ChatCompletionRequest defaults top_logprobs to 0 + # (not None), so a naive exclude_none dump still forwards it — + # llama-server rejects any request carrying top_logprobs unless + # logprobs=true, even when it's the falsy default of 0. + captured = {} + + def handler(request: httpx.Request) -> httpx.Response: + import json + + captured["payload"] = json.loads(request.content) + return httpx.Response( + 200, + json={"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}}], "usage": {}}, + ) + + infer = _infer_with_client(handler) + await infer.create_chat_completion(_request(), RawRequestProxy(None, {})) + + assert "logprobs" not in captured["payload"] + assert "top_logprobs" not in captured["payload"] + + @pytest.mark.asyncio + async def test_rejects_unsupported_image_content(self): + infer = _infer_with_client(lambda request: httpx.Response(500)) + request = _request() + request.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "x"}}]}] + result = await infer.create_chat_completion(request, RawRequestProxy(None, {})) + assert isinstance(result, ErrorResponse) + + @pytest.mark.asyncio + async def test_http_error_becomes_error_response(self): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(400, json={"error": {"message": "bad request"}}) + + infer = _infer_with_client(handler) + result = await infer.create_chat_completion(_request(), RawRequestProxy(None, {})) + assert isinstance(result, ErrorResponse) + assert "bad request" in result.error.message + + @pytest.mark.asyncio + async def test_no_client_falls_back_to_not_supported(self): + infer = _infer_with_client(lambda request: httpx.Response(200)) + infer._client = None + result = await infer.create_chat_completion(_request(), RawRequestProxy(None, {})) + assert isinstance(result, ErrorResponse) + + +class TestStreamingProjection: + @pytest.mark.asyncio + async def test_streams_deltas_tool_calls_and_final_usage(self): + sse_body = ( + 'data: {"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n' + 'data: {"choices":[{"index":0,"delta":{"content":"Hel"},"finish_reason":null}]}\n\n' + 'data: {"choices":[{"index":0,"delta":{"content":"lo"},"finish_reason":null}]}\n\n' + 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function",' + '"function":{"name":"get_weather","arguments":"{}"}}]},"finish_reason":null}]}\n\n' + 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],' + '"usage":{"prompt_tokens":5,"completion_tokens":2,"total_tokens":7}}\n\n' + "data: [DONE]\n\n" + ) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=sse_body.encode(), headers={"content-type": "text/event-stream"}) + + infer = _infer_with_client(handler) + result = await infer.create_chat_completion(_request(stream=True), RawRequestProxy(None, {})) + + chunks = [chunk async for chunk in result] + assert chunks[-1] == "data: [DONE]\n\n" + + import json + + content = "" + tool_call_seen = False + usage_seen = None + for raw in chunks[:-1]: + payload = json.loads(raw[len("data: ") :]) + for choice in payload["choices"]: + delta = choice["delta"] + if delta.get("content"): + content += delta["content"] + if delta.get("tool_calls"): + tool_call_seen = True + assert delta["tool_calls"][0]["function"]["name"] == "get_weather" + if payload.get("usage"): + usage_seen = payload["usage"] + + assert content == "Hello" + assert tool_call_seen + assert usage_seen is not None + assert usage_seen["prompt_tokens"] == 5 + assert usage_seen["completion_tokens"] == 2 + assert usage_seen["total_tokens"] == 7 + + @pytest.mark.asyncio + async def test_stream_http_error_yields_error_chunk(self): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(400, json={"error": {"message": "boom"}}) + + infer = _infer_with_client(handler) + result = await infer.create_chat_completion(_request(stream=True), RawRequestProxy(None, {})) + chunks = [chunk async for chunk in result] + assert any("boom" in c for c in chunks) + # A well-behaved SSE stream always terminates with [DONE], even on an + # in-band error — clients that wait for it rather than connection + # close would otherwise hang. + assert chunks[-1] == "data: [DONE]\n\n" diff --git a/tests/test_preflight_llama_server.py b/tests/test_preflight_llama_server.py new file mode 100644 index 0000000..3acdc90 --- /dev/null +++ b/tests/test_preflight_llama_server.py @@ -0,0 +1,110 @@ +"""Tests for the LlamaServerPreflight estimator (wraps LlamaCppPreflight).""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +from modelship.infer.infer_config import ( + LlamaServerConfig, + ModelLoader, + ModelshipModelConfig, + ModelUsecase, +) +from modelship.preflight import HardwareProfile +from modelship.preflight.llama_cpp import LlamaServerPreflight, _GGUFMeta + +_LLAMA_META = _GGUFMeta(block_count=32, head_count_kv=8, head_dim=128, context_length=131072) + + +def _make_config( + *, + resolved_path: str | None = None, + llama_server_kwargs: dict | None = None, + num_gpus: float = 0, +) -> ModelshipModelConfig: + cfg = ModelshipModelConfig( + name="test-model", + model="org/test-model", + usecase=ModelUsecase.generate, + loader=ModelLoader.llama_server, + llama_server_config=LlamaServerConfig(**(llama_server_kwargs or {})), + num_gpus=num_gpus, + ) + cfg._resolved_path = resolved_path + return cfg + + +def _write_dummy_gguf(tmp_path: Path) -> Path: + path = tmp_path / "model.gguf" + path.write_bytes(b"\0" * 1024) + return path + + +class TestLlamaServerPreflight: + def test_single_slot_matches_llama_cpp_math(self, tmp_path): + # parallel=1 (default): no division, identical to LlamaCppPreflight. + cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) + hw = HardwareProfile(ram_bytes=4 * 1024**3) + + with ( + patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=_LLAMA_META), + patch("modelship.preflight.llama_cpp._weight_bytes", return_value=int(1.75 * 1024**3)), + ): + rec = LlamaServerPreflight().recommend(cfg, hw) + + assert "n_ctx" in rec + assert rec["n_ctx"] % 256 == 0 + + def test_parallel_divides_total_budget(self, tmp_path): + # Same hardware/model, parallel=4 must yield a per-slot n_ctx roughly + # 1/4 of the single-slot recommendation (the launch command + # re-multiplies by parallel to reconstruct the RAM-safe total). + cfg_single = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path)), llama_server_kwargs={"parallel": 1}) + cfg_parallel = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path)), llama_server_kwargs={"parallel": 4}) + hw = HardwareProfile(ram_bytes=4 * 1024**3) + + with ( + patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=_LLAMA_META), + patch("modelship.preflight.llama_cpp._weight_bytes", return_value=int(1.75 * 1024**3)), + ): + rec_single = LlamaServerPreflight().recommend(cfg_single, hw) + rec_parallel = LlamaServerPreflight().recommend(cfg_parallel, hw) + + assert rec_parallel["n_ctx"] * 4 <= rec_single["n_ctx"] + 256 * 4 + assert rec_parallel["n_ctx"] < rec_single["n_ctx"] + + def test_parallel_too_high_for_budget_returns_empty(self, tmp_path): + # A tiny budget divided across many slots drops below the minimum + # usable n_ctx; the estimator should decline rather than recommend + # something unusably small. + cfg = _make_config( + resolved_path=str(_write_dummy_gguf(tmp_path)), llama_server_kwargs={"parallel": 64}, num_gpus=0 + ) + hw = HardwareProfile(ram_bytes=2 * 1024**3) + + with ( + patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=_LLAMA_META), + patch("modelship.preflight.llama_cpp._weight_bytes", return_value=int(1.9 * 1024**3)), + ): + rec = LlamaServerPreflight().recommend(cfg, hw) + + assert rec == {} + + def test_gpu_offload_skips_ram_sizing(self, tmp_path): + cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path)), num_gpus=1) + hw = HardwareProfile(ram_bytes=64 * 1024**3) + assert LlamaServerPreflight().recommend(cfg, hw) == {} + + def test_run_preflight_dispatches_to_llama_server(self, tmp_path): + from modelship.preflight import run_preflight + + cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) + hw = HardwareProfile(ram_bytes=4 * 1024**3) + + with ( + patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=_LLAMA_META), + patch("modelship.preflight.llama_cpp._weight_bytes", return_value=int(1.75 * 1024**3)), + ): + rec = run_preflight(cfg, hw) + assert "n_ctx" in rec From eb8a2a901d78f89535822b14e921421595fc4438 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:17:50 +0000 Subject: [PATCH 02/35] fix: harden llama_server loader streaming and subprocess log draining - Catch httpx.TransportError mid-stream (dropped connection, crashed subprocess) and emit a clean SSE error chunk + [DONE] instead of letting it propagate as an unhandled 500. - Use asyncio.get_running_loop() instead of the deprecated get_event_loop() inside the async _launch(). - Decode subprocess output with errors="replace" so a non-UTF-8 byte in the child's stdout/stderr can't kill the log-drain thread and block the pipe. --- modelship/infer/llama_server/llama_server_infer.py | 7 ++++++- tests/test_llama_server_infer.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 252d29f..50b39fc 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -153,7 +153,7 @@ async def start(self) -> None: self._set_max_context_length(self.config.n_ctx) async def _launch(self, binary: str, model_path: str) -> None: - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() port = await loop.run_in_executor(None, _free_port) self._port = port @@ -190,6 +190,7 @@ async def _launch(self, binary: str, model_path: str) -> None: stderr=subprocess.PIPE, text=True, bufsize=1, + errors="replace", ) self._recent_log_lines.clear() assert self._proc.stdout is not None @@ -337,6 +338,10 @@ async def _stream_chat_completion(self, payload: dict[str, Any], request_id: str buffered.append(choice.delta.content) yield _encode_chunk(chunk) yield "data: [DONE]\n\n" + except httpx.TransportError as e: + logger.warning("chat request %s failed mid-stream: %s", request_id, e) + yield _encode_error(f"llama-server request failed: {e}") + yield "data: [DONE]\n\n" finally: logger.log(TRACE, "chat response %s (stream): %r", request_id, "".join(buffered)) diff --git a/tests/test_llama_server_infer.py b/tests/test_llama_server_infer.py index 4d90ada..bc6882f 100644 --- a/tests/test_llama_server_infer.py +++ b/tests/test_llama_server_infer.py @@ -314,3 +314,16 @@ def handler(request: httpx.Request) -> httpx.Response: # in-band error — clients that wait for it rather than connection # close would otherwise hang. assert chunks[-1] == "data: [DONE]\n\n" + + @pytest.mark.asyncio + async def test_stream_transport_error_yields_error_chunk(self): + # Simulates a dropped connection / crashed subprocess mid-stream: httpx + # raises TransportError rather than handing back a Response at all. + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ReadError("connection reset") + + infer = _infer_with_client(handler) + result = await infer.create_chat_completion(_request(stream=True), RawRequestProxy(None, {})) + chunks = [chunk async for chunk in result] + assert any("connection reset" in c for c in chunks) + assert chunks[-1] == "data: [DONE]\n\n" From fa62d570e6fd02f72008cd31773759750681280b Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:22:46 +0000 Subject: [PATCH 03/35] fix: reject non-positive parallel and close httpx client on shutdown - LlamaServerConfig.parallel now requires >= 1; 0 or negative would divide/multiply into a broken -c launch arg or a zero-slot server. - shutdown() previously dropped the httpx.AsyncClient reference without closing it, leaking the connection pool's sockets. Schedule aclose() as a background task when a loop is running, keeping a strong reference so it isn't GC'd before it completes. --- modelship/infer/infer_config.py | 2 +- .../infer/llama_server/llama_server_infer.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/modelship/infer/infer_config.py b/modelship/infer/infer_config.py index a02d4cf..d909d13 100644 --- a/modelship/infer/infer_config.py +++ b/modelship/infer/infer_config.py @@ -174,7 +174,7 @@ class LlamaServerConfig(BaseModel): n_gpu_layers: int = -1 # Concurrent request slots. llama-server splits its total context (`-c`) # across slots, so the process is launched with `n_ctx * parallel`. - parallel: int = 1 + parallel: int = Field(default=1, ge=1) # Built-in template name (e.g. "chatml") or a path to a Jinja file; # None lets llama-server use the GGUF's embedded chat template. chat_template: str | None = None diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 50b39fc..c425f84 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -61,6 +61,12 @@ class _EarlyCrashError(RuntimeError): treated as a transient bind race and retried with a fresh port.""" +# asyncio only holds a weak reference to a task's coroutine, so a fire-and-forget +# task with no other referent can be GC'd before it runs. Keep a strong reference +# here for the lifetime of each client-close task, started from the (sync) shutdown(). +_pending_client_closes: set[asyncio.Task] = set() + + class LlamaServerInfer(BaseInfer): """Drives a `llama-server` subprocess over its native OpenAI-compatible HTTP API. Unlike `llama_cpp` (in-process `Llama` bindings + modelship's @@ -102,7 +108,16 @@ def shutdown(self) -> None: for task in self._log_tasks: task.cancel() self._log_tasks = [] - self._client = None + if self._client is not None: + client, self._client = self._client, None + try: + loop = asyncio.get_running_loop() + except RuntimeError: + pass # no running loop (e.g. interpreter teardown) — let GC reclaim the socket + else: + task = loop.create_task(client.aclose()) + _pending_client_closes.add(task) + task.add_done_callback(_pending_client_closes.discard) def __del__(self): self.shutdown() From ed2ce533162a50c61488f1f7fa04c15c98c0400f Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:52:02 +0000 Subject: [PATCH 04/35] feat: ship llama-server in Docker images and wire GPU offload (Stage B2) Docker images now bundle a digest-pinned llama.cpp build at /opt/llama.cpp (upstream's official server images; CUDA 13 build for the gpu variant, which falls back to its dlopen'd CPU backends when no GPU is visible) with MSHIP_LLAMA_SERVER_BIN preset to a wrapper that scopes LD_LIBRARY_PATH to the subprocess. The loader passes -ngl n_gpu_layers when the deployment reserves GPUs and forces -ngl 0 otherwise, since Ray only sets CUDA_VISIBLE_DEVICES for actors that reserve GPUs. --- Dockerfile | 51 ++++++++++++++++++- docs/development.md | 18 +++++++ modelship/infer/infer_config.py | 2 + .../infer/llama_server/llama_server_infer.py | 6 +++ tests/test_integration.py | 4 +- tests/test_llama_server_infer.py | 26 +++++++++- 6 files changed, 103 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index fb2d712..8d92466 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,48 @@ ARG MSHIP_VARIANT=gpu ARG UID=1000 ARG GID=1000 +# llama.cpp's official server images, pinned by manifest digest (the digest IS +# the sha256 of the content). Upstream publishes no Linux CUDA binary in its +# GitHub releases, so the GPU variant sources their CUDA 13 Docker build +# instead. Bump the tag and digest together. +ARG LLAMA_CPP_IMAGE_GPU=ghcr.io/ggml-org/llama.cpp:server-cuda13-b9859@sha256:e8e003c66cb77615dfef2f6ae1b7f5ad0de7bd048e19c40357220cb4141d1cdc +ARG LLAMA_CPP_IMAGE_CPU=ghcr.io/ggml-org/llama.cpp:server-b9859@sha256:f415de2e2c3e61b3dfab40d7fd26136c13d342c1ae4b3ffa8657fcc6a2f43d60 + +# ============================================================================= +# llama-server — assembles /opt/llama.cpp for the llama_server loader. +# +# /app in the upstream images holds the llama-server binary plus the .so +# backends it dlopen()s (GGML_BACKEND_DL). The CUDA backend is skipped +# gracefully when no GPU/driver is present, so the GPU build also runs +# CPU-only. libggml-cuda.so dynamically links libcudart/libcublas(Lt) — copied +# in from the same image so they can't skew against the venv's torch-bundled +# CUDA libs — and the driver's libcuda.so.1, which the NVIDIA Container +# Toolkit provides at run time. +# ============================================================================= +FROM ${LLAMA_CPP_IMAGE_GPU} AS llama-server-gpu + +RUN set -e && \ + mkdir -p /opt/llama.cpp && \ + cp -a /app/. /opt/llama.cpp/ && \ + ldconfig && \ + for lib in libcudart.so.13 libcublas.so.13 libcublasLt.so.13; do \ + cp -L "$(ldconfig -p | awk -v lib="$lib" '$1 == lib { print $NF; exit }')" /opt/llama.cpp/; \ + done + +FROM ${LLAMA_CPP_IMAGE_CPU} AS llama-server-cpu + +RUN mkdir -p /opt/llama.cpp && cp -a /app/. /opt/llama.cpp/ + +# The raw binary can't run standalone: it resolves its sibling .so files via +# the loader path. The wrapper scopes LD_LIBRARY_PATH to the llama-server +# process only — globally, /opt/llama.cpp would shadow llama-cpp-python's own +# bundled libllama/libggml. +FROM llama-server-${MSHIP_VARIANT} AS llama-server + +RUN printf '#!/bin/sh\nexport LD_LIBRARY_PATH="/opt/llama.cpp${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"\nexec /opt/llama.cpp/llama-server "$@"\n' \ + > /opt/llama.cpp/llama-server.sh && \ + chmod +x /opt/llama.cpp/llama-server.sh + # ============================================================================= # base — minimal runtime OS + uv + non-root user + env vars. # @@ -36,7 +78,8 @@ RUN apt-get update -y && \ gcc \ gnupg \ gosu \ - libc6-dev && \ + libc6-dev \ + libgomp1 && \ rm -rf /var/lib/apt/lists/* # Register the NVIDIA CUDA apt repo and install cuda-cudart (GPU variant only). @@ -82,6 +125,12 @@ ENV UV_PYTHON_INSTALL_DIR=/usr/local/uv/python ENV PATH="$UV_PROJECT_ENVIRONMENT/bin:$PATH" ENV MSHIP_PLUGIN_WHEEL_DIR=/opt/modelship/plugin-wheels +# Pinned llama.cpp build for the llama_server loader (see the llama-server +# stages above). llama-server additionally needs libgomp1 (ggml CPU backends) +# and libssl3 (already pulled in via curl). +COPY --from=llama-server /opt/llama.cpp /opt/llama.cpp +ENV MSHIP_LLAMA_SERVER_BIN=/opt/llama.cpp/llama-server.sh + # onnxruntime-gpu (pulled in by the kokoroonnx plugin) dlopen()s # libonnxruntime_providers_cuda.so which has plain DT_NEEDED entries for # libcublasLt.so.13 / libcudnn.so.9 / etc. Torch cu130 bundles these under diff --git a/docs/development.md b/docs/development.md index 7623b17..ac74dd2 100644 --- a/docs/development.md +++ b/docs/development.md @@ -54,6 +54,7 @@ The following environment variables are set in the dev image with sensible defau | `MSHIP_USE_EXISTING_RAY_CLUSTER` | `false` | Set to `true` to connect to a Ray cluster you manage (must run on a cluster node) instead of starting one; implies deploy-and-exit | | `MSHIP_GATEWAY_REPLICAS` | `1` | Number of API gateway replicas. Raise for routing/ingress HA and to spread request-proxying load under high concurrency; replicas keep routing tables in sync via the deploy coordinator's watch loop. | | `MSHIP_GATEWAY_MAX_ONGOING` | `1024` | Per-replica Ray Serve concurrency cap for the gateway. The gateway holds a slot for the whole lifetime of each streamed response, so a low cap throttles before the engine does. | +| `MSHIP_LLAMA_SERVER_BIN` | `/opt/llama.cpp/llama-server.sh` | `llama-server` executable used by the `llama_server` loader. The image ships a pinned llama.cpp build; see [llama-server binary](#llama-server-binary-llama_server-loader) for running outside the image. | ### Installing plugin dependencies for IntelliSense @@ -107,6 +108,23 @@ The dev image drops into a shell. Start the server — it starts its own Ray hea uv run mship_deploy.py ``` +## llama-server binary (`llama_server` loader) + +The `llama_server` loader launches a `llama-server` subprocess and finds it via `MSHIP_LLAMA_SERVER_BIN`. Inside the Docker images (dev and prod, both variants) this is preconfigured: a pinned llama.cpp build lives at `/opt/llama.cpp`, and the env var points at its wrapper script. The GPU variant ships upstream's CUDA build, which falls back to its bundled CPU backends when no GPU is visible. + +To run outside the image (e.g. directly on a Linux or macOS host), download a build from [llama.cpp releases](https://github.com/ggml-org/llama.cpp/releases) and extract it. The raw `llama-server` binary does **not** run standalone — it dynamically links the `libggml*`/`libllama*` libraries that ship as sibling files in the same archive, so point `MSHIP_LLAMA_SERVER_BIN` at a small wrapper that puts the extracted directory on the loader path: + +```bash +#!/bin/sh +# llama-server.sh — use DYLD_LIBRARY_PATH instead on macOS +export LD_LIBRARY_PATH="/path/to/extracted${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +exec /path/to/extracted/llama-server "$@" +``` + +```bash +export MSHIP_LLAMA_SERVER_BIN=/path/to/llama-server.sh +``` + ## Production Builds To build the production images locally: diff --git a/modelship/infer/infer_config.py b/modelship/infer/infer_config.py index d909d13..93188d7 100644 --- a/modelship/infer/infer_config.py +++ b/modelship/infer/infer_config.py @@ -171,6 +171,8 @@ class LlamaServerConfig(BaseModel): n_ctx: int = 2048 n_batch: int = 512 + # Layers to offload when the deployment reserves GPUs (num_gpus > 0): + # -1 auto-fits the offload to free VRAM, <= -2 offloads all layers. n_gpu_layers: int = -1 # Concurrent request slots. llama-server splits its total context (`-c`) # across slots, so the process is launched with `n_ctx * parallel`. diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index c425f84..43d0045 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -193,6 +193,12 @@ async def _launch(self, binary: str, model_path: str) -> None: "--api-key", self._api_key, ] + if self.model_config.num_gpus > 0: + args += ["-ngl", str(self.config.n_gpu_layers)] + else: + # Ray only sets CUDA_VISIBLE_DEVICES for actors that reserve GPUs, so + # a num_gpus=0 deployment may still see every GPU — force no offload. + args += ["-ngl", "0"] if self.config.chat_template: flag = "--chat-template-file" if os.path.isfile(self.config.chat_template) else "--chat-template" args += [flag, self.config.chat_template] diff --git a/tests/test_integration.py b/tests/test_integration.py index 2297072..ce7e40f 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1500,7 +1500,7 @@ def test_basic_response_through_llama_server(self, client): resp = client.responses.create( model="chat-llama-server", input="Say hello in one word.", - max_output_tokens=256, + max_output_tokens=512, ) assert resp.status in {"completed", "incomplete"} assert resp.output_text.strip() @@ -1509,7 +1509,7 @@ def test_streaming_response_through_llama_server(self, client): stream = client.responses.create( model="chat-llama-server", input="Say hello in one word.", - max_output_tokens=256, + max_output_tokens=512, stream=True, ) text_deltas: list[str] = [] diff --git a/tests/test_llama_server_infer.py b/tests/test_llama_server_infer.py index bc6882f..b5f1d7d 100644 --- a/tests/test_llama_server_infer.py +++ b/tests/test_llama_server_infer.py @@ -67,12 +67,13 @@ def _write_fake_executable(tmp_path, source: str, name: str = "fake-llama-server return str(script) -def _make_config(**llama_server_kwargs) -> ModelshipModelConfig: +def _make_config(num_gpus: float = 0, **llama_server_kwargs) -> ModelshipModelConfig: cfg = ModelshipModelConfig( name="test-model", model="org/test-model", usecase=ModelUsecase.generate, loader=ModelLoader.llama_server, + num_gpus=num_gpus, llama_server_config=LlamaServerConfig(**llama_server_kwargs), ) cfg._resolved_path = "/fake/model.gguf" @@ -114,6 +115,29 @@ async def test_missing_resolved_path_raises(self, tmp_path, monkeypatch): with pytest.raises(ValueError, match="resolved model path"): await infer.start() + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("num_gpus", "config_kwargs", "expected_ngl"), + [ + (0, {}, "0"), # no reserved GPU: never offload, even with a GPU build + (0, {"n_gpu_layers": 24}, "0"), + (1, {}, "-1"), # llama-server auto-fit default + (1, {"n_gpu_layers": 24}, "24"), + (2, {"n_gpu_layers": -2}, "-2"), # <= -2 means all layers + ], + ) + async def test_ngl_follows_num_gpus(self, tmp_path, monkeypatch, num_gpus, config_kwargs, expected_ngl): + binary = _write_fake_executable(tmp_path, _FAKE_HEALTHY_SERVER) + monkeypatch.setenv("MSHIP_LLAMA_SERVER_BIN", binary) + + infer = LlamaServerInfer(_make_config(num_gpus=num_gpus, **config_kwargs)) + await infer.start() + try: + args = list(infer._proc.args) + assert args[args.index("-ngl") + 1] == expected_ngl + finally: + infer.shutdown() + @pytest.mark.asyncio async def test_immediate_crash_retries_then_raises(self, tmp_path, monkeypatch): binary = _write_fake_executable(tmp_path, _FAKE_CRASHING_SERVER) From 46b41ac1a9aaa6264b6acb98754b9142c0698b8b Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 09:23:23 +0000 Subject: [PATCH 05/35] fix: optimize llama_server loader concurrency, timeouts, and protocol alignment - Offload subprocess spawning (fork/exec) to run_in_executor to avoid blocking the main asyncio event loop during actor model cold-starts. - Run wait-and-kill process termination logic in a background daemon thread on shutdown to prevent blocking the Ray Serve actor's event loop. - Use dedicated threading.Thread log-draining daemon threads instead of loop.run_in_executor to avoid exhausting the default executor thread pool. - Explicitly close stdout/stderr pipes on shutdown to immediately unblock and terminate background log-draining threads. - Catch httpx.HTTPError instead of TransportError during health check probes and completions to safely handle network timeouts. - Set tool_calls=tool_calls or None in ChatCompletionStreamResponse choices to omit empty tool calls and maximize OpenAI compatibility. - Added comprehensive unit test covering the non-blocking and timeout fallback behavior during shutdown. --- .../infer/llama_server/llama_server_infer.py | 86 +++++++++++++------ modelship/openai/protocol/chat.py | 2 +- tests/test_llama_server_infer.py | 45 ++++++++++ 3 files changed, 105 insertions(+), 28 deletions(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 43d0045..53361fc 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -5,6 +5,7 @@ import secrets import socket import subprocess +import threading import time from collections import deque from collections.abc import AsyncGenerator @@ -94,20 +95,38 @@ def __init__(self, model_config: ModelshipModelConfig): self._api_key: str = secrets.token_hex(32) self._client: httpx.AsyncClient | None = None self._recent_log_lines: deque[str] = deque(maxlen=_RECENT_LOG_LINES) - self._log_tasks: list[asyncio.Future] = [] + self._log_threads: list[threading.Thread] = [] def shutdown(self) -> None: - if self._proc is not None and self._proc.poll() is None: - logger.info("Shutting down llama-server for %s", self.model_config.name) - self._proc.terminate() - try: - self._proc.wait(timeout=10) - except subprocess.TimeoutExpired: - self._proc.kill() + if self._proc is not None: + if self._proc.poll() is None: + logger.info("Shutting down llama-server for %s", self.model_config.name) + self._proc.terminate() + + proc = self._proc + + def _wait_and_kill(): + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + with contextlib.suppress(Exception): + proc.kill() + except Exception: + pass + + thread = threading.Thread(target=_wait_and_kill, daemon=True) + thread.start() + + # Close pipes to immediately unblock and terminate log-draining threads + if self._proc.stdout is not None: + with contextlib.suppress(Exception): + self._proc.stdout.close() + if self._proc.stderr is not None: + with contextlib.suppress(Exception): + self._proc.stderr.close() + self._proc = None - for task in self._log_tasks: - task.cancel() - self._log_tasks = [] + self._log_threads = [] if self._client is not None: client, self._client = self._client, None try: @@ -205,21 +224,34 @@ async def _launch(self, binary: str, model_path: str) -> None: args += list(self.config.extra_args) logger.info("llama-server launch args for '%s': %s", self.model_config.name, _redact(args)) - self._proc = subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - bufsize=1, - errors="replace", + self._proc = await loop.run_in_executor( + None, + lambda: subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + errors="replace", + ), ) self._recent_log_lines.clear() assert self._proc.stdout is not None assert self._proc.stderr is not None - self._log_tasks = [ - loop.run_in_executor(None, self._drain_stream, self._proc.stdout, "stdout"), - loop.run_in_executor(None, self._drain_stream, self._proc.stderr, "stderr"), - ] + + thread_stdout = threading.Thread( + target=self._drain_stream, + args=(self._proc.stdout, "stdout"), + daemon=True, + ) + thread_stderr = threading.Thread( + target=self._drain_stream, + args=(self._proc.stderr, "stderr"), + daemon=True, + ) + thread_stdout.start() + thread_stderr.start() + self._log_threads = [thread_stdout, thread_stderr] def _drain_stream(self, stream: Any, tag: str) -> None: """Consume a pipe to TRACE-level logs so the child never blocks on a @@ -231,7 +263,7 @@ def _drain_stream(self, stream: Any, tag: str) -> None: line = raw_line.rstrip("\n") logger.log(TRACE, "[%s:%s] %s", self.model_config.name, tag, line) self._recent_log_lines.append(line) - except ValueError: + except (ValueError, OSError): pass # stream closed from the main thread while we were reading finally: with contextlib.suppress(Exception): @@ -267,7 +299,7 @@ async def _wait_healthy(self) -> None: if resp.status_code == 200: logger.info("llama-server healthy for '%s' on port %d", self.model_config.name, self._port) return - except httpx.TransportError: + except httpx.HTTPError: pass await asyncio.sleep(_HEALTH_POLL_INTERVAL_S) @@ -319,7 +351,7 @@ async def create_chat_completion( detail = _extract_error_detail(e.response) logger.warning("chat request %s failed: %s", request_id, detail) return create_error_response(detail, status_code=e.response.status_code) - except httpx.TransportError as e: + except httpx.HTTPError as e: logger.warning("chat request %s failed: %s", request_id, e) return create_error_response(f"llama-server request failed: {e}", status_code=502) @@ -359,7 +391,7 @@ async def _stream_chat_completion(self, payload: dict[str, Any], request_id: str buffered.append(choice.delta.content) yield _encode_chunk(chunk) yield "data: [DONE]\n\n" - except httpx.TransportError as e: + except httpx.HTTPError as e: logger.warning("chat request %s failed mid-stream: %s", request_id, e) yield _encode_error(f"llama-server request failed: {e}") yield "data: [DONE]\n\n" @@ -473,7 +505,7 @@ def _project_stream_chunk(data: dict, *, model_name: str) -> ChatCompletionStrea role=delta.get("role"), content=delta.get("content"), reasoning=delta.get("reasoning_content"), - tool_calls=tool_calls, + tool_calls=tool_calls or None, ), finish_reason=choice.get("finish_reason"), ) diff --git a/modelship/openai/protocol/chat.py b/modelship/openai/protocol/chat.py index af01730..29ae3d2 100644 --- a/modelship/openai/protocol/chat.py +++ b/modelship/openai/protocol/chat.py @@ -155,7 +155,7 @@ class DeltaMessage(OpenAIBaseModel): role: str | None = None content: str | None = None reasoning: str | None = None - tool_calls: list[DeltaToolCall] = Field(default_factory=list) + tool_calls: list[DeltaToolCall] | None = None class ChatCompletionResponseStreamChoice(OpenAIBaseModel): diff --git a/tests/test_llama_server_infer.py b/tests/test_llama_server_infer.py index b5f1d7d..e388ae0 100644 --- a/tests/test_llama_server_infer.py +++ b/tests/test_llama_server_infer.py @@ -4,6 +4,7 @@ from __future__ import annotations +import contextlib import stat import sys import textwrap @@ -149,6 +150,50 @@ async def test_immediate_crash_retries_then_raises(self, tmp_path, monkeypatch): ): await infer.start() + @pytest.mark.asyncio + async def test_shutdown_is_non_blocking_and_kills_on_timeout(self, tmp_path, monkeypatch): + import subprocess + import threading + import time + + binary = _write_fake_executable(tmp_path, _FAKE_HEALTHY_SERVER) + monkeypatch.setenv("MSHIP_LLAMA_SERVER_BIN", binary) + + infer = LlamaServerInfer(_make_config()) + await infer.start() + + proc = infer._proc + assert proc is not None + + original_kill = proc.kill + + wait_called = threading.Event() + kill_called = threading.Event() + + def mocked_wait(timeout=None): + wait_called.set() + raise subprocess.TimeoutExpired(cmd=proc.args, timeout=timeout) + + def mocked_kill(): + with contextlib.suppress(Exception): + original_kill() + kill_called.set() + + proc.wait = mocked_wait + proc.kill = mocked_kill + + start_time = time.monotonic() + infer.shutdown() + end_time = time.monotonic() + + # Verify that shutdown returned immediately (did not block for the timeout) + assert end_time - start_time < 0.5 + assert infer._proc is None + + # Wait for the background thread to finish and call wait and kill + assert wait_called.wait(timeout=2.0) + assert kill_called.wait(timeout=2.0) + # --------------------------------------------------------------------------- # Request/response projection over a mocked httpx transport From 5bcf726b217d92f4d8ee9fa76f1f91a2ad82c950 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 09:32:38 +0000 Subject: [PATCH 06/35] fix: assert self._proc is not None to resolve pyright type-checking error --- modelship/infer/llama_server/llama_server_infer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 53361fc..430630e 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -236,6 +236,7 @@ async def _launch(self, binary: str, model_path: str) -> None: ), ) self._recent_log_lines.clear() + assert self._proc is not None assert self._proc.stdout is not None assert self._proc.stderr is not None From dbcbf7d5d0570596a1afba04c21b83c7d7b18b9d Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:03:58 +0000 Subject: [PATCH 07/35] feat: extract 3-field DTO and rewire llama_server non-stream projection (Stage B3) - Define ParsedChatOutput DTO representing the parsed output and build_from_parsed builder in modelship/openai/chat_utils.py. - Rewire _project_chat_response in modelship/infer/llama_server/llama_server_infer.py to extract message content, reasoning, and tool calls into ParsedChatOutput and delegate to build_from_parsed. - Add unit tests verifying multi-choice construction, custom created timestamps, and finish reason fallback derivation under tests/test_chat_utils.py. - Run formatting and static type checking with Ruff and Pyright to ensure zero warnings or errors. --- .../infer/llama_server/llama_server_infer.py | 39 ++++----- modelship/openai/chat_utils.py | 80 +++++++++++++++++++ tests/test_chat_utils.py | 78 +++++++++++++++++- 3 files changed, 178 insertions(+), 19 deletions(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 430630e..6f627f9 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -16,14 +16,17 @@ from modelship.infer.base_infer import BaseInfer from modelship.infer.infer_config import LlamaServerConfig, ModelshipModelConfig, RawRequestProxy from modelship.logging import TRACE, get_logger -from modelship.openai.chat_utils import UnsupportedContentError, normalize_chat_messages +from modelship.openai.chat_utils import ( + ParsedChatOutput, + UnsupportedContentError, + build_from_parsed, + normalize_chat_messages, +) from modelship.openai.protocol import ( ChatCompletionRequest, ChatCompletionResponse, - ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, - ChatMessage, DeltaFunctionCall, DeltaMessage, DeltaToolCall, @@ -358,7 +361,7 @@ async def create_chat_completion( data = resp.json() logger.log(TRACE, "chat response %s: %s", request_id, data) - return _project_chat_response(data, model_name=self.model_config.name) + return _project_chat_response(data, model_name=self.model_config.name, request_id=request_id) async def _stream_chat_completion(self, payload: dict[str, Any], request_id: str) -> AsyncGenerator[str, None]: assert self._client is not None @@ -461,26 +464,26 @@ def _project_usage(raw_usage: dict | None) -> UsageInfo: ) -def _project_chat_response(data: dict, *, model_name: str) -> ChatCompletionResponse: +def _project_chat_response(data: dict, *, model_name: str, request_id: str) -> ChatCompletionResponse: choices = [] + finish_reasons = [] for choice in data.get("choices", []): message = choice.get("message") or {} - choices.append( - ChatCompletionResponseChoice( - index=choice.get("index", 0), - message=ChatMessage( - role=message.get("role", "assistant"), - content=message.get("content"), - reasoning=message.get("reasoning_content"), - tool_calls=_project_tool_calls(message.get("tool_calls")), - ), - finish_reason=choice.get("finish_reason") or "stop", - ) + dto = ParsedChatOutput( + content=message.get("content"), + reasoning=message.get("reasoning_content"), + tool_calls=_project_tool_calls(message.get("tool_calls")), ) - return ChatCompletionResponse( - model=model_name, + choices.append(dto) + finish_reasons.append(choice.get("finish_reason") or "stop") + + return build_from_parsed( + request_id=data.get("id") or request_id, + model_name=model_name, choices=choices, usage=_project_usage(data.get("usage")), + finish_reasons=finish_reasons, + created=data.get("created"), ) diff --git a/modelship/openai/chat_utils.py b/modelship/openai/chat_utils.py index aa292b7..013c93f 100644 --- a/modelship/openai/chat_utils.py +++ b/modelship/openai/chat_utils.py @@ -1,8 +1,17 @@ """Validation and normalization helpers for OpenAI chat-completion messages.""" +import time +from dataclasses import dataclass, field from typing import Any from modelship.logging import get_logger +from modelship.openai.protocol import ( + ChatCompletionResponse, + ChatCompletionResponseChoice, + ChatMessage, + ToolCall, + UsageInfo, +) logger = get_logger("openai.chat_utils") @@ -190,3 +199,74 @@ def _validate_part( return part raise UnsupportedContentError(f"messages[{msg_idx}].content: unsupported content part type {ptype!r}") + + +@dataclass(frozen=True) +class ParsedChatOutput: + """Aggregate result of parsing a model's full chat-completion text. + + This is a loader-agnostic 3-field DTO representing the parsed output. + """ + + content: str | None + reasoning: str | None + tool_calls: list[ToolCall] = field(default_factory=list) + + @property + def has_tool_calls(self) -> bool: + return bool(self.tool_calls) + + +def build_from_parsed( + *, + request_id: str, + model_name: str, + choices: list[ParsedChatOutput], + usage: UsageInfo, + finish_reasons: list[str | None] | str | None = None, + created: int | None = None, + logprobs: list[Any] | None = None, +) -> ChatCompletionResponse: + """Build a ChatCompletionResponse from parsed choice DTOs. + + Allows multi-choice responses from day one. + """ + if created is None: + created = int(time.time()) + + response_choices = [] + for idx, parsed in enumerate(choices): + # Determine finish reason for this choice + if isinstance(finish_reasons, list): + fr = finish_reasons[idx] if idx < len(finish_reasons) else "stop" + elif isinstance(finish_reasons, str): + fr = finish_reasons + else: + # Fallback derivation logic similar to finish_reason_for + fr = "tool_calls" if parsed.has_tool_calls else "stop" + + choice_logprobs = None + if logprobs is not None and idx < len(logprobs): + choice_logprobs = logprobs[idx] + + response_choices.append( + ChatCompletionResponseChoice( + index=idx, + message=ChatMessage( + role="assistant", + content=parsed.content, + reasoning=parsed.reasoning, + tool_calls=parsed.tool_calls, + ), + logprobs=choice_logprobs, + finish_reason=fr, + ) + ) + + return ChatCompletionResponse( + id=request_id, + model=model_name, + choices=response_choices, + usage=usage, + created=created, + ) diff --git a/tests/test_chat_utils.py b/tests/test_chat_utils.py index 4df57cd..05a26b2 100644 --- a/tests/test_chat_utils.py +++ b/tests/test_chat_utils.py @@ -1,6 +1,11 @@ """Tests for modelship.openai.chat_utils.normalize_chat_messages.""" -from modelship.openai.chat_utils import normalize_chat_messages +from modelship.openai.chat_utils import ( + ParsedChatOutput, + build_from_parsed, + normalize_chat_messages, +) +from modelship.openai.protocol import FunctionCall, ToolCall, UsageInfo def test_tool_message_name_backfilled_from_assistant_call(): @@ -98,3 +103,74 @@ def test_text_part_collapse_still_works(): out = normalize_chat_messages(messages) assert out[0]["content"] == "hello\nworld" + + +def test_build_from_parsed_multi_choice_and_dto(): + choices = [ + ParsedChatOutput( + content="Hello from choice 0", + reasoning="Thinking 0...", + tool_calls=[ + ToolCall( + id="call_1", + type="function", + function=FunctionCall(name="get_weather", arguments="{}"), + ) + ], + ), + ParsedChatOutput( + content="Hello from choice 1", + reasoning="Thinking 1...", + tool_calls=[], + ), + ] + usage = UsageInfo(prompt_tokens=10, completion_tokens=20, total_tokens=30) + + # Test single string finish_reason + res1 = build_from_parsed( + request_id="test_req_1", + model_name="test_model", + choices=choices, + usage=usage, + finish_reasons="length", + created=12345, + ) + + assert res1.id == "test_req_1" + assert res1.model == "test_model" + assert len(res1.choices) == 2 + assert res1.choices[0].index == 0 + assert res1.choices[0].message.content == "Hello from choice 0" + assert res1.choices[0].message.reasoning == "Thinking 0..." + assert len(res1.choices[0].message.tool_calls) == 1 + assert res1.choices[0].message.tool_calls[0].function.name == "get_weather" + assert res1.choices[0].finish_reason == "length" + + assert res1.choices[1].index == 1 + assert res1.choices[1].message.content == "Hello from choice 1" + assert res1.choices[1].message.reasoning == "Thinking 1..." + assert len(res1.choices[1].message.tool_calls) == 0 + assert res1.choices[1].finish_reason == "length" + + # Test automatic finish_reason derivation + res2 = build_from_parsed( + request_id="test_req_2", + model_name="test_model", + choices=choices, + usage=usage, + created=12345, + ) + assert res2.choices[0].finish_reason == "tool_calls" + assert res2.choices[1].finish_reason == "stop" + + # Test list finish_reasons + res3 = build_from_parsed( + request_id="test_req_3", + model_name="test_model", + choices=choices, + usage=usage, + finish_reasons=["length", "stop"], + created=12345, + ) + assert res3.choices[0].finish_reason == "length" + assert res3.choices[1].finish_reason == "stop" From 51b92a7314879073a6f6da999d6f5a6ae5e0ac9a Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:23:48 +0000 Subject: [PATCH 08/35] fix: address concurrency, early-crash thread leaks, and closed-loop shutdown issues in llama_server loader - Initialize and apply threading.Lock synchronization to all operations (append, clear, join) on self._recent_log_lines to prevent RuntimeError: deque mutated during iteration. - Invoke self.shutdown() on _EarlyCrashError in start() to cleanly close pipes, reap the failed subprocess, and terminate old log threads before retrying launch on a fresh port. - Wrap the entire loop acquisition and task creation logic inside shutdown() within the try...except RuntimeError block to safely catch and ignore RuntimeError raised when the event loop is closed/closing during teardown. --- .../infer/llama_server/llama_server_infer.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 6f627f9..55c2cd9 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -97,6 +97,7 @@ def __init__(self, model_config: ModelshipModelConfig): self._port: int | None = None self._api_key: str = secrets.token_hex(32) self._client: httpx.AsyncClient | None = None + self._log_lock = threading.Lock() self._recent_log_lines: deque[str] = deque(maxlen=_RECENT_LOG_LINES) self._log_threads: list[threading.Thread] = [] @@ -134,12 +135,11 @@ def _wait_and_kill(): client, self._client = self._client, None try: loop = asyncio.get_running_loop() - except RuntimeError: - pass # no running loop (e.g. interpreter teardown) — let GC reclaim the socket - else: task = loop.create_task(client.aclose()) _pending_client_closes.add(task) task.add_done_callback(_pending_client_closes.discard) + except RuntimeError: + pass # no running loop or closed loop (e.g. interpreter teardown) — let GC reclaim the socket def __del__(self): self.shutdown() @@ -175,6 +175,7 @@ async def start(self) -> None: _LAUNCH_RETRY_LIMIT, e, ) + self.shutdown() else: raise RuntimeError( f"llama-server for '{self.model_config.name}' failed to start after " @@ -238,7 +239,8 @@ async def _launch(self, binary: str, model_path: str) -> None: errors="replace", ), ) - self._recent_log_lines.clear() + with self._log_lock: + self._recent_log_lines.clear() assert self._proc is not None assert self._proc.stdout is not None assert self._proc.stderr is not None @@ -266,7 +268,8 @@ def _drain_stream(self, stream: Any, tag: str) -> None: for raw_line in iter(stream.readline, ""): line = raw_line.rstrip("\n") logger.log(TRACE, "[%s:%s] %s", self.model_config.name, tag, line) - self._recent_log_lines.append(line) + with self._log_lock: + self._recent_log_lines.append(line) except (ValueError, OSError): pass # stream closed from the main thread while we were reading finally: @@ -286,7 +289,8 @@ async def _wait_healthy(self) -> None: while True: rc = self._proc.poll() if rc is not None: - tail = "\n".join(self._recent_log_lines) + with self._log_lock: + tail = "\n".join(self._recent_log_lines) message = f"llama-server for '{self.model_config.name}' exited (rc={rc}) during startup: {tail}" if time.monotonic() - spawned_at < _EARLY_CRASH_WINDOW_S: raise _EarlyCrashError(message) From a2829a3e22fba93cb319438092ff34cf309daa24 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:29:42 +0000 Subject: [PATCH 09/35] feat: implement embeddings, vision, logprobs, and concurrency coupling for llama_server loader (Stage B4) - Add `--embedding` flag to llama-server launch args under `usecase: embed` and implement `create_embedding` with a secure, projection-based response model. - Add `mmproj` config field to `LlamaServerConfig` and configure resolve-on-startup. - Pass `supports_image` based on `mmproj` config state to `normalize_chat_messages` to enable gateway-side multimodal content validation. - Default `max_ongoing_requests` to `parallel` for the `llama_server` loader. - Support logprobs and top_logprobs forwarding, as well as Choice/Stream-choice projection. - Add comprehensive test coverage in tests/test_llama_server_infer.py and document prompt cache drop in docs/model-configuration.md. --- docs/model-configuration.md | 2 + modelship/deploy/actor_options.py | 9 +- modelship/deploy/config.py | 7 + modelship/infer/infer_config.py | 2 + .../infer/llama_server/llama_server_infer.py | 119 +++++++++++- tests/test_llama_server_infer.py | 183 +++++++++++++++++- 6 files changed, 312 insertions(+), 10 deletions(-) diff --git a/docs/model-configuration.md b/docs/model-configuration.md index 0eee591..02c7940 100644 --- a/docs/model-configuration.md +++ b/docs/model-configuration.md @@ -425,6 +425,8 @@ models: capacity: 4GiB ``` +> **Note for the `llama_server` loader:** The `llama_server` loader does not support the explicit `cache` configuration block. It manages request-level prefix caching internally and automatically within its thread slots (analogous to standard in-memory caching), but does not support modelship's persistent on-disk prompt cache (`type: disk`) or explicit capacity configuration. + #### Tool calling (`tool_choice`) Tool-call behavior is driven entirely by the per-request OpenAI `tool_choice` parameter — there is no deploy-level flag. Sending a `tools` array is the opt-in. diff --git a/modelship/deploy/actor_options.py b/modelship/deploy/actor_options.py index bb1db78..47f9757 100644 --- a/modelship/deploy/actor_options.py +++ b/modelship/deploy/actor_options.py @@ -152,6 +152,11 @@ def build_deployment_options(config: ModelshipModelConfig, plugin_wheel: Path | # Per-model Ray Serve concurrency cap; only override the default when set. # The reservation helpers read only the GPU/CPU keys, so this is inert there. - if config.max_ongoing_requests is not None: - opts["max_ongoing_requests"] = config.max_ongoing_requests + max_ongoing = config.max_ongoing_requests + if max_ongoing is None and config.loader == ModelLoader.llama_server: + parallel = config.llama_server_config.parallel if config.llama_server_config else 1 + max_ongoing = parallel + + if max_ongoing is not None: + opts["max_ongoing_requests"] = max_ongoing return opts diff --git a/modelship/deploy/config.py b/modelship/deploy/config.py index 18504bd..d127a67 100644 --- a/modelship/deploy/config.py +++ b/modelship/deploy/config.py @@ -157,6 +157,13 @@ def resolve_all_model_sources(yml_conf: ModelshipConfig) -> None: cfg._resolved_path = resolve_model_source(cfg.model, trust_remote_code=trust_remote_code) logger.info("Resolved '%s' -> %s", cfg.name, cfg._resolved_path) + if cfg.loader == ModelLoader.llama_server and cfg.llama_server_config and cfg.llama_server_config.mmproj: + logger.info("Resolving mmproj source for '%s': %s", cfg.name, cfg.llama_server_config.mmproj) + cfg.llama_server_config.mmproj = resolve_model_source( + cfg.llama_server_config.mmproj, trust_remote_code=trust_remote_code + ) + logger.info("Resolved mmproj -> %s", cfg.llama_server_config.mmproj) + # GGUF is not supported on the vllm loader: vLLM 0.24 moved GGUF out of # tree, and the only external plugin is incompatible with 0.24's # quantization API. Reject early with a pointer to llama_cpp instead of diff --git a/modelship/infer/infer_config.py b/modelship/infer/infer_config.py index 93188d7..d0811a1 100644 --- a/modelship/infer/infer_config.py +++ b/modelship/infer/infer_config.py @@ -180,6 +180,8 @@ class LlamaServerConfig(BaseModel): # Built-in template name (e.g. "chatml") or a path to a Jinja file; # None lets llama-server use the GGUF's embedded chat template. chat_template: str | None = None + # Path to the multimodal projector file (e.g. clip-model-f16.gguf) + mmproj: str | None = None # Escape hatch for launch flags not otherwise surfaced, appended verbatim. extra_args: list[str] = Field(default_factory=list) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 55c2cd9..3bb8fae 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -14,7 +14,7 @@ import httpx from modelship.infer.base_infer import BaseInfer -from modelship.infer.infer_config import LlamaServerConfig, ModelshipModelConfig, RawRequestProxy +from modelship.infer.infer_config import LlamaServerConfig, ModelshipModelConfig, ModelUsecase, RawRequestProxy from modelship.logging import TRACE, get_logger from modelship.openai.chat_utils import ( ParsedChatOutput, @@ -23,6 +23,7 @@ normalize_chat_messages, ) from modelship.openai.protocol import ( + ChatCompletionLogProbs, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseStreamChoice, @@ -30,6 +31,9 @@ DeltaFunctionCall, DeltaMessage, DeltaToolCall, + EmbeddingRequest, + EmbeddingResponse, + EmbeddingResponseData, ErrorResponse, FunctionCall, ToolCall, @@ -225,6 +229,18 @@ async def _launch(self, binary: str, model_path: str) -> None: if self.config.chat_template: flag = "--chat-template-file" if os.path.isfile(self.config.chat_template) else "--chat-template" args += [flag, self.config.chat_template] + if self.config.mmproj: + from modelship.infer.model_resolver import resolve_model_source + + mmproj_ref = self.config.mmproj + try: + mmproj_path = await loop.run_in_executor(None, lambda: resolve_model_source(mmproj_ref)) + except Exception as e: + logger.warning("Failed to resolve mmproj %r, using as is: %s", mmproj_ref, e) + mmproj_path = mmproj_ref + args += ["--mmproj", mmproj_path] + if self.model_config.usecase == ModelUsecase.embed: + args += ["--embedding"] args += list(self.config.extra_args) logger.info("llama-server launch args for '%s': %s", self.model_config.name, _redact(args)) @@ -312,6 +328,19 @@ async def _wait_healthy(self) -> None: await asyncio.sleep(_HEALTH_POLL_INTERVAL_S) async def warmup(self) -> None: + if self.model_config.usecase == ModelUsecase.embed: + logger.info("Warming up llama-server embedding model: %s", self.model_config.name) + request = EmbeddingRequest( + model=self.model_config.name, + input="warmup", + ) + result = await self.create_embedding(request, RawRequestProxy(None, {})) + if isinstance(result, ErrorResponse): + logger.warning("warmup embedding failed for '%s': %s", self.model_config.name, result.error.message) + else: + logger.info("warmup embedding done for %s", self.model_config.name) + return + logger.info("Warming up llama-server chat model: %s", self.model_config.name) request = ChatCompletionRequest( model=self.model_config.name, @@ -324,6 +353,33 @@ async def warmup(self) -> None: else: logger.info("warmup chat done for %s", self.model_config.name) + async def create_embedding( + self, request: EmbeddingRequest, raw_request: RawRequestProxy + ) -> ErrorResponse | EmbeddingResponse: + if self._client is None: + return await super().create_embedding(request, raw_request) + + request_id = f"embd-{base_request_id(raw_request)}" + logger.info("embedding request %s", request_id) + + payload = request.model_dump(exclude_none=True, exclude={"model"}) + payload["model"] = self.model_config.name + + try: + resp = await self._client.post("/v1/embeddings", json=payload) + resp.raise_for_status() + except httpx.HTTPStatusError as e: + detail = _extract_error_detail(e.response) + logger.warning("embedding request %s failed: %s", request_id, detail) + return create_error_response(detail, status_code=e.response.status_code) + except httpx.HTTPError as e: + logger.warning("embedding request %s failed: %s", request_id, e) + return create_error_response(f"llama-server request failed: {e}", status_code=502) + + data = resp.json() + logger.log(TRACE, "embedding response %s: %s", request_id, data) + return _project_embedding_response(data, model_name=self.model_config.name) + async def create_chat_completion( self, request: ChatCompletionRequest, raw_request: RawRequestProxy ) -> ErrorResponse | ChatCompletionResponse | AsyncGenerator[str, None]: @@ -341,8 +397,9 @@ async def create_chat_completion( request.tool_choice, ) + supports_image = bool(self.config.mmproj) try: - messages = normalize_chat_messages(request.messages, supports_image=False, supports_audio=False) + messages = normalize_chat_messages(request.messages, supports_image=supports_image, supports_audio=False) except UnsupportedContentError as e: logger.warning("chat request %s rejected: %s", request_id, e) return create_error_response(e) @@ -422,13 +479,17 @@ def _redact(args: list[str]) -> list[str]: def _build_payload(request: ChatCompletionRequest, messages: list[dict], *, model_name: str) -> dict[str, Any]: - # logprobs/top_logprobs aren't wired into the response projection yet (B3 - # adds that). ChatCompletionRequest defaults top_logprobs to 0 (not None), - # so exclude_none alone would still forward it — and llama-server rejects - # top_logprobs being present at all unless logprobs=true, even at 0. - payload = request.model_dump(exclude_none=True, exclude={"messages", "model", "logprobs", "top_logprobs"}) + # If request.logprobs is True, we want to forward logprobs and top_logprobs (if > 0) to llama-server. + exclude_fields = {"messages", "model", "logprobs", "top_logprobs"} + payload = request.model_dump(exclude_none=True, exclude=exclude_fields) payload["messages"] = messages payload["model"] = model_name + + if request.logprobs: + payload["logprobs"] = True + if request.top_logprobs is not None and request.top_logprobs > 0: + payload["top_logprobs"] = request.top_logprobs + return payload @@ -471,6 +532,7 @@ def _project_usage(raw_usage: dict | None) -> UsageInfo: def _project_chat_response(data: dict, *, model_name: str, request_id: str) -> ChatCompletionResponse: choices = [] finish_reasons = [] + logprobs_list = [] for choice in data.get("choices", []): message = choice.get("message") or {} dto = ParsedChatOutput( @@ -481,6 +543,14 @@ def _project_chat_response(data: dict, *, model_name: str, request_id: str) -> C choices.append(dto) finish_reasons.append(choice.get("finish_reason") or "stop") + choice_logprobs = None + if choice.get("logprobs") is not None: + try: + choice_logprobs = ChatCompletionLogProbs.model_validate(choice.get("logprobs")) + except Exception as e: + logger.warning("Failed to validate choice logprobs: %s", e) + logprobs_list.append(choice_logprobs) + return build_from_parsed( request_id=data.get("id") or request_id, model_name=model_name, @@ -488,6 +558,7 @@ def _project_chat_response(data: dict, *, model_name: str, request_id: str) -> C usage=_project_usage(data.get("usage")), finish_reasons=finish_reasons, created=data.get("created"), + logprobs=logprobs_list, ) @@ -506,6 +577,14 @@ def _project_stream_chunk(data: dict, *, model_name: str) -> ChatCompletionStrea function=DeltaFunctionCall(name=function.get("name"), arguments=function.get("arguments")), ) ) + + choice_logprobs = None + if choice.get("logprobs") is not None: + try: + choice_logprobs = ChatCompletionLogProbs.model_validate(choice.get("logprobs")) + except Exception as e: + logger.warning("Failed to validate stream choice logprobs: %s", e) + choices.append( ChatCompletionResponseStreamChoice( index=choice.get("index", 0), @@ -515,6 +594,7 @@ def _project_stream_chunk(data: dict, *, model_name: str) -> ChatCompletionStrea reasoning=delta.get("reasoning_content"), tool_calls=tool_calls or None, ), + logprobs=choice_logprobs, finish_reason=choice.get("finish_reason"), ) ) @@ -525,6 +605,31 @@ def _project_stream_chunk(data: dict, *, model_name: str) -> ChatCompletionStrea ) +def _project_embedding_response(data: dict, *, model_name: str) -> EmbeddingResponse: + raw_data_list = data.get("data", []) + projected_data = [] + for item in raw_data_list: + projected_data.append( + EmbeddingResponseData( + index=item.get("index", 0), embedding=item.get("embedding", []), object=item.get("object", "embedding") + ) + ) + raw_usage = data.get("usage") or {} + usage = UsageInfo( + prompt_tokens=raw_usage.get("prompt_tokens", 0) or 0, + completion_tokens=raw_usage.get("completion_tokens", 0) or 0, + total_tokens=raw_usage.get("total_tokens", 0) or 0, + ) + return EmbeddingResponse( + id=data.get("id") or f"embd-{random_uuid()}", + object=data.get("object", "list"), + created=data.get("created") or int(time.time()), + model=model_name, + data=projected_data, + usage=usage, + ) + + def _encode_chunk(chunk: ChatCompletionStreamResponse) -> str: return f"data: {json.dumps(chunk.model_dump(mode='json'))}\n\n" diff --git a/tests/test_llama_server_infer.py b/tests/test_llama_server_infer.py index e388ae0..5e6e7fe 100644 --- a/tests/test_llama_server_infer.py +++ b/tests/test_llama_server_infer.py @@ -21,7 +21,7 @@ RawRequestProxy, ) from modelship.infer.llama_server.llama_server_infer import LlamaServerInfer -from modelship.openai.protocol import ChatCompletionRequest, ErrorResponse +from modelship.openai.protocol import ChatCompletionRequest, EmbeddingRequest, EmbeddingResponse, ErrorResponse # --------------------------------------------------------------------------- # Fake llama-server executables (plain scripts; no real binary needed in CI) @@ -396,3 +396,184 @@ def handler(request: httpx.Request) -> httpx.Response: chunks = [chunk async for chunk in result] assert any("connection reset" in c for c in chunks) assert chunks[-1] == "data: [DONE]\n\n" + + @pytest.mark.asyncio + async def test_stage_b4_embedding_support(self, tmp_path, monkeypatch): + # Verify launch args has --embedding when usecase: embed + binary = _write_fake_executable(tmp_path, _FAKE_HEALTHY_SERVER) + monkeypatch.setenv("MSHIP_LLAMA_SERVER_BIN", binary) + + model_config = ModelshipModelConfig( + name="test-model", + model="org/test-model", + usecase=ModelUsecase.embed, + loader=ModelLoader.llama_server, + ) + model_config._resolved_path = "/fake/model.gguf" + + infer = LlamaServerInfer(model_config) + await infer.start() + try: + args = list(infer._proc.args) + assert "--embedding" in args + finally: + infer.shutdown() + + # Verify create_embedding with projected embedding response + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/embeddings" + return httpx.Response( + 200, + json={ + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [0.1, 0.2, 0.3], + } + ], + "model": "test-model", + "usage": { + "prompt_tokens": 5, + "completion_tokens": 0, + "total_tokens": 5, + }, + "timings": "secret", # extension field to be stripped + }, + ) + + infer_client = LlamaServerInfer(model_config) + infer_client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler), base_url="http://test") + + req = EmbeddingRequest(model="test-model", input="hello") + res = await infer_client.create_embedding(req, RawRequestProxy(None, {})) + assert isinstance(res, EmbeddingResponse) + assert res.data[0].embedding == [0.1, 0.2, 0.3] + # Extra field is not projected + assert "timings" not in res.model_dump() + + @pytest.mark.asyncio + async def test_stage_b4_vision_support(self, tmp_path, monkeypatch): + # Verify launch args has --mmproj when configured + binary = _write_fake_executable(tmp_path, _FAKE_HEALTHY_SERVER) + monkeypatch.setenv("MSHIP_LLAMA_SERVER_BIN", binary) + + model_config = ModelshipModelConfig( + name="test-model", + model="org/test-model", + usecase=ModelUsecase.generate, + loader=ModelLoader.llama_server, + llama_server_config=LlamaServerConfig(mmproj="foo/mmproj.gguf"), + ) + model_config._resolved_path = "/fake/model.gguf" + + infer = LlamaServerInfer(model_config) + + with patch("modelship.infer.model_resolver.resolve_model_source", return_value="/resolved/mmproj.gguf"): + await infer.start() + try: + args = list(infer._proc.args) + assert "--mmproj" in args + assert "/resolved/mmproj.gguf" in args + finally: + infer.shutdown() + + # Verify gateway rejection is skipped when mmproj is configured + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"choices": [{"index": 0, "message": {"role": "assistant", "content": "I see an image"}}]}, + ) + + infer_client = LlamaServerInfer(model_config) + infer_client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler), base_url="http://test") + + req = ChatCompletionRequest( + model="test-model", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}, + ], + } + ], + ) + res = await infer_client.create_chat_completion(req, RawRequestProxy(None, {})) + assert not isinstance(res, ErrorResponse) + assert res.choices[0].message.content == "I see an image" + + @pytest.mark.asyncio + async def test_stage_b4_concurrency_coupling(self): + # Verify max_ongoing_requests is set to parallel by default when unset + model_config = ModelshipModelConfig( + name="test-model", + model="org/test-model", + usecase=ModelUsecase.generate, + loader=ModelLoader.llama_server, + llama_server_config=LlamaServerConfig(parallel=4), + ) + from modelship.deploy.actor_options import build_deployment_options + + opts = build_deployment_options(model_config) + assert opts["max_ongoing_requests"] == 4 + + # When explicitly configured, it should keep the explicit value + model_config.max_ongoing_requests = 10 + opts = build_deployment_options(model_config) + assert opts["max_ongoing_requests"] == 10 + + @pytest.mark.asyncio + async def test_stage_b4_logprobs_support(self): + # Verify logprobs in non-streaming response + def handler(request: httpx.Request) -> httpx.Response: + import json + + payload = json.loads(request.read()) + assert payload.get("logprobs") is True + assert payload.get("top_logprobs") == 2 + return httpx.Response( + 200, + json={ + "choices": [ + { + "message": {"content": "hello"}, + "logprobs": { + "content": [ + { + "token": "hello", + "logprob": -0.1, + "top_logprobs": [ + {"token": "hello", "logprob": -0.1}, + {"token": "hi", "logprob": -1.5}, + ], + } + ] + }, + } + ] + }, + ) + + model_config = ModelshipModelConfig( + name="test-model", + model="org/test-model", + usecase=ModelUsecase.generate, + loader=ModelLoader.llama_server, + ) + infer_client = LlamaServerInfer(model_config) + infer_client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler), base_url="http://test") + + req = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + logprobs=True, + top_logprobs=2, + ) + res = await infer_client.create_chat_completion(req, RawRequestProxy(None, {})) + assert not isinstance(res, ErrorResponse) + assert res.choices[0].logprobs is not None + assert res.choices[0].logprobs.content[0].token == "hello" + assert res.choices[0].logprobs.content[0].top_logprobs[1].token == "hi" From c3915b5c415c2aa565b9936c398481113f423297 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:42:33 +0000 Subject: [PATCH 10/35] fix: call self.shutdown() on any exception during llama-server startup - Wrap the entire startup retry loop in a try...except Exception block to guarantee that self.shutdown() is called on any launch, wait-healthy, or timeout exception. - This prevents potential background subprocess and log-draining thread resource leaks on startup failures. --- .../infer/llama_server/llama_server_infer.py | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 3bb8fae..dfb9e03 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -165,26 +165,30 @@ async def start(self) -> None: logger.info("Starting llama-server for model: %s", self.model_config.name) last_error: Exception | None = None - for attempt in range(1, _LAUNCH_RETRY_LIMIT + 1): - try: - await self._launch(binary, model_path) - await self._wait_healthy() - break - except _EarlyCrashError as e: - last_error = e - logger.warning( - "llama-server for '%s' exited immediately on attempt %d/%d (likely a port race); retrying: %s", - self.model_config.name, - attempt, - _LAUNCH_RETRY_LIMIT, - e, + try: + for attempt in range(1, _LAUNCH_RETRY_LIMIT + 1): + try: + await self._launch(binary, model_path) + await self._wait_healthy() + break + except _EarlyCrashError as e: + last_error = e + logger.warning( + "llama-server for '%s' exited immediately on attempt %d/%d (likely a port race); retrying: %s", + self.model_config.name, + attempt, + _LAUNCH_RETRY_LIMIT, + e, + ) + self.shutdown() + else: + raise RuntimeError( + f"llama-server for '{self.model_config.name}' failed to start after " + f"{_LAUNCH_RETRY_LIMIT} attempts: {last_error}" ) - self.shutdown() - else: - raise RuntimeError( - f"llama-server for '{self.model_config.name}' failed to start after " - f"{_LAUNCH_RETRY_LIMIT} attempts: {last_error}" - ) + except Exception: + self.shutdown() + raise assert self._port is not None self._client = httpx.AsyncClient( From dc294df2b25e8bf4c1d4d00bf61a843fc35beff5 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 12:42:21 +0000 Subject: [PATCH 11/35] fix: intercept and parse inline JSON error payloads on 2xx responses in llama_server loader - Add an explicit 'error' key check to create_chat_completion and create_embedding post-response processing. - If a 2xx response contains a JSON body with an 'error' key, extract the error message and gracefully return an ErrorResponse instead of trying to project malformed choices/embeddings. - Add robust unit testing verifying inline error payloads for both chat completions and embeddings. --- .../infer/llama_server/llama_server_infer.py | 10 +++++ tests/test_llama_server_infer.py | 45 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index dfb9e03..866a746 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -382,6 +382,11 @@ async def create_embedding( data = resp.json() logger.log(TRACE, "embedding response %s: %s", request_id, data) + if isinstance(data, dict) and "error" in data: + error_data = data["error"] or {} + message = error_data.get("message") if isinstance(error_data, dict) else str(error_data) + logger.warning("embedding request %s failed with inline error: %s", request_id, message) + return create_error_response(message or "Unknown error returned from llama-server", status_code=502) return _project_embedding_response(data, model_name=self.model_config.name) async def create_chat_completion( @@ -426,6 +431,11 @@ async def create_chat_completion( data = resp.json() logger.log(TRACE, "chat response %s: %s", request_id, data) + if isinstance(data, dict) and "error" in data: + error_data = data["error"] or {} + message = error_data.get("message") if isinstance(error_data, dict) else str(error_data) + logger.warning("chat request %s failed with inline error: %s", request_id, message) + return create_error_response(message or "Unknown error returned from llama-server", status_code=502) return _project_chat_response(data, model_name=self.model_config.name, request_id=request_id) async def _stream_chat_completion(self, payload: dict[str, Any], request_id: str) -> AsyncGenerator[str, None]: diff --git a/tests/test_llama_server_infer.py b/tests/test_llama_server_infer.py index 5e6e7fe..d0fe396 100644 --- a/tests/test_llama_server_infer.py +++ b/tests/test_llama_server_infer.py @@ -577,3 +577,48 @@ def handler(request: httpx.Request) -> httpx.Response: assert res.choices[0].logprobs is not None assert res.choices[0].logprobs.content[0].token == "hello" assert res.choices[0].logprobs.content[0].top_logprobs[1].token == "hi" + + @pytest.mark.asyncio + async def test_stage_b4_inline_json_errors_handled(self): + # Verify 200 OK with inline error payload in chat completion + def chat_handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"error": {"message": "inline chat error detail"}}, + ) + + model_config = ModelshipModelConfig( + name="test-model", + model="org/test-model", + usecase=ModelUsecase.generate, + loader=ModelLoader.llama_server, + ) + infer_client = LlamaServerInfer(model_config) + infer_client._client = httpx.AsyncClient(transport=httpx.MockTransport(chat_handler), base_url="http://test") + + res = await infer_client.create_chat_completion(_request(), RawRequestProxy(None, {})) + assert isinstance(res, ErrorResponse) + assert res.error.message == "inline chat error detail" + + # Verify 200 OK with inline error payload in embedding + def embed_handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"error": "inline embedding error detail"}, + ) + + model_config_embed = ModelshipModelConfig( + name="test-model", + model="org/test-model", + usecase=ModelUsecase.embed, + loader=ModelLoader.llama_server, + ) + infer_client_embed = LlamaServerInfer(model_config_embed) + infer_client_embed._client = httpx.AsyncClient( + transport=httpx.MockTransport(embed_handler), base_url="http://test" + ) + + req = EmbeddingRequest(model="test-model", input="hello") + res_embed = await infer_client_embed.create_embedding(req, RawRequestProxy(None, {})) + assert isinstance(res_embed, ErrorResponse) + assert res_embed.error.message == "inline embedding error detail" From c9c79b83d1862443fb935b977d06cd04c76d2cc3 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 13:28:07 +0000 Subject: [PATCH 12/35] fix: intercept and handle mid-stream JSON error payloads from llama-server - Add an explicit 'error' key check on decoded JSON chunks in _stream_chat_completion. - If a chunk contains an 'error' key, yield an encoded error chunk, send the stream termination [DONE] chunk, and terminate the generator by returning. - Add unit testing to verify that mid-stream errors are correctly intercepted, output, and terminate the stream gracefully. --- .../infer/llama_server/llama_server_infer.py | 7 +++++++ tests/test_llama_server_infer.py | 21 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 866a746..9d397fd 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -464,6 +464,13 @@ async def _stream_chat_completion(self, payload: dict[str, Any], request_id: str except json.JSONDecodeError: logger.warning("chat request %s: unparseable stream chunk: %r", request_id, data_str) continue + if isinstance(data, dict) and "error" in data: + error_data = data["error"] or {} + message = error_data.get("message") if isinstance(error_data, dict) else str(error_data) + logger.warning("chat request %s failed mid-stream with error: %s", request_id, message) + yield _encode_error(message or "Unknown mid-stream error from llama-server") + yield "data: [DONE]\n\n" + return chunk = _project_stream_chunk(data, model_name=self.model_config.name) for choice in chunk.choices: if choice.delta.content: diff --git a/tests/test_llama_server_infer.py b/tests/test_llama_server_infer.py index d0fe396..6dc24bf 100644 --- a/tests/test_llama_server_infer.py +++ b/tests/test_llama_server_infer.py @@ -622,3 +622,24 @@ def embed_handler(request: httpx.Request) -> httpx.Response: res_embed = await infer_client_embed.create_embedding(req, RawRequestProxy(None, {})) assert isinstance(res_embed, ErrorResponse) assert res_embed.error.message == "inline embedding error detail" + + @pytest.mark.asyncio + async def test_stage_b4_mid_stream_json_errors_handled(self): + # Verify mid-stream JSON error terminates stream and yields error + sse_body = ( + 'data: {"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n' + 'data: {"choices":[{"index":0,"delta":{"content":"Hel"},"finish_reason":null}]}\n\n' + 'data: {"error": {"message": "inline mid-stream error detail"}}\n\n' + ) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=sse_body.encode(), headers={"content-type": "text/event-stream"}) + + infer = _infer_with_client(handler) + result = await infer.create_chat_completion(_request(stream=True), RawRequestProxy(None, {})) + + chunks = [chunk async for chunk in result] + # First 2 are normal choices, then the error chunk, then [DONE] + assert len(chunks) == 4 + assert "inline mid-stream error detail" in chunks[2] + assert chunks[-1] == "data: [DONE]\n\n" From d24a5fd16df40b06a1c5e91df6af43451ef78bcd Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 13:56:56 +0000 Subject: [PATCH 13/35] fix: secure pending_client_closes against python interpreter teardown - Add explicit None checks for module-level _pending_client_closes before accesses. - Catch AttributeError and TypeError in addition to RuntimeError during loop task scheduling inside shutdown() to handle arbritrary garbage collection order of module-level globals during python interpreter teardown. --- modelship/infer/llama_server/llama_server_infer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 9d397fd..7f766a5 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -140,9 +140,12 @@ def _wait_and_kill(): try: loop = asyncio.get_running_loop() task = loop.create_task(client.aclose()) - _pending_client_closes.add(task) - task.add_done_callback(_pending_client_closes.discard) - except RuntimeError: + if _pending_client_closes is not None: + _pending_client_closes.add(task) + task.add_done_callback( + lambda t: _pending_client_closes.discard(t) if _pending_client_closes is not None else None + ) + except (RuntimeError, AttributeError, TypeError): pass # no running loop or closed loop (e.g. interpreter teardown) — let GC reclaim the socket def __del__(self): From 0cc92916b09759165b1c0e0e446b34f4d460eb2f Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:00:32 +0000 Subject: [PATCH 14/35] fix: suppress interpreter teardown exceptions inside __del__ - Use contextlib.suppress(BaseException) to wrap self.shutdown() inside __del__. - This ensures any shutdown exceptions due to garbage collection of referenced modules/globals are silently ignored during interpreter exit, preventing noisy warnings. --- modelship/infer/llama_server/llama_server_infer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 7f766a5..8c4cdc3 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -149,7 +149,8 @@ def _wait_and_kill(): pass # no running loop or closed loop (e.g. interpreter teardown) — let GC reclaim the socket def __del__(self): - self.shutdown() + with contextlib.suppress(BaseException): + self.shutdown() async def start(self) -> None: binary = os.environ.get("MSHIP_LLAMA_SERVER_BIN") From 9a73c8407be2b1db7f59760fa0c8713408cab396 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:04:05 +0000 Subject: [PATCH 15/35] fix: use explicit None check for created timestamp fallback in embeddings projection - Replace the 'or' operator with an explicit 'is None' check for the 'created' field in _project_embedding_response. - This ensures that valid falsy numeric values (like 0 for a timestamp) are not unintentionally overridden with the current time fallback. --- modelship/infer/llama_server/llama_server_infer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 8c4cdc3..47b77ad 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -645,10 +645,13 @@ def _project_embedding_response(data: dict, *, model_name: str) -> EmbeddingResp completion_tokens=raw_usage.get("completion_tokens", 0) or 0, total_tokens=raw_usage.get("total_tokens", 0) or 0, ) + created = data.get("created") + if created is None: + created = int(time.time()) return EmbeddingResponse( id=data.get("id") or f"embd-{random_uuid()}", object=data.get("object", "list"), - created=data.get("created") or int(time.time()), + created=created, model=model_name, data=projected_data, usage=usage, From ce06e724416c5e3121c8e0831f944d598cf924d0 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:38:21 +0000 Subject: [PATCH 16/35] fix: derive finish_reason for out-of-range list entries instead of hardcoding stop Previously a finish_reasons list shorter than choices silently fell back to "stop" even when the choice had tool calls; now it uses the same tool-calls-aware derivation as the no-finish_reasons case. --- modelship/openai/chat_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelship/openai/chat_utils.py b/modelship/openai/chat_utils.py index 013c93f..e9fa61b 100644 --- a/modelship/openai/chat_utils.py +++ b/modelship/openai/chat_utils.py @@ -237,8 +237,8 @@ def build_from_parsed( response_choices = [] for idx, parsed in enumerate(choices): # Determine finish reason for this choice - if isinstance(finish_reasons, list): - fr = finish_reasons[idx] if idx < len(finish_reasons) else "stop" + if isinstance(finish_reasons, list) and idx < len(finish_reasons): + fr = finish_reasons[idx] elif isinstance(finish_reasons, str): fr = finish_reasons else: From 8445cb2929e90efe4a540f30c1c38cbd1b7df62e Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:38:27 +0000 Subject: [PATCH 17/35] fix: guard against malformed and non-object JSON responses from llama-server resp.json() could raise JSONDecodeError uncaught on a malformed body, and a valid-but-non-dict payload (e.g. list/string) would reach the response projection helpers and raise AttributeError. Applies to non-streaming chat and embedding responses as well as individual SSE stream chunks. --- .../infer/llama_server/llama_server_infer.py | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 47b77ad..ab22c74 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -384,9 +384,12 @@ async def create_embedding( logger.warning("embedding request %s failed: %s", request_id, e) return create_error_response(f"llama-server request failed: {e}", status_code=502) - data = resp.json() + data = _parse_json_object(resp) logger.log(TRACE, "embedding response %s: %s", request_id, data) - if isinstance(data, dict) and "error" in data: + if data is None: + logger.warning("embedding request %s returned a malformed response: %s", request_id, resp.text) + return create_error_response("llama-server returned a malformed response", status_code=502) + if "error" in data: error_data = data["error"] or {} message = error_data.get("message") if isinstance(error_data, dict) else str(error_data) logger.warning("embedding request %s failed with inline error: %s", request_id, message) @@ -433,9 +436,12 @@ async def create_chat_completion( logger.warning("chat request %s failed: %s", request_id, e) return create_error_response(f"llama-server request failed: {e}", status_code=502) - data = resp.json() + data = _parse_json_object(resp) logger.log(TRACE, "chat response %s: %s", request_id, data) - if isinstance(data, dict) and "error" in data: + if data is None: + logger.warning("chat request %s returned a malformed response: %s", request_id, resp.text) + return create_error_response("llama-server returned a malformed response", status_code=502) + if "error" in data: error_data = data["error"] or {} message = error_data.get("message") if isinstance(error_data, dict) else str(error_data) logger.warning("chat request %s failed with inline error: %s", request_id, message) @@ -468,7 +474,10 @@ async def _stream_chat_completion(self, payload: dict[str, Any], request_id: str except json.JSONDecodeError: logger.warning("chat request %s: unparseable stream chunk: %r", request_id, data_str) continue - if isinstance(data, dict) and "error" in data: + if not isinstance(data, dict): + logger.warning("chat request %s: skipping non-object stream chunk: %r", request_id, data) + continue + if "error" in data: error_data = data["error"] or {} message = error_data.get("message") if isinstance(error_data, dict) else str(error_data) logger.warning("chat request %s failed mid-stream with error: %s", request_id, message) @@ -518,6 +527,15 @@ def _build_payload(request: ChatCompletionRequest, messages: list[dict], *, mode return payload +def _parse_json_object(response: httpx.Response) -> dict | None: + """Parse a response body as JSON, returning None if it isn't a JSON object.""" + try: + data = response.json() + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + def _extract_error_detail(response: httpx.Response) -> str: try: data = response.json() From a5664ed9f70e197911c717787e10cb71dd27cc60 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:20:19 +0000 Subject: [PATCH 18/35] fix: resolve mmproj once on the driver instead of again in the actor resolve_all_model_sources already resolves llama_server_config.mmproj to an absolute path before the actor launches, same as the primary model path (_resolved_path). The actor was redundantly re-resolving it with a try/except fallback that added no robustness, since a failure there just falls back to the already-resolved value. --- .../infer/llama_server/llama_server_infer.py | 13 ++++-------- tests/test_llama_server_infer.py | 20 ++++++++++--------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index ab22c74..86071d7 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -238,15 +238,10 @@ async def _launch(self, binary: str, model_path: str) -> None: flag = "--chat-template-file" if os.path.isfile(self.config.chat_template) else "--chat-template" args += [flag, self.config.chat_template] if self.config.mmproj: - from modelship.infer.model_resolver import resolve_model_source - - mmproj_ref = self.config.mmproj - try: - mmproj_path = await loop.run_in_executor(None, lambda: resolve_model_source(mmproj_ref)) - except Exception as e: - logger.warning("Failed to resolve mmproj %r, using as is: %s", mmproj_ref, e) - mmproj_path = mmproj_ref - args += ["--mmproj", mmproj_path] + # Resolved once on the driver by resolve_all_model_sources (same as + # the primary model path, self.model_config._resolved_path) — no + # re-resolution here. + args += ["--mmproj", self.config.mmproj] if self.model_config.usecase == ModelUsecase.embed: args += ["--embedding"] args += list(self.config.extra_args) diff --git a/tests/test_llama_server_infer.py b/tests/test_llama_server_infer.py index 6dc24bf..985b6c0 100644 --- a/tests/test_llama_server_infer.py +++ b/tests/test_llama_server_infer.py @@ -459,25 +459,27 @@ async def test_stage_b4_vision_support(self, tmp_path, monkeypatch): binary = _write_fake_executable(tmp_path, _FAKE_HEALTHY_SERVER) monkeypatch.setenv("MSHIP_LLAMA_SERVER_BIN", binary) + # mmproj is resolved once on the driver (resolve_all_model_sources), same + # as the primary model path — the config already carries the resolved + # path by the time the actor launches. model_config = ModelshipModelConfig( name="test-model", model="org/test-model", usecase=ModelUsecase.generate, loader=ModelLoader.llama_server, - llama_server_config=LlamaServerConfig(mmproj="foo/mmproj.gguf"), + llama_server_config=LlamaServerConfig(mmproj="/resolved/mmproj.gguf"), ) model_config._resolved_path = "/fake/model.gguf" infer = LlamaServerInfer(model_config) - with patch("modelship.infer.model_resolver.resolve_model_source", return_value="/resolved/mmproj.gguf"): - await infer.start() - try: - args = list(infer._proc.args) - assert "--mmproj" in args - assert "/resolved/mmproj.gguf" in args - finally: - infer.shutdown() + await infer.start() + try: + args = list(infer._proc.args) + assert "--mmproj" in args + assert "/resolved/mmproj.gguf" in args + finally: + infer.shutdown() # Verify gateway rejection is skipped when mmproj is configured def handler(request: httpx.Request) -> httpx.Response: From bde2966b4e946bb25193828741719a7f5a706183 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:20:26 +0000 Subject: [PATCH 19/35] docs: document the llama_server loader model-configuration.md gets a full loader section (field table, tool_choice and response_format gaps/wins vs llama_cpp, vision, embeddings, examples) plus the missing loader/llama_server_config table entries; CLAUDE.md and AGENTS.md architecture maps and sharp-edges gain llama_server coverage; config/examples gets a working llama-server.yaml. This loader shipped across several PRs without any of this ever landing. --- AGENTS.md | 3 +- CLAUDE.md | 3 +- config/examples/README.md | 1 + config/examples/llama-server.yaml | 47 +++++++++++++++++++++ docs/model-configuration.md | 69 +++++++++++++++++++++++++++++-- 5 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 config/examples/llama-server.yaml diff --git a/AGENTS.md b/AGENTS.md index b53aea6..bdfe417 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,7 +69,7 @@ The Docker image's `CMD` is `uv run --no-sync mship_deploy.py` (against the venv - `modelship/openai/api.py` — FastAPI gateway. Uses `RequestWatcher` + a single shared `DisconnectRegistry` Ray actor (keyed by request id) to propagate client disconnects across process boundaries. - `modelship/infer/model_deployment.py` — the single `@serve.deployment` actor class; lazily imports the right backend based on `config.loader`. - `modelship/infer/infer_config.py` — pydantic config schemas **and** `RawRequestProxy` / `DisconnectRegistry`. `RawRequestProxy` exists because FastAPI `Request` cannot cross Ray process boundaries; any new attribute vLLM reads from `raw_request` must be added there. -- `modelship/infer/{vllm,transformers,diffusers,llama_cpp,custom}/` — one subdir per loader. Each has an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. +- `modelship/infer/{vllm,transformers,diffusers,llama_cpp,custom}/` — one subdir per loader. Each has an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. `modelship/infer/llama_server/llama_server_infer.py` is a flat file with no `openai/` subpackage — it proxies a `llama-server` subprocess's own OpenAI-compatible HTTP API rather than parsing output in-process. - `modelship/plugins/base_plugin.py` — `BasePlugin` ABC that plugin packages subclass as `ModelPlugin`. - `plugins/*` — workspace packages, each opt-in via a root extra. The plugin module name and the extra name must match (`ensure_plugin()` calls `importlib.import_module(config.plugin)` and the error message says `uv sync --extra `). @@ -104,6 +104,7 @@ Commit messages matter: use Conventional Commits prefixes so the changelog gener - vLLM version is pinned (`vllm==0.24.0`). Do not bump casually — the TP scheduling logic in `mship_deploy.py:build_deployment_options` defaults to the Ray V2 executor, and the loader imports vLLM-internal `entrypoints.*` module paths that upstream restructures between minors. - **GGUF is not supported on the `vllm` loader.** 0.24 moved GGUF out of tree; the only external `vllm-gguf-plugin` (`0.0.2`) has a stale `override_quantization_method` signature incompatible with 0.24's quantization API (it breaks *all* quantized models, not just GGUF), so it is deliberately not installed. `resolve_all_model_sources` rejects a `.gguf` on the vllm loader at driver preflight and points to `llama_cpp`. For GGUF use `loader: llama_cpp`; feed the vllm loader safetensors or an AWQ/GPTQ/FP8 quant. - `llama_cpp` loader supports CPU or whole-GPU offload. The gpu extra installs a prebuilt CUDA (cu130) wheel from `abetlen.github.io/llama-cpp-python/whl/cu130`; the cpu extra keeps the plain PyPI CPU-only wheel. `num_gpus` must be `0` or a whole integer for llama_cpp — fractional is rejected at config time (llama.cpp has no VRAM-fraction knob). `num_gpus >= 1` honors `n_gpu_layers`; `stable_diffusion_cpp` remains forced to `num_gpus: 0` in `mship_deploy.py:build_deployment_options`. +- `llama_server` loader (GGUF, same whole-GPU-only `num_gpus` rule as `llama_cpp`) launches a `llama-server` subprocess — found via `MSHIP_LLAMA_SERVER_BIN`, pinned in the Docker images at `/opt/llama.cpp/llama-server.sh` — and proxies its native OpenAI API instead of parsing output in-process. `--parallel` slots give real request concurrency, unlike `llama_cpp`'s single `asyncio.Lock`. Tool-call/reasoning parsing is llama-server's own, auto-detected per chat template: named-function `tool_choice` forcing is unsupported globally (silently falls back to `auto`), and `tool_choice: required` is grammar-enforced for harmony-style templates but a silent no-op for hermes-style ones (e.g. Qwen3); bare `response_format: {"type": "json_object"}` (no `schema` key) is also unenforced despite llama-server's own docs claiming support — `type: json_schema` requests (what modelship sends whenever a schema is given) are unaffected. No persistent on-disk prompt cache. See `docs/model-configuration.md`'s llama_server section for the full field table and examples. - Metrics are on by default on port **8079** (not 8000). Disable with `--no-metrics` or `MSHIP_METRICS=false`. - Log level `TRACE` (below `DEBUG`) is a custom level and logs full request/response payloads. - Docker images are multi-arch (amd64 + arm64). The CPU image uses the unified `Dockerfile` with `--build-arg MSHIP_VARIANT=cpu` and has a different tag suffix (`:latest-cpu`). diff --git a/CLAUDE.md b/CLAUDE.md index 5c77006..b9425c4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,7 +49,7 @@ Docker's `CMD` is `uv run --no-sync mship_deploy.py` (auto-detecting CPUs/GPUs u - `modelship/openai/api.py` — FastAPI gateway. Uses `RequestWatcher` + a single shared `DisconnectRegistry` Ray actor (keyed by request id) to propagate client disconnects across process boundaries and cancel in-flight inference. - `modelship/infer/model_deployment.py` — the single `@serve.deployment` actor class; lazily imports the right backend from `config.loader`. - `modelship/infer/infer_config.py` — pydantic config schemas plus `RawRequestProxy` / `DisconnectRegistry`. `RawRequestProxy` exists because FastAPI `Request` can't cross Ray process boundaries. **Any new attribute vLLM reads from `raw_request` must be added there.** -- `modelship/infer/{vllm,transformers,diffusers,llama_cpp,custom}/` — one subdir per loader, each with an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. +- `modelship/infer/{vllm,transformers,diffusers,llama_cpp,custom}/` — one subdir per loader, each with an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. `modelship/infer/llama_server/llama_server_infer.py` is the exception: a single flat file (no `openai/` subpackage) — it proxies a `llama-server` subprocess's own OpenAI-compatible HTTP API rather than running modelship's parsers in-process. - `modelship/plugins/base_plugin.py` — `BasePlugin` ABC that each plugin package subclasses as `ModelPlugin`. - `plugins/*` — workspace packages, each opt-in via a matching root extra. The plugin module name and extra name **must match** (`ensure_plugin()` does `importlib.import_module(config.plugin)`). @@ -72,6 +72,7 @@ Under `tests/`, `pytest-asyncio` for async. Tests **mock out Ray Serve** — the - Metrics live on port **8079** (not 8000). `MSHIP_METRICS=false` or `--no-metrics` disables. When `mship_deploy` starts its own head (no `--use-existing-ray-cluster`), `connect_ray` pins that port via `ray.init(_metrics_export_port=…)` — a **private** Ray kwarg (accepted through `**kwargs`). A `TestConnectRay` test guards it so a Ray bump that drops it fails loudly. - `TRACE` is a custom log level below `DEBUG`; it logs full request/response payloads. - Docker CPU image uses the unified `Dockerfile` with `--build-arg MSHIP_VARIANT=cpu` and has a `:latest-cpu` tag suffix. +- `llama_server` loader launches a `llama-server` subprocess (found via `MSHIP_LLAMA_SERVER_BIN`, pinned in the Docker images at `/opt/llama.cpp`) and proxies its native OpenAI API — concurrency comes from `--parallel` slots instead of `llama_cpp`'s single `asyncio.Lock`. Same whole-GPU-only `num_gpus` rule as `llama_cpp`. Tool-call/reasoning parsing is llama-server's own, auto-detected per chat template: named-function `tool_choice` forcing is globally unsupported (silently falls back to `auto`), and `tool_choice: required` is grammar-enforced for harmony-style templates but a silent no-op for hermes-style ones (e.g. Qwen3); bare `response_format: {"type": "json_object"}` (no `schema` key) is also unenforced despite llama-server's own docs claiming support (verified against the b9859 binary directly) — `type: json_schema` requests, which is what modelship actually sends whenever a schema is given, are unaffected and correctly constrained. See `docs/model-configuration.md`'s llama_server section. No persistent on-disk prompt cache (llama-server has no equivalent to `llama_cpp`'s `cache: {type: disk}`). ## Further reading diff --git a/config/examples/README.md b/config/examples/README.md index 7b15ded..6634a2c 100644 --- a/config/examples/README.md +++ b/config/examples/README.md @@ -5,6 +5,7 @@ Ready-to-run `models.yaml` configs for common scenarios. Mount one into the cont | File | What it runs | Hardware | |---|---|---| | [llama-cpp.yaml](llama-cpp.yaml) | Quantized GGUF chat + embeddings | CPU (any arch) | +| [llama-server.yaml](llama-server.yaml) | Quantized GGUF chat (concurrent), vision, embeddings via a llama-server subprocess | CPU or NVIDIA GPU | | [transformers-cpu.yaml](transformers-cpu.yaml) | Llama 3.2 1B + Nomic embed + Whisper + MMS-TTS | CPU | | [vllm.yaml](vllm.yaml) | High-throughput chat with tool calling, embeddings, Whisper | NVIDIA GPU | | [diffusers.yaml](diffusers.yaml) | SDXL Turbo image generation | NVIDIA GPU | diff --git a/config/examples/llama-server.yaml b/config/examples/llama-server.yaml new file mode 100644 index 0000000..e66ec36 --- /dev/null +++ b/config/examples/llama-server.yaml @@ -0,0 +1,47 @@ +# llama_server loader examples — GGUF inference via a llama-server subprocess +# proxied over its native OpenAI-compatible HTTP API. Prefer this over +# llama_cpp for concurrent traffic: --parallel slots overlap requests instead +# of serializing behind a single lock. Requires MSHIP_LLAMA_SERVER_BIN to +# point at a llama-server executable (see docs/development.md); the Docker +# images ship a pinned build already. + +models: + # Chat: GGUF pulled from a HuggingFace repo, 4 concurrent request slots. + - name: "qwen" + model: "lmstudio-community/Qwen2.5-7B-Instruct-GGUF:*Q4_K_M.gguf" + usecase: "generate" + loader: "llama_server" + num_cpus: 3 + llama_server_config: + parallel: 4 + + # Chat: GGUF from a local path (bind-mount the directory into the container). + # Tool calling and reasoning parsing are llama-server's own, auto-detected + # from the chat template — no modelship-level parser config. See + # docs/model-configuration.md for the tool_choice gaps vs. llama_cpp. + - name: "llama3-local" + model: "/.cache/huggingface/llama-3-8b-instruct.Q4_K_M.gguf" + usecase: "generate" + loader: "llama_server" + num_cpus: 4 + llama_server_config: + n_ctx: 4096 + n_batch: 512 + parallel: 2 + + # Vision: mmproj enables image input; requests with image content are + # rejected at the gateway when it's unset. + - name: "llava" + model: "second-state/Llava-v1.5-7B-GGUF:llava-v1.5-7b-Q4_K_M.gguf" + usecase: "generate" + loader: "llama_server" + num_cpus: 3 + llama_server_config: + mmproj: "second-state/Llava-v1.5-7B-GGUF:llava-v1.5-7b-mmproj-model-f16.gguf" + + # Embeddings + - name: "nomic-embed" + model: "nomic-ai/nomic-embed-text-v1.5-GGUF:nomic-embed-text-v1.5.Q4_K_M.gguf" + usecase: "embed" + loader: "llama_server" + num_cpus: 2 diff --git a/docs/model-configuration.md b/docs/model-configuration.md index 02c7940..b3375f2 100644 --- a/docs/model-configuration.md +++ b/docs/model-configuration.md @@ -112,7 +112,7 @@ Selection is **all-or-nothing**: the highest tier whose *complete* stack fits is | `name` | string | Model identifier used in API requests | | `model` | string | HuggingFace repo ID, local path, or `repo:filename` (see [Model source](#model-source)). Required for built-in loaders; optional for `loader: custom` | | `usecase` | string | `generate`, `embed`, `transcription`, `translation`, `tts`, or `image` | -| `loader` | string | `vllm`, `transformers`, `diffusers`, `llama_cpp`, `stable_diffusion_cpp`, or `custom` | +| `loader` | string | `vllm`, `transformers`, `diffusers`, `llama_cpp`, `llama_server`, `stable_diffusion_cpp`, or `custom` | | `plugin` | string | Plugin module name (required when `loader: custom`); automatically loaded from wheels when referenced | | `num_gpus` | float \| int | GPU allocation. Fractional `< 1` shares one GPU (also sets vLLM `gpu_memory_utilization`); integer `≥ 1` requests that many whole GPUs (for `vllm`, this auto-sets `tensor_parallel_size = num_gpus` unless tp/pp is already specified). | | `num_cpus` | float | CPU units to allocate (default `0.1`) | @@ -123,6 +123,7 @@ Selection is **all-or-nothing**: the highest tier whose *complete* stack fits is | `transformers_config` | object | Transformers loader options (see below) | | `diffusers_config` | object | Diffusers pipeline options (see below) | | `llama_cpp_config` | object | llama.cpp loader options (see below) | +| `llama_server_config` | object | llama-server loader options (see below) | | `stable_diffusion_cpp_config` | object | stable-diffusion.cpp loader options (see below) | | `plugin_config` | object | Plugin-specific options passed through to the plugin | | `chat_template_kwargs` | object | Extra variables forwarded into the chat-template render on text loaders (`vllm`, `transformers`, `llama_cpp`) — e.g. `enable_thinking: false` for Qwen3. Only has an effect if the model's template branches on the key; ignored on paths that bypass the template (llama.cpp's native chat-handler fallback when `chat_format` is set or no template resolves). A per-request `chat_template_kwargs` overrides the model default on `vllm`. | @@ -425,8 +426,6 @@ models: capacity: 4GiB ``` -> **Note for the `llama_server` loader:** The `llama_server` loader does not support the explicit `cache` configuration block. It manages request-level prefix caching internally and automatically within its thread slots (analogous to standard in-memory caching), but does not support modelship's persistent on-disk prompt cache (`type: disk`) or explicit capacity configuration. - #### Tool calling (`tool_choice`) Tool-call behavior is driven entirely by the per-request OpenAI `tool_choice` parameter — there is no deploy-level flag. Sending a `tools` array is the opt-in. @@ -437,7 +436,7 @@ When a request carries `tools` and the model has a usable `tool_call_parser`, de - `required` / `{"type":"function","function":{"name":"X"}}` — the free-text branch is dropped, so the model **must** call a tool. Because that root cannot emit ``, it is enforced even on a reasoning-capable model. - `none` — tools are suppressed entirely. -Per-loader enforcement: **vLLM** honors `tool_choice` natively. **llama_cpp** enforces `required`/named via the grammar above (when a grammar-emitting parser is resolved — currently the JSON family `hermes` and the Gemma family; others are left unconstrained, logged once). **transformers** has no constrained decoding, so `required`/named are best-effort (tools are passed and the model is trusted). +Per-loader enforcement: **vLLM** honors `tool_choice` natively. **llama_cpp** enforces `required`/named via the grammar above (when a grammar-emitting parser is resolved — currently the JSON family `hermes` and the Gemma family; others are left unconstrained, logged once). **transformers** has no constrained decoding, so `required`/named are best-effort (tools are passed and the model is trusted). **llama_server** delegates entirely to llama-server's own grammar, which is per-model-family rather than a fixed modelship rule — see the [llama_server Loader](#llama_server-loader) section below for the specifics and gaps. Caveats: the grammar enforces *structure*, not *choice* (it can't pick the right value); JSON-schema numeric bounds aren't expressible as GBNF ranges (digit shape only); and the tool-call grammar takes precedence over a `response_format` grammar on the same request — the two cannot be combined. @@ -470,6 +469,68 @@ models: loader: llama_cpp ``` +## llama_server Loader + +The `llama_server` loader runs GGUF models by launching a [`llama-server`](https://github.com/ggml-org/llama.cpp) subprocess and proxying its native OpenAI-compatible HTTP API, instead of binding llama.cpp in-process like `llama_cpp` does. Chat templating, tool-call parsing, and reasoning parsing are all llama-server's own (`--jinja --reasoning-format auto`), not modelship's. This trades modelship's own parser coverage for llama-server's `--parallel` request slots, so concurrent requests actually overlap instead of serializing behind a single lock (as `llama_cpp` does) — prefer this loader over `llama_cpp` for any deployment expecting concurrent traffic. It requires the `llama-server` binary to be discoverable via `MSHIP_LLAMA_SERVER_BIN` (see [development.md](development.md#llama-server-binary-llama_server-loader)); the Docker images ship a pinned build at `/opt/llama.cpp`. + +Like `llama_cpp`, `num_gpus` must be `0` (CPU-only) or a whole integer number of GPUs — fractional is rejected at config time. + +| Field | Type | Default | Description | +|---|---|---|---| +| `n_ctx` | int | `2048` | Per-slot context length. The launch command multiplies this by `parallel` for llama-server's total `-c` (it splits one context budget across slots) | +| `n_batch` | int | `512` | Batch size for prompt processing | +| `n_gpu_layers` | int | `-1` | Layers to offload to GPU when `num_gpus >= 1` (`-1` auto-fits free VRAM, `<= -2` offloads all layers). Forced to `0` when `num_gpus` is `0` | +| `parallel` | int | `1` | Concurrent request slots (`--parallel`). When `max_ongoing_requests` isn't set explicitly, it defaults to this value so overflow queues in Ray Serve rather than inside llama-server | +| `chat_template` | string | — | Built-in template name (e.g. `chatml`) or a path to a Jinja file. Omit to use the GGUF's embedded chat template | +| `mmproj` | string | — | Multimodal projector file/repo ref (e.g. a CLIP model) for vision models — see [Vision](#vision-gguf) below | +| `extra_args` | list[string] | `[]` | Escape hatch: extra flags appended verbatim to the `llama-server` launch command | + +> **Note:** the `cache` block (`llama_cpp`'s persistent on-disk prompt cache) is **not supported**. llama-server manages request-level prefix caching internally and automatically within its slots (comparable to `llama_cpp`'s `type: ram` cache), but has no equivalent to modelship's restart-persistent `type: disk` cache. + +```yaml +models: + - name: "qwen-llama-server" + model: "lmstudio-community/Qwen2.5-7B-Instruct-GGUF:*Q4_K_M.gguf" + usecase: "generate" + loader: "llama_server" + llama_server_config: + parallel: 4 +``` + +### Tool calling and reasoning gaps vs. `llama_cpp` + +llama-server auto-detects both the tool-call and reasoning parser from the model's chat template — there is no modelship-level override. Two gaps are real and per-model-family, not per-loader, so test against the specific model in use before relying on either: + +- **Named-function forcing is unsupported.** `tool_choice: {"type": "function", "function": {"name": "X"}}` silently falls back to `auto` (llama-server logs a warning; modelship does not surface it as an error). +- **`tool_choice: required` enforcement depends on the model's chat template family.** It's grammar-enforced for harmony-style templates (e.g. gpt-oss) but a silent no-op for hermes-style templates (e.g. Qwen3) — the model may still answer in free text with no error. +- **Bare `response_format: {"type": "json_object"}` (no `schema` key) is not enforced**, despite llama-server's own docs describing it as supported "plain JSON output" — verified directly against the b9859 binary. The model can answer in free text with no error. This doesn't affect `type: json_schema` requests (which modelship sends whenever a schema is given, e.g. structured outputs) — those carry a `schema` and llama-server does constrain them correctly. + +Two wins over `llama_cpp`, also verified per-model-family: `response_format`/`json_schema` can be combined with reasoning in the same request (`llama_cpp` rejects that combination outright), and `logprobs`/`top_logprobs` are forwarded and returned (`llama_cpp` doesn't support either). + +### Vision (GGUF) + +Set `mmproj` to a multimodal projector file (local path or `repo:filename`, resolved the same way as `model:`) to enable image input; requests with `image_url`/`input_image` content parts are rejected at the gateway when `mmproj` isn't configured. + +```yaml +models: + - name: "llava-llama-server" + model: "second-state/Llava-v1.5-7B-GGUF:llava-v1.5-7b-Q4_K_M.gguf" + usecase: "generate" + loader: "llama_server" + llama_server_config: + mmproj: "second-state/Llava-v1.5-7B-GGUF:llava-v1.5-7b-mmproj-model-f16.gguf" +``` + +### Embeddings (GGUF) + +```yaml +models: + - name: nomic-embed-server + model: "nomic-ai/nomic-embed-text-v1.5-GGUF:nomic-embed-text-v1.5.Q4_K_M.gguf" + usecase: embed + loader: llama_server +``` + ## stable-diffusion.cpp Loader The `stable_diffusion_cpp` loader uses [stable-diffusion.cpp](https://github.com/leejet/stable-diffusion.cpp) (via [stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python)) for **CPU-only image generation** — the image counterpart to the `llama_cpp` text loader. It runs GGUF-quantized single-file diffusion checkpoints (SD1.5, SDXL, SD-Turbo, all-in-one Flux) in a few GB of RAM, with no GPU. Any `num_gpus` is ignored (a warning is logged and the actor is allocated `num_gpus: 0`). `usecase` is always `image` (defaulted if omitted) and it serves `/v1/images/generations`, `/v1/images/edits`, and `/v1/images/variations`. From a6573446f618dae00bd65c6d4b56bde46a42a6cf Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:20:34 +0000 Subject: [PATCH 20/35] test: add missing llama_server integration coverage Mirrors llama_cpp integration coverage the llama_server loader never got: tool-call markers quoted inside reasoning not double-counted, the response_format cluster (json_schema constrains unprompted output, non-stream + stream, and coexists with tool_choice=none), GPU offload (chat + tool calling), and real embeddings through a live binary (previously mock-only). Also adds tool_choice=required/named-function gap regression tests and a response_format+reasoning free-win test called out in the roadmap but never written. Found along the way: bare response_format={"type": "json_object"} (no schema key) is not enforced by llama-server despite its own docs claiming support, verified directly against the b9859 binary. Documented as a gap test rather than asserted as parity. --- tests/test_integration.py | 336 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 336 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index ce7e40f..37d8fdf 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -150,6 +150,39 @@ "parallel": 4, }, }, + "chat-llama-server-plain": { + "name": "chat-llama-server-plain", + # Same non-reasoning Qwen2.5-0.5B GGUF as `chat-llama-mship`, through + # the llama_server loader instead of llama_cpp. Used for the + # response_format tests mirrored from `TestChatLlamaCpp`, which need + # a model that doesn't emit a `...` preamble. + "model": "lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF:*Q4_K_M.gguf", + "usecase": "generate", + "loader": "llama_server", + "num_cpus": 1, + }, + "chat-llama-server-gpu": { + "name": "chat-llama-server-gpu", + # Same GGUF as chat-llama-server-plain, on a whole GPU — mirrors + # `chat-llama-gpu`'s coverage of the offload path (actor GPU + # allocation, -ngl honored) for the llama_server loader. + "model": "lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF:*Q4_K_M.gguf", + "usecase": "generate", + "loader": "llama_server", + "num_gpus": 1, + "num_cpus": 1, + }, + "embed-model-llama-server": { + "name": "embed-model-llama-server", + # Real embeddings through a live llama-server subprocess (`--embedding`) + # — the existing `test_embeddings` integration test only exercises the + # transformers loader; llama_server's B4 embeddings support was + # otherwise only unit-tested against a mocked httpx transport. + "model": "nomic-ai/nomic-embed-text-v1.5-GGUF:nomic-embed-text-v1.5.Q4_K_M.gguf", + "usecase": "embed", + "loader": "llama_server", + "num_cpus": 1, + }, "chat-transformers": { "name": "chat-transformers", "model": "Qwen/Qwen2.5-0.5B-Instruct", @@ -1449,6 +1482,132 @@ def test_reasoning_with_tools_llama_server(self, client): assert "Paris" in tool_calls[0].function.arguments assert completion.choices[0].finish_reason == "tool_calls" + def test_tool_markers_inside_reasoning_not_double_counted_llama_server(self, client): + """Mirrors `TestChatLlamaCppReasoning.test_tool_markers_inside_reasoning_not_double_counted`, + but for llama-server's own parser instead of modelship's + `ChatOutputStreamer` — the bug pattern (a `...` + illustration quoted inside `` reasoning getting parsed as a + second, real call) is plausible for any single-pass parser, not just + modelship's. + + Coaxes the model into illustrating tool-call syntax inside its + reasoning before making one actual call, and asserts exactly one real + `tool_calls` entry comes out. Real models are non-deterministic; if + the prompt fails to produce literal markers in reasoning, the + marker-routing assertion is skipped rather than flaking — the + single-tool-call assertion still has value either way. + """ + completion = client.chat.completions.create( + model="chat-llama-server", + messages=[ + { + "role": "system", + "content": ( + "You are an assistant with access to tools. When you think inside " + "..., FIRST quote one example of tool-call syntax " + "verbatim inside angle brackets — e.g. write the literal text " + '{"name":"example","arguments":{}} as part ' + "of your reasoning to remind yourself of the format. THEN decide " + "which real tool to call." + ), + }, + {"role": "user", "content": "What is the weather in Paris?"}, + ], + tools=[_WEATHER_TOOL], + tool_choice="auto", + max_tokens=1024, + ) + message = completion.choices[0].message + reasoning = getattr(message, "reasoning", None) or message.model_extra.get("reasoning") or "" + tool_calls = message.tool_calls or [] + + assert tool_calls, ( + f"expected exactly one real tool call, got content={message.content!r}, reasoning={reasoning!r}" + ) + assert len(tool_calls) == 1, ( + f"expected exactly one tool call (markers inside must not be double-counted); " + f"got {len(tool_calls)} calls={[tc.function.name for tc in tool_calls]}" + ) + assert tool_calls[0].function.name == "get_weather" + assert completion.choices[0].finish_reason == "tool_calls" + + if "" in reasoning: + assert "" in reasoning, ( + f"reasoning has an unmatched marker (open without close): {reasoning!r}" + ) + + def test_response_format_with_reasoning_llama_server(self, client): + """Free win vs. `llama_cpp` (which rejects this combo outright — see + `TestChatLlamaCppReasoning.test_response_format_with_reasoning_deployment_rejected`): + llama-server handles a JSON-schema `response_format` combined with + reasoning natively, routing `...` to `message.reasoning` + and the schema-conforming JSON to `message.content`.""" + completion = client.chat.completions.create( + model="chat-llama-server", + messages=[{"role": "user", "content": "What is 2+2?"}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + }, + "strict": True, + }, + }, + max_tokens=1024, + ) + message = completion.choices[0].message + reasoning = getattr(message, "reasoning", None) or message.model_extra.get("reasoning") + assert reasoning, f"expected reasoning, got message={message!r}" + assert message.content + parsed = json.loads(message.content) + assert "answer" in parsed + + def test_tool_choice_required_is_a_noop_on_hermes_family(self, client): + """Documents a real gap (A1 spike finding): `tool_choice: required` is + grammar-enforced on harmony-style chat templates (e.g. gpt-oss) but a + silent no-op on hermes-style ones — Qwen3 (this deployment) included. + Real grammar forcing makes the free-text branch structurally + unreachable (`message.content` would be empty/None); on this + hermes-style model it stays reachable even under `required`, proving + no grammar constraint was applied. (Verified against a live run: this + 0.6B model is unstable enough to *also* emit a spurious tool call + alongside genuine free text on an irrelevant prompt — which is a + model-quality quirk, not evidence of forcing, so this asserts on + content reachability rather than tool_calls absence.) If llama.cpp + starts enforcing this for hermes models, `content` goes empty and + this test fails — update the docs/CLAUDE.md gap notes. + """ + completion = client.chat.completions.create( + model="chat-llama-server", + messages=[{"role": "user", "content": "Say hello in one word."}], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=256, + ) + message = completion.choices[0].message + assert message.content, f"expected the free-text branch to stay reachable, got message={message!r}" + + def test_named_function_tool_choice_falls_back_to_auto(self, client): + """Documents a real gap (A1 spike finding): object-form `tool_choice` + (named-function forcing) is globally unsupported in llama.cpp b9859 — + it silently falls back to `auto` rather than forcing the named + function or erroring. Same content-reachability technique and + irrelevant-tool prompt as the `required` gap test above — real + forcing would make the free-text branch structurally unreachable.""" + completion = client.chat.completions.create( + model="chat-llama-server", + messages=[{"role": "user", "content": "Say hello in one word."}], + tools=[_WEATHER_TOOL], + tool_choice={"type": "function", "function": {"name": "get_weather"}}, + max_tokens=256, + ) + message = completion.choices[0].message + assert message.content, f"expected the free-text branch to stay reachable, got message={message!r}" + def test_concurrent_requests_are_not_serialized(self, client): """The loader's headline win over llama_cpp: llama-server's `--parallel` slots let several requests run concurrently instead of @@ -1524,6 +1683,183 @@ def test_streaming_response_through_llama_server(self, client): assert completed.status in {"completed", "incomplete"} +@pytest.mark.integration +@pytest.mark.llama_server +class TestChatLlamaServerResponseFormat: + """Mirrors `TestChatLlamaCpp`'s response_format cluster for the + llama_server loader. Uses `chat-llama-server-plain` (non-reasoning + Qwen2.5-0.5B) rather than `chat-llama-server` (Qwen3, always emits a + `...` preamble) so these tests read the same as their llama_cpp + counterparts — response_format + reasoning together is covered + separately by `TestChatLlamaServer.test_response_format_with_reasoning_llama_server`. + """ + + @pytest.fixture(autouse=True, scope="class") + def _deploy(self, model_deployer): + model_deployer.deploy("chat-llama-server-plain") + + def test_response_format_json_object_without_schema_is_unconstrained(self, client): + """Real gap vs. `llama_cpp`, discovered while mirroring this test: llama-server's + own docs claim bare `{"type": "json_object"}` (no `schema` key) produces + "plain JSON output" like other OpenAI-inspired providers, but verified + directly against the b9859 binary (`curl` straight to `/v1/chat/completions`, + bypassing modelship) this isn't enforced — the model answers in free + text with no error. Constraining does work once a `schema` key is + attached to the `response_format` object (an llama-server extension, + not in the OpenAI spec — `type: json_schema`, which modelship's + protocol sends for schema-constrained requests, does carry a schema + and IS honored — see `test_response_format_json_schema_constrains_unprompted_output`). + If this test starts failing (content parses as JSON), llama-server + started honoring plain `json_object` — update this note and + CLAUDE.md/AGENTS.md's llama_server gap list. + """ + completion = client.chat.completions.create( + model="chat-llama-server-plain", + messages=[{"role": "user", "content": "What is the capital of France?"}], + response_format={"type": "json_object"}, + max_tokens=64, + ) + content = completion.choices[0].message.content + assert content + with pytest.raises(json.JSONDecodeError): + json.loads(content) + + def test_response_format_json_schema_constrains_unprompted_output(self, client): + """A natural-language question + json_schema → schema-conformant output.""" + schema = { + "type": "object", + "properties": { + "city": {"type": "string"}, + "country": {"type": "string"}, + }, + "required": ["city", "country"], + "additionalProperties": False, + } + completion = client.chat.completions.create( + model="chat-llama-server-plain", + messages=[{"role": "user", "content": "Where is the Eiffel Tower located?"}], + response_format={ + "type": "json_schema", + "json_schema": {"name": "location", "schema": schema, "strict": True}, + }, + max_tokens=64, + ) + content = completion.choices[0].message.content + assert content + parsed = json.loads(content) + assert set(parsed.keys()) == {"city", "country"} + assert isinstance(parsed["city"], str) and parsed["city"] + assert isinstance(parsed["country"], str) and parsed["country"] + + def test_response_format_json_schema_streaming_constrains_unprompted_output(self, client): + """Same intent on the streaming path.""" + schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + "additionalProperties": False, + } + stream = client.chat.completions.create( + model="chat-llama-server-plain", + messages=[{"role": "user", "content": "What is the capital of France?"}], + response_format={ + "type": "json_schema", + "json_schema": {"name": "answer", "schema": schema, "strict": True}, + }, + max_tokens=64, + stream=True, + ) + chunks = [] + for chunk in stream: + if chunk.choices and chunk.choices[0].delta.content: + chunks.append(chunk.choices[0].delta.content) + content = "".join(chunks) + assert content + parsed = json.loads(content) + assert set(parsed.keys()) == {"answer"} + assert isinstance(parsed["answer"], str) and parsed["answer"] + + def test_response_format_coexists_with_tool_choice_none(self, client): + """tool_choice='none' is the safe escape valve: tools listed but inert, + schema enforced on content output. + """ + schema = { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + "additionalProperties": False, + } + completion = client.chat.completions.create( + model="chat-llama-server-plain", + messages=[{"role": "user", "content": "Where is the Eiffel Tower located?"}], + tools=[_WEATHER_TOOL], + tool_choice="none", + response_format={ + "type": "json_schema", + "json_schema": {"name": "location", "schema": schema, "strict": True}, + }, + max_tokens=64, + ) + assert not completion.choices[0].message.tool_calls + content = completion.choices[0].message.content + assert content + parsed = json.loads(content) + assert set(parsed.keys()) == {"city", "country"} + + +@pytest.mark.integration +@pytest.mark.llama_server +class TestChatLlamaServerGpu: + """End-to-end GPU offload through the llama_server loader. + + Same GGUF and tool-calling shape as `TestChatLlamaServerResponseFormat` + (CPU), but deployed with `num_gpus=1` so the actor gets a whole GPU and + the loader passes `-ngl` for real offload instead of the forced `-ngl 0` + it uses when `num_gpus` is `0`. Mirrors `TestChatLlamaCppGpu`. + """ + + @pytest.fixture(autouse=True, scope="class") + def _deploy(self, model_deployer): + model_deployer.deploy("chat-llama-server-gpu") + + def test_chat_completion(self, client): + completion = client.chat.completions.create( + model="chat-llama-server-gpu", + messages=[{"role": "user", "content": "What is the capital of France?"}], + max_tokens=32, + ) + content = completion.choices[0].message.content + assert content + assert "Paris" in content + + def test_tool_calling_llama_server_gpu_loader(self, client): + completion = client.chat.completions.create( + model="chat-llama-server-gpu", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + tool_choice="auto", + max_tokens=128, + ) + tool_calls = completion.choices[0].message.tool_calls + assert tool_calls, f"expected a tool call, got content={completion.choices[0].message.content!r}" + assert tool_calls[0].function.name == "get_weather" + assert "Paris" in tool_calls[0].function.arguments + assert completion.choices[0].finish_reason == "tool_calls" + + +@pytest.mark.integration +@pytest.mark.llama_server +def test_embeddings_llama_server(client, model_deployer): + """Real embeddings through a live llama-server subprocess (`--embedding`). + `test_embeddings` only exercises the transformers loader; this is the + first live-binary coverage of llama_server's B4 embeddings support + (previously only unit-tested against a mocked httpx transport).""" + model_deployer.deploy("embed-model-llama-server") + response = client.embeddings.create(model="embed-model-llama-server", input=["Hello world", "Modelship is great"]) + assert len(response.data) == 2 + assert len(response.data[0].embedding) > 0 + + @pytest.mark.integration class TestChatTransformersLlama3Json: """End-to-end llama3_json tool calling through the transformers loader. From 188179eebf5bdaaa288a7ddb77382f78346bf4ce Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:51:15 +0000 Subject: [PATCH 21/35] feat: quarantine vLLM-internal touchpoints behind engine_ops Stage C1 of the parser-migration roadmap: build_vllm_request, render_and_params, extract_prompt_token_ids, make_parsers, derive_reasoning_ended, and generate wrap every vllm.entrypoints/ vllm.parser/vllm.v1.engine call the vllm loader needs, so a future vLLM bump's blast radius stays confined to one file. vllm_infer.py is untouched (still runs OpenAIServingChat) -- C3/C4 repoint it onto these functions. --- modelship/infer/vllm/engine_ops.py | 161 ++++++++++++++++ tests/test_vllm_engine_ops.py | 288 +++++++++++++++++++++++++++++ 2 files changed, 449 insertions(+) create mode 100644 modelship/infer/vllm/engine_ops.py create mode 100644 tests/test_vllm_engine_ops.py diff --git a/modelship/infer/vllm/engine_ops.py b/modelship/infer/vllm/engine_ops.py new file mode 100644 index 0000000..7f9c162 --- /dev/null +++ b/modelship/infer/vllm/engine_ops.py @@ -0,0 +1,161 @@ +"""Quarantine for every vLLM-internal touchpoint the vllm loader needs. + +Modelship types go in; a parsed 3-tuple (via vllm.parser) or a raw engine +stream comes out. Nothing outside this module should import from +`vllm.entrypoints.*`/`vllm.parser`/`vllm.v1.engine.*` directly — that keeps a +vLLM version bump's blast radius confined to one file. +""" + +from collections.abc import AsyncGenerator, Mapping +from typing import Any + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest as VllmChatCompletionRequest, +) +from vllm.entrypoints.openai.engine.protocol import ErrorResponse as VllmErrorResponse +from vllm.entrypoints.serve.render.serving import OpenAIServingRender +from vllm.entrypoints.serve.utils.api_utils import get_max_tokens +from vllm.inputs import EngineInput +from vllm.outputs import RequestOutput +from vllm.parser import Parser +from vllm.renderers.inputs.preprocess import extract_prompt_components, extract_prompt_len +from vllm.sampling_params import SamplingParams +from vllm.tokenizers import TokenizerLike +from vllm.v1.engine.async_llm import AsyncLLM + +from modelship.openai.protocol import ChatCompletionRequest + + +def build_vllm_request( + request: ChatCompletionRequest, + chat_template_kwargs: dict[str, Any] | None, +) -> VllmChatCompletionRequest: + """Shape a modelship chat request into vLLM's own request model. + + Merges the model's default `chat_template_kwargs` under any per-request + value (request wins) — vLLM renders the chat template internally, so + unlike llama.cpp-family loaders this can't be patched in after the fact. + """ + request_data = request.model_dump() + if chat_template_kwargs: + request_data["chat_template_kwargs"] = { + **chat_template_kwargs, + **(request_data.get("chat_template_kwargs") or {}), + } + return VllmChatCompletionRequest(**request_data) + + +async def render_and_params( + render: OpenAIServingRender, + vllm_req: VllmChatCompletionRequest, +) -> tuple[EngineInput, SamplingParams] | VllmErrorResponse: + """Render the chat template and derive `SamplingParams`, in that order. + + `render_chat` mutates `vllm_req` in place as a side effect of rendering + (`ToolParser.adjust_request` sets `structured_outputs` / + `_grammar_from_tool_parser`), and `to_sampling_params` reads that + mutation. The order is load-bearing — this function exists specifically + so callers can't split the two apart or run them against a rebuilt copy + of the request, which would silently drop the mutation. + """ + result = await render.render_chat(vllm_req) + if isinstance(result, VllmErrorResponse): + return result + _conversation, engine_inputs = result + if len(engine_inputs) != 1: + raise RuntimeError(f"expected exactly 1 rendered engine prompt for a chat request, got {len(engine_inputs)}") + engine_input = engine_inputs[0] + + max_tokens = get_max_tokens( + render.model_config.max_model_len, + vllm_req.max_completion_tokens if vllm_req.max_completion_tokens is not None else vllm_req.max_tokens, + extract_prompt_len(render.model_config, engine_input), + render.default_sampling_params, + render.override_max_tokens, + truncate_prompt_tokens=vllm_req.truncate_prompt_tokens, + ) + sampling_params = vllm_req.to_sampling_params(max_tokens, render.default_sampling_params) + return engine_input, sampling_params + + +def extract_prompt_token_ids(render: OpenAIServingRender, engine_input: EngineInput) -> list[int]: + """Extract the rendered prompt's token IDs, needed for `derive_reasoning_ended`.""" + return list(extract_prompt_components(render.model_config, engine_input).token_ids or []) + + +def make_parsers( + render: OpenAIServingRender, + tokenizer: TokenizerLike, + vllm_req: VllmChatCompletionRequest, + chat_template_kwargs: dict[str, Any] | None, + n: int, +) -> list[Parser | None]: + """Instantiate one parser per choice. + + Parsers carry per-choice streaming state (`Parser._stream_state`), so a + request with `n > 1` needs `n` independent instances — sharing one across + choices corrupts state on every choice after the first. `render.parser` + is the same class `render_chat` already resolved internally via + `ParserManager.get_parser`, so this can't drift out of sync with it. + """ + if render.parser is None: + return [None] * n + parser_cls = render.parser + return [ + parser_cls(tokenizer, vllm_req.tools, chat_template_kwargs=chat_template_kwargs) # type: ignore[arg-type] + for _ in range(n) + ] + + +def derive_reasoning_ended( + vllm_req: VllmChatCompletionRequest, + parser: Parser | None, + prompt_token_ids: list[int], +) -> bool | None: + """Replicates the reasoning_ended precedence in vLLM's own chat completion serving. + + Mistral's grammar (when built) already encodes an optional `think?` rule + covering both reasoning and non-reasoning outputs, so `reasoning_ended` + is forced True whenever `_grammar_from_tool_parser` is set. But that flag + is only set on the grammar-building branch of + `MistralToolParser.adjust_request` — a request with tools but no + structured-outputs constraint active takes an early-return branch that + leaves it False, so this must not assume the flag is reliably True + whenever a mistral tool parser is in play. + """ + if not vllm_req.include_reasoning: + return True + if vllm_req._grammar_from_tool_parser: + return True + if parser is not None and parser.reasoning_parser is not None: + return parser.is_reasoning_end(prompt_token_ids) + return None + + +def generate( + engine: AsyncLLM, + engine_input: EngineInput, + sampling_params: SamplingParams, + request_id: str, + *, + reasoning_ended: bool | None, + parser: Parser | None, + chat_template_kwargs: dict[str, Any] | None, + trace_headers: Mapping[str, str] | None = None, + priority: int = 0, + data_parallel_rank: int | None = None, +) -> AsyncGenerator[RequestOutput, None]: + """Thin wrapper over `AsyncLLM.generate` — the only place this loader touches the engine directly.""" + reasoning_parser_kwargs = None + if parser is not None and parser.reasoning_parser is not None: + reasoning_parser_kwargs = {"chat_template_kwargs": chat_template_kwargs} + return engine.generate( + engine_input, + sampling_params, + request_id, + trace_headers=trace_headers, + priority=priority, + data_parallel_rank=data_parallel_rank, + reasoning_ended=reasoning_ended, + reasoning_parser_kwargs=reasoning_parser_kwargs, + ) diff --git a/tests/test_vllm_engine_ops.py b/tests/test_vllm_engine_ops.py new file mode 100644 index 0000000..9fa4276 --- /dev/null +++ b/tests/test_vllm_engine_ops.py @@ -0,0 +1,288 @@ +"""Guard tests for modelship/infer/vllm/engine_ops.py, the vLLM-internal quarantine layer. + +Two tiers: +- Fast, always-run unit tests for the pure branching logic (`build_vllm_request`, + `derive_reasoning_ended`, `make_parsers`) using real vLLM request/parser *types* + but no engine, no tokenizer download, no GPU. +- `TestVllmParserAcceptsOurRequest`: real (non-mocked) vLLM render pipeline against + real cached tokenizers (hermes+qwen3 and mistral tool parsers — the two families + the A2 spike validated), built via `renderer_from_config` the same way vLLM's own + GPU-less render server does. No engine/weights load, so no GPU is needed, but it + does need the tokenizer files reachable (skips cleanly if not). This exercises the + actual vLLM call signatures engine_ops depends on, so a vLLM version bump that + renames/removes any of them fails here loudly instead of surfacing downstream. +""" + +import inspect +from typing import Any +from unittest.mock import Mock + +import pytest +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest as VllmChatCompletionRequest, +) +from vllm.entrypoints.serve.render.serving import OpenAIServingRender +from vllm.parser import Parser +from vllm.tokenizers import TokenizerLike + +from modelship.infer.vllm import engine_ops +from modelship.openai.protocol import ChatCompletionRequest + + +def _vllm_req(**overrides: Any) -> VllmChatCompletionRequest: + base: dict[str, Any] = {"model": "m", "messages": [{"role": "user", "content": "hi"}]} + base.update(overrides) + return VllmChatCompletionRequest(**base) + + +class TestBuildVllmRequest: + def test_request_chat_template_kwargs_wins_over_model_default(self): + # chat_template_kwargs isn't a declared ChatCompletionRequest field — it + # arrives as a client-supplied extra (model_config extra="allow") — so + # build it via model_validate rather than the constructor kwargs. + request = ChatCompletionRequest.model_validate( + { + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "chat_template_kwargs": {"enable_thinking": False}, + } + ) + vllm_req = engine_ops.build_vllm_request(request, chat_template_kwargs={"enable_thinking": True, "x": 1}) + assert vllm_req.chat_template_kwargs == {"enable_thinking": False, "x": 1} + + def test_no_model_defaults_leaves_request_value_untouched(self): + request = ChatCompletionRequest.model_validate( + { + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "chat_template_kwargs": {"a": 1}, + } + ) + vllm_req = engine_ops.build_vllm_request(request, chat_template_kwargs=None) + assert vllm_req.chat_template_kwargs == {"a": 1} + + +class TestDeriveReasoningEnded: + def test_include_reasoning_false_forces_true(self): + vllm_req = _vllm_req(include_reasoning=False) + assert engine_ops.derive_reasoning_ended(vllm_req, parser=None, prompt_token_ids=[]) is True + + def test_grammar_from_tool_parser_forces_true(self): + vllm_req = _vllm_req() + vllm_req._grammar_from_tool_parser = True + assert engine_ops.derive_reasoning_ended(vllm_req, parser=None, prompt_token_ids=[]) is True + + def test_reasoning_parser_present_defers_to_is_reasoning_end(self): + vllm_req = _vllm_req() + parser = Mock(spec=Parser) + parser.reasoning_parser = Mock() + parser.is_reasoning_end.return_value = True + result = engine_ops.derive_reasoning_ended(vllm_req, parser=parser, prompt_token_ids=[1, 2, 3]) + assert result is True + parser.is_reasoning_end.assert_called_once_with([1, 2, 3]) + + def test_no_reasoning_parser_and_no_grammar_flag_is_none(self): + vllm_req = _vllm_req() + parser = Mock(spec=Parser) + parser.reasoning_parser = None + assert engine_ops.derive_reasoning_ended(vllm_req, parser=parser, prompt_token_ids=[]) is None + + def test_no_parser_at_all_is_none(self): + vllm_req = _vllm_req() + assert engine_ops.derive_reasoning_ended(vllm_req, parser=None, prompt_token_ids=[]) is None + + +class TestMakeParsers: + def test_no_parser_class_returns_n_nones(self): + render = Mock(spec=OpenAIServingRender) + render.parser = None + result = engine_ops.make_parsers(render, tokenizer=Mock(), vllm_req=_vllm_req(), chat_template_kwargs=None, n=3) + assert result == [None, None, None] + + def test_instantiates_one_independent_parser_per_choice(self): + render = Mock(spec=OpenAIServingRender) + parser_cls = Mock() + instances = [Mock(), Mock(), Mock()] + parser_cls.side_effect = instances + render.parser = parser_cls + vllm_req = _vllm_req(tools=None) + tokenizer = Mock() + + result = engine_ops.make_parsers(render, tokenizer, vllm_req, chat_template_kwargs={"k": "v"}, n=3) + + assert result == instances + assert parser_cls.call_count == 3 + for call in parser_cls.call_args_list: + args, kwargs = call + assert args == (tokenizer, vllm_req.tools) + assert kwargs == {"chat_template_kwargs": {"k": "v"}} + + +class TestSignaturesGuardVllmBump: + """No engine/tokenizer needed — pure import-time signature checks. + + These fail immediately (not silently) if a vLLM version bump renames or + drops a kwarg engine_ops relies on, instead of only surfacing as a runtime + AttributeError/TypeError deep in a real request path. + """ + + def test_async_llm_generate_accepts_reasoning_kwargs(self): + from vllm.v1.engine.async_llm import AsyncLLM + + params = inspect.signature(AsyncLLM.generate).parameters + assert "reasoning_ended" in params + assert "reasoning_parser_kwargs" in params + + def test_chat_completion_request_has_grammar_from_tool_parser_private_attr(self): + vllm_req = _vllm_req() + assert hasattr(vllm_req, "_grammar_from_tool_parser") + assert vllm_req._grammar_from_tool_parser is False + + def test_to_sampling_params_signature_unchanged(self): + params = inspect.signature(VllmChatCompletionRequest.to_sampling_params).parameters + assert list(params)[1:] == ["max_tokens", "default_sampling_params"] + + def test_render_chat_signature_unchanged(self): + params = inspect.signature(OpenAIServingRender.render_chat).parameters + assert "request" in params + + +class TestVllmParserAcceptsOurRequest: + """Real vLLM render pipeline, real cached tokenizers, no engine/GPU. + + Mirrors the A2 spike's two model families. Built via `renderer_from_config` + the same way vLLM's own GPU-less render server does (`init_render_app_state` + in vllm/entrypoints/openai/api_server.py) — this needs the tokenizer files, + not weights, so it stays fast and GPU-free. Skips cleanly if the tokenizer + can't be fetched (no network / no HF auth for gated repos). + """ + + def _build_render( + self, model: str, *, tokenizer_mode: str = "auto", **tool_reasoning_kwargs: Any + ) -> tuple[OpenAIServingRender, TokenizerLike]: + from vllm.engine.arg_utils import AsyncEngineArgs + from vllm.entrypoints.openai.models.protocol import BaseModelPath + from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry + from vllm.entrypoints.serve.utils.request_logger import RequestLogger + from vllm.renderers import renderer_from_config + from vllm.usage.usage_lib import UsageContext + + try: + engine_args = AsyncEngineArgs( + model=model, tokenizer_mode=tokenizer_mode, max_model_len=4096, enforce_eager=True + ) + vllm_config = engine_args.create_engine_config(usage_context=UsageContext.OPENAI_API_SERVER) + renderer = renderer_from_config(vllm_config) + except Exception as e: + pytest.skip(f"could not build a GPU-free render pipeline for {model!r}: {e}") + + registry = OpenAIModelRegistry( + model_config=vllm_config.model_config, + base_model_paths=[BaseModelPath(name="test-model", model_path=model)], + ) + render = OpenAIServingRender( + model_config=vllm_config.model_config, + renderer=renderer, + model_registry=registry, + request_logger=RequestLogger(max_log_len=None), + chat_template=None, + chat_template_content_format="auto", + enable_auto_tools=True, + **tool_reasoning_kwargs, + ) + assert renderer.tokenizer is not None + return render, renderer.tokenizer + + def _tool_request(self, **overrides: Any) -> VllmChatCompletionRequest: + base: dict[str, Any] = dict( + model="test-model", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + tool_choice="auto", + max_tokens=64, + ) + base.update(overrides) + return VllmChatCompletionRequest(**base) + + @pytest.mark.asyncio + async def test_hermes_qwen3_reasoning_and_tools(self): + render, tokenizer = self._build_render("Qwen/Qwen3-0.6B", tool_parser="hermes", reasoning_parser="qwen3") + vllm_req = self._tool_request() + + result = await engine_ops.render_and_params(render, vllm_req) + assert not isinstance(result, engine_ops.VllmErrorResponse) + engine_input, sampling_params = result + assert sampling_params.max_tokens == 64 + + prompt_ids = engine_ops.extract_prompt_token_ids(render, engine_input) + assert prompt_ids + + n = 2 + parsers = engine_ops.make_parsers(render, tokenizer, vllm_req, chat_template_kwargs=None, n=n) + assert len(parsers) == n + assert all(p is not None for p in parsers) + + reasoning_ended = engine_ops.derive_reasoning_ended(vllm_req, parsers[0], prompt_ids) + assert reasoning_ended is False # prompt has no prior reasoning to have already ended + + fake_output = ( + "I should check the weather.\n" + '\n{"name": "get_weather", "arguments": {"location": "Paris"}}\n' + ) + for parser in parsers: + assert parser is not None + reasoning, content, tool_calls = parser.parse( + fake_output, vllm_req, enable_auto_tools=True, model_output_token_ids=[] + ) + assert reasoning == "I should check the weather." + assert content is None + assert tool_calls is not None and len(tool_calls) == 1 + assert tool_calls[0].name == "get_weather" + + @pytest.mark.asyncio + async def test_mistral_tool_only_no_grammar_flag(self): + # A2's confirmed correction: a tool-only request (no structured-outputs + # constraint active) takes MistralToolParser.adjust_request's early-return + # branch and never sets _grammar_from_tool_parser, so reasoning_ended must + # NOT assume the flag is reliably True whenever a mistral tool parser is + # in play. + render, tokenizer = self._build_render( + "mistralai/Mistral-7B-Instruct-v0.3", tokenizer_mode="mistral", tool_parser="mistral" + ) + vllm_req = self._tool_request() + + result = await engine_ops.render_and_params(render, vllm_req) + assert not isinstance(result, engine_ops.VllmErrorResponse) + engine_input, _sampling_params = result + + assert vllm_req._grammar_from_tool_parser is False + + prompt_ids = engine_ops.extract_prompt_token_ids(render, engine_input) + parsers = engine_ops.make_parsers(render, tokenizer, vllm_req, chat_template_kwargs=None, n=1) + assert len(parsers) == 1 + parser = parsers[0] + assert parser is not None + + reasoning_ended = engine_ops.derive_reasoning_ended(vllm_req, parser, prompt_ids) + assert reasoning_ended is None # no reasoning parser configured for this model + + # No AttributeError on a plain-text (non-tool-call) output. + reasoning, content, tool_calls = parser.parse( + "Just a plain text response.", vllm_req, enable_auto_tools=True, model_output_token_ids=[] + ) + assert reasoning is None + assert content == "Just a plain text response." + assert tool_calls in (None, []) From 9c545b0dfa04da227c2b88f33e32fb82fbbddb05 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:55:27 +0000 Subject: [PATCH 22/35] refactor: repoint llama_cpp non-stream chat onto build_from_parsed Stage C2 of the parser-migration roadmap. llama_cpp's non-stream tool/reasoning path now parses via the shared ChatOutputStreamer (parse_chat_completion_text, promoted from a private helper) and builds its response through chat_utils.build_from_parsed instead of parsers.streaming.build_chat_completion_response -- proving the builder works against a second real parsed-output source (llama_server via B3 was the first). Streaming and the transformers loader are untouched. --- .../infer/llama_cpp/openai/serving_chat.py | 34 ++++++++++++++----- modelship/openai/parsers/streaming.py | 11 ++++-- tests/test_tool_calling.py | 12 +++---- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/modelship/infer/llama_cpp/openai/serving_chat.py b/modelship/infer/llama_cpp/openai/serving_chat.py index d9104e1..9b56e4c 100644 --- a/modelship/infer/llama_cpp/openai/serving_chat.py +++ b/modelship/infer/llama_cpp/openai/serving_chat.py @@ -14,14 +14,20 @@ from modelship.infer.llama_cpp.tool_grammar import build_tool_call_grammar from modelship.infer.llama_cpp.utils import LlamaCppToolCallRenderer from modelship.logging import TRACE, get_logger -from modelship.openai.chat_utils import UnsupportedContentError, normalize_chat_messages +from modelship.openai.chat_utils import ( + ParsedChatOutput, + UnsupportedContentError, + build_from_parsed, + normalize_chat_messages, +) from modelship.openai.parsers.reasoning import get_parser as get_reasoning_parser -from modelship.openai.parsers.streaming import build_chat_completion_response, stream_chat_completion +from modelship.openai.parsers.streaming import finish_reason_for, parse_chat_completion_text, stream_chat_completion from modelship.openai.parsers.tool_calling import get_parser, request_forces_tool_call, resolve_tools_for_request from modelship.openai.protocol import ( ChatCompletionRequest, ChatCompletionResponse, ErrorResponse, + UsageInfo, create_error_response, ) from modelship.utils import base_request_id @@ -284,15 +290,25 @@ async def _handle_with_parsers( usage = result.get("usage") or {} completion_tokens = int(usage.get("completion_tokens") or renderer.count_tokens(completion_text)) - return build_chat_completion_response( - request_id=request_id, - model_name=self.model_name, - text=completion_text, + parsed = parse_chat_completion_text( + completion_text, parser_name=tool_parser_name, reasoning_parser_name=reasoning_parser_name, - prompt_tokens=int(usage.get("prompt_tokens") or prompt_tokens), - completion_tokens=completion_tokens, - max_tokens=max_tokens, + ) + finish_reason = finish_reason_for(parsed, completion_tokens, max_tokens) + prompt_tokens_used = int(usage.get("prompt_tokens") or prompt_tokens) + return build_from_parsed( + request_id=request_id, + model_name=self.model_name, + choices=[ + ParsedChatOutput(content=parsed.content, reasoning=parsed.reasoning, tool_calls=parsed.tool_calls) + ], + usage=UsageInfo( + prompt_tokens=prompt_tokens_used, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens_used + completion_tokens, + ), + finish_reasons=[finish_reason], created=int(time.time()), ) diff --git a/modelship/openai/parsers/streaming.py b/modelship/openai/parsers/streaming.py index 86cbcc3..09fc723 100644 --- a/modelship/openai/parsers/streaming.py +++ b/modelship/openai/parsers/streaming.py @@ -96,7 +96,7 @@ def build_chat_completion_response( can have unwanted special tokens (``<|eot_id|>``, ``[INST]``, etc.) silently dropped before parsing. """ - parsed = _parse_full( + parsed = parse_chat_completion_text( text, parser_name=parser_name, reasoning_parser_name=reasoning_parser_name, @@ -223,13 +223,20 @@ async def stream_chat_completion( yield "data: [DONE]\n\n" -def _parse_full( +def parse_chat_completion_text( text: str, *, parser_name: str | None, reasoning_parser_name: str | None, noise_specials: tuple[str, ...] = (), ) -> ParsedChatOutput: + """Run the shared :class:`ChatOutputStreamer` over a full completion text. + + Public because non-streaming loader paths outside this module (e.g. + llama_cpp's ``_handle_with_parsers``) parse text themselves and feed + the result into :func:`modelship.openai.chat_utils.build_from_parsed` + rather than going through :func:`build_chat_completion_response`. + """ streamer = _make_streamer( parser_name=parser_name, reasoning_parser_name=reasoning_parser_name, diff --git a/tests/test_tool_calling.py b/tests/test_tool_calling.py index 7c77a58..54febca 100644 --- a/tests/test_tool_calling.py +++ b/tests/test_tool_calling.py @@ -8,7 +8,7 @@ import pytest from modelship.openai.parsers.output import ChatOutputStreamer -from modelship.openai.parsers.streaming import _parse_full +from modelship.openai.parsers.streaming import parse_chat_completion_text from modelship.openai.parsers.tool_calling import ( available_parsers, get_parser, @@ -266,7 +266,7 @@ def test_skipped_malformed_block_does_not_block_subsequent_finalization(self): class TestParseFullFinalization: - """Non-streaming one-shot parsing via ``_parse_full``. + """Non-streaming one-shot parsing via ``parse_chat_completion_text``. Tool-call finalization happens on ``finalize()``, so a single ``extract_streaming`` + ``finalize`` pass over the full text must @@ -278,7 +278,7 @@ def test_single_complete_call_one_shot(self): # must still finalize the call (this is the reported llama_cpp bug). text = '\n{"name": "HassTurnOff", "arguments": {"area": "Living Room"}}\n' - parsed = _parse_full(text, parser_name="hermes", reasoning_parser_name=None) + parsed = parse_chat_completion_text(text, parser_name="hermes", reasoning_parser_name=None) assert parsed.content is None assert len(parsed.tool_calls) == 1 assert parsed.tool_calls[0].function.name == "HassTurnOff" @@ -286,7 +286,7 @@ def test_single_complete_call_one_shot(self): def test_one_shot_matches_incremental_drive(self): text = '\n{"name": "HassTurnOff", "arguments": {"area": "Living Room"}}\n' - one_shot = _parse_full(text, parser_name="hermes", reasoning_parser_name=None) + one_shot = parse_chat_completion_text(text, parser_name="hermes", reasoning_parser_name=None) streamer = ChatOutputStreamer(HermesToolCallParser()) cumulative = "" @@ -304,7 +304,7 @@ def test_one_shot_matches_incremental_drive(self): def test_content_only_text_unchanged(self): text = "Just a plain answer, no tools here." - parsed = _parse_full(text, parser_name="hermes", reasoning_parser_name=None) + parsed = parse_chat_completion_text(text, parser_name="hermes", reasoning_parser_name=None) assert parsed.content == text assert parsed.tool_calls == [] @@ -313,7 +313,7 @@ def test_two_calls_one_shot_both_finalized(self): '\n{"name": "a", "arguments": {"x": 1}}\n' '\n{"name": "b", "arguments": {"y": 2}}\n' ) - parsed = _parse_full(text, parser_name="hermes", reasoning_parser_name=None) + parsed = parse_chat_completion_text(text, parser_name="hermes", reasoning_parser_name=None) assert [tc.function.name for tc in parsed.tool_calls] == ["a", "b"] assert parsed.tool_calls[0].id != parsed.tool_calls[1].id From e51818793c89b193e066135ad7a3dd8255147274 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 16:36:14 +0000 Subject: [PATCH 23/35] feat: rewire vLLM non-stream chat onto engine_ops Stage C3 of the parser-migration roadmap. Non-stream chat completions now go through the engine_ops pipeline (render_and_params -> generate -> parse -> build_from_parsed) instead of vLLM's OpenAIServingChat monolith; streaming is unchanged. Covers n>1, logprobs, and strips vLLM extension fields structurally (response is built from modelship types only). Adds BaseInfer.run_cancellable(), a generic disconnect-guard usable by any loader (poll RawRequestProxy.is_disconnected() alongside in-flight work, cancel whichever loses, call an overridable on_generation_aborted hook) -- vLLM is the first caller since AsyncLLM.generate() already self-aborts on task cancellation. Real GPU integration run caught and fixed a pre-existing bug: init_serving_chat never passed reasoning_parser to OpenAIServingRender, so a reasoning-only deployment's engine_ops-based parser silently lost reasoning extraction. --- modelship/infer/base_infer.py | 62 +++++++++- modelship/infer/vllm/engine_ops.py | 176 +++++++++++++++++++++++++++- modelship/infer/vllm/vllm_infer.py | 114 ++++++++++++++---- tests/test_base_infer_disconnect.py | 118 +++++++++++++++++++ tests/test_integration.py | 36 ++++++ tests/test_vllm_vision.py | 9 +- 6 files changed, 487 insertions(+), 28 deletions(-) create mode 100644 tests/test_base_infer_disconnect.py diff --git a/modelship/infer/base_infer.py b/modelship/infer/base_infer.py index 6f16f8d..748228b 100644 --- a/modelship/infer/base_infer.py +++ b/modelship/infer/base_infer.py @@ -1,6 +1,9 @@ +import asyncio +import contextlib import struct from abc import ABC, abstractmethod -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Coroutine +from typing import Any, TypeVar from modelship.infer.infer_config import ModelshipModelConfig, RawRequestProxy from modelship.logging import get_logger @@ -50,6 +53,15 @@ ) MINIMAL_WAV = _MINIMAL_WAV_HEADER + b"\x00\x00" +_DISCONNECT_POLL_INTERVAL_S = 0.1 + +T = TypeVar("T") + + +class ClientDisconnectedError(Exception): + """Raised by `BaseInfer.run_cancellable` when the client disconnects before + the guarded work finishes.""" + class BaseInfer(ABC): def __init__(self, model_config: ModelshipModelConfig): @@ -66,6 +78,54 @@ def _set_max_context_length(self, length: int | None) -> None: self.max_context_length = length logger.info("max_context_length for %s: %s", self.model_config.name, self.max_context_length) + async def run_cancellable(self, work: Coroutine[Any, Any, T], raw_request: RawRequestProxy) -> T: + """Run `work` to completion, or cancel it and raise `ClientDisconnectedError` + if the client disconnects first. + + A non-streaming Ray Serve call has no socket to watch: unlike streaming + (where Starlette's own `StreamingResponse` races disconnect against the + body iterator and cancellation propagates down through the whole chain + automatically), a single-shot non-stream call would otherwise run to + completion for a client that's already gone. This polls + `RawRequestProxy.is_disconnected()` (the same cross-process + DisconnectRegistry signal the streaming path's disconnect ultimately + traces back to) alongside `work` and cancels whichever loses. + + Cancelling the task is often sufficient by itself — e.g. vLLM's + `AsyncLLM.generate()` aborts its own engine-side request when its + consuming task is cancelled, needing no extra cleanup here. Loaders + whose engine needs cleanup beyond task cancellation (freeing a + connection/slot, etc.) should override `on_generation_aborted`. + """ + task = asyncio.ensure_future(work) + watch = asyncio.ensure_future(self._poll_disconnect(raw_request)) + done, _pending = await asyncio.wait({task, watch}, return_when=asyncio.FIRST_COMPLETED) + if task in done: + watch.cancel() + with contextlib.suppress(asyncio.CancelledError): + await watch + return task.result() + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + await self.on_generation_aborted() + raise ClientDisconnectedError + + async def on_generation_aborted(self) -> None: + """Hook for loaders whose engine needs cleanup beyond task cancellation + when `run_cancellable` aborts a request on client disconnect. No-op by + default — most engines are naturally cleaned up by cancellation alone + (or, like a blocking call already running in a thread pool, can't be + interrupted early regardless of what happens here).""" + return None + + @staticmethod + async def _poll_disconnect(raw_request: RawRequestProxy) -> None: + while True: + if await raw_request.is_disconnected(): + return + await asyncio.sleep(_DISCONNECT_POLL_INTERVAL_S) + @abstractmethod def shutdown(self) -> None: """Synchronously release resources (engine processes, GPU memory, etc.). diff --git a/modelship/infer/vllm/engine_ops.py b/modelship/infer/vllm/engine_ops.py index 7f9c162..4124086 100644 --- a/modelship/infer/vllm/engine_ops.py +++ b/modelship/infer/vllm/engine_ops.py @@ -6,16 +6,21 @@ vLLM version bump's blast radius confined to one file. """ -from collections.abc import AsyncGenerator, Mapping +from collections.abc import AsyncGenerator, Mapping, Sequence from typing import Any +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedToolChoiceParam, +) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest as VllmChatCompletionRequest, ) from vllm.entrypoints.openai.engine.protocol import ErrorResponse as VllmErrorResponse +from vllm.entrypoints.openai.engine.protocol import FunctionCall as VllmFunctionCall from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.entrypoints.serve.utils.api_utils import get_max_tokens from vllm.inputs import EngineInput +from vllm.logprobs import Logprob from vllm.outputs import RequestOutput from vllm.parser import Parser from vllm.renderers.inputs.preprocess import extract_prompt_components, extract_prompt_len @@ -23,7 +28,16 @@ from vllm.tokenizers import TokenizerLike from vllm.v1.engine.async_llm import AsyncLLM -from modelship.openai.protocol import ChatCompletionRequest +from modelship.openai.chat_utils import ParsedChatOutput +from modelship.openai.protocol import ( + ChatCompletionLogProb, + ChatCompletionLogProbs, + ChatCompletionLogProbsContent, + ChatCompletionRequest, + FunctionCall, + ToolCall, + random_uuid, +) def build_vllm_request( @@ -159,3 +173,161 @@ def generate( reasoning_ended=reasoning_ended, reasoning_parser_kwargs=reasoning_parser_kwargs, ) + + +def project_tool_calls(vllm_tool_calls: list[VllmFunctionCall] | None) -> list[ToolCall]: + """Project a parser's vLLM-shaped tool calls onto modelship's OpenAI `ToolCall`. + + `vllm.parser.Parser.parse()`'s `FunctionCall` has the same `id`/`name`/`arguments` + shape as modelship's own; `id` is only set when the tool_call_id_type config + minted one (e.g. kimi_k2), so most calls need one generated here. + """ + return [ + ToolCall( + id=tc.id or f"chatcmpl-tool-{random_uuid()}", + function=FunctionCall(name=tc.name, arguments=tc.arguments), + ) + for tc in (vllm_tool_calls or []) + ] + + +def build_chat_logprobs( + token_ids: Sequence[int], + top_logprobs: Sequence[dict[int, Logprob] | None], + tokenizer: TokenizerLike, + num_output_top_logprobs: int | None, +) -> ChatCompletionLogProbs: + """Project a choice's per-token logprobs onto modelship's OpenAI logprobs shape. + + Mirrors `OpenAIServingChat._create_chat_logprobs`/`_get_top_logprobs`, minus the + `return_tokens_as_token_ids` branch — modelship's request has no such field, so + tokens are always decoded to text. + """ + content: list[ChatCompletionLogProbsContent] = [] + for i, token_id in enumerate(token_ids): + step_top_logprobs = top_logprobs[i] + chosen = step_top_logprobs.get(token_id) if step_top_logprobs else None + if chosen is None: + token = tokenizer.decode(token_id) + content.append( + ChatCompletionLogProbsContent(token=token, bytes=list(token.encode("utf-8", errors="replace"))) + ) + continue + decoded = chosen.decoded_token if chosen.decoded_token is not None else tokenizer.decode(token_id) + content.append( + ChatCompletionLogProbsContent( + token=decoded, + logprob=max(chosen.logprob, -9999.0), + bytes=list(decoded.encode("utf-8", errors="replace")), + top_logprobs=[ + ChatCompletionLogProb( + token=(tok := lp.decoded_token if lp.decoded_token is not None else tokenizer.decode(tid)), + logprob=max(lp.logprob, -9999.0), + bytes=list(tok.encode("utf-8", errors="replace")), + ) + for idx, (tid, lp) in enumerate(step_top_logprobs.items()) + if (num_output_top_logprobs and idx < num_output_top_logprobs) or num_output_top_logprobs == -1 + ] + if step_top_logprobs + else [], + ) + ) + return ChatCompletionLogProbs(content=content) + + +async def consume_final_output( + engine: AsyncLLM, + engine_input: EngineInput, + sampling_params: SamplingParams, + request_id: str, + *, + reasoning_ended: bool | None, + parser: Parser | None, + chat_template_kwargs: dict[str, Any] | None, +) -> RequestOutput: + """Drive `generate()` to completion and return the final `RequestOutput`. + + Non-streaming only needs the last output (it carries every choice's full + text). Cancelling the task awaiting this coroutine (e.g. on client + disconnect) propagates into the `async for` below and into `AsyncLLM.generate`'s + own `except (CancelledError, GeneratorExit): abort(...)` — no separate abort + call is needed here. + """ + final: RequestOutput | None = None + async for res in generate( + engine, + engine_input, + sampling_params, + request_id, + reasoning_ended=reasoning_ended, + parser=parser, + chat_template_kwargs=chat_template_kwargs, + ): + final = res + if final is None: + raise RuntimeError(f"engine produced no output for request {request_id}") + return final + + +def _finish_reason_for_choice( + vllm_req: VllmChatCompletionRequest, + has_tool_calls: bool, + engine_finish_reason: str | None, +) -> str: + """OpenAI `finish_reason` for one choice, mirroring `OpenAIServingChat`'s precedence. + + A parsed tool call reports finish_reason="tool_calls" for auto/required + tool_choice, but the engine's own reason (usually "stop") for a named-function + tool_choice — the client already knows which function was called, so the turn + just "stopped" rather than the model "deciding" to call a tool. + """ + if not has_tool_calls: + return engine_finish_reason or "stop" + if isinstance(vllm_req.tool_choice, ChatCompletionNamedToolChoiceParam): + return engine_finish_reason or "stop" + return "tool_calls" + + +def build_choices( + final_res: RequestOutput, + vllm_req: VllmChatCompletionRequest, + parser: Parser | None, + tokenizer: TokenizerLike, + *, + enable_auto_tools: bool, + want_logprobs: bool, + num_output_top_logprobs: int | None, +) -> tuple[list[ParsedChatOutput], list[str | None], list[ChatCompletionLogProbs | None]]: + """Parse every choice in a finished `RequestOutput` into modelship's response DTOs. + + Non-streaming reuses one shared `parser` instance across every choice — + `.parse()` is stateless per full-text call, unlike the streaming path's + per-choice `Parser._stream_state` (see `make_parsers`). + """ + choices: list[ParsedChatOutput] = [] + finish_reasons: list[str | None] = [] + logprobs_list: list[ChatCompletionLogProbs | None] = [] + + for output in final_res.outputs: + if parser is not None: + reasoning, content, raw_tool_calls = parser.parse( + output.text, + vllm_req, + enable_auto_tools=enable_auto_tools, + model_output_token_ids=output.token_ids, + ) + else: + reasoning, content, raw_tool_calls = None, output.text, None + + dto = ParsedChatOutput(content=content, reasoning=reasoning, tool_calls=project_tool_calls(raw_tool_calls)) + choices.append(dto) + finish_reasons.append(_finish_reason_for_choice(vllm_req, dto.has_tool_calls, output.finish_reason)) + + if want_logprobs and output.logprobs is not None: + logprobs_list.append( + build_chat_logprobs(output.token_ids, output.logprobs, tokenizer, num_output_top_logprobs) + ) + else: + logprobs_list.append(None) + + return choices, finish_reasons, logprobs_list diff --git a/modelship/infer/vllm/vllm_infer.py b/modelship/infer/vllm/vllm_infer.py index b597436..2113e72 100644 --- a/modelship/infer/vllm/vllm_infer.py +++ b/modelship/infer/vllm/vllm_infer.py @@ -11,9 +11,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest as VllmChatCompletionRequest, ) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionResponse as VllmChatCompletionResponse, -) from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse as VllmErrorResponse, @@ -50,12 +47,13 @@ from vllm.usage.usage_lib import UsageContext from vllm.v1.engine.async_llm import AsyncLLM -from modelship.infer.base_infer import MINIMAL_WAV, BaseInfer +from modelship.infer.base_infer import MINIMAL_WAV, BaseInfer, ClientDisconnectedError from modelship.infer.infer_config import ModelshipModelConfig, ModelUsecase, RawRequestProxy, VllmEngineConfig +from modelship.infer.vllm import engine_ops from modelship.infer.vllm.capabilities import VllmCapabilities from modelship.logging import get_logger from modelship.metrics import _ENABLED as _METRICS_ENABLED -from modelship.openai.chat_utils import UnsupportedContentError, normalize_chat_messages +from modelship.openai.chat_utils import UnsupportedContentError, build_from_parsed, normalize_chat_messages from modelship.openai.parsers.reasoning import resolve_active_reasoning_parser from modelship.openai.parsers.utils import render_generation_prompt from modelship.openai.protocol import ( @@ -70,9 +68,11 @@ TranslationRequest, TranslationResponse, TranslationResponseVerbose, + UsageInfo, create_error_response, ) from modelship.preflight import discover_hardware, merge_with_user_overrides, run_preflight +from modelship.utils import base_request_id logger = get_logger("infer.vllm") @@ -283,7 +283,10 @@ async def init_serving_chat(self) -> OpenAIServingChat | None: or "" ) - openai_serving_render = OpenAIServingRender( + # Stashed for the non-stream path (engine_ops pipeline), which renders + # and parses directly rather than going through OpenAIServingChat. + self._enable_auto_tools = enable_tools + self.openai_serving_render = OpenAIServingRender( model_config=self.engine.model_config, renderer=self.engine.renderer, model_registry=models.registry, @@ -292,12 +295,13 @@ async def init_serving_chat(self) -> OpenAIServingChat | None: chat_template_content_format=self.vllm_engine_kwargs.chat_template_content_format, enable_auto_tools=enable_tools, tool_parser=tool_parser_name, + reasoning_parser=reasoning_parser_name, ) return OpenAIServingChat( engine_client=self.engine, models=models, - openai_serving_render=openai_serving_render, + openai_serving_render=self.openai_serving_render, response_role="assistant", request_logger=RequestLogger(max_log_len=None), chat_template=None, @@ -376,24 +380,90 @@ async def create_chat_completion( ) except UnsupportedContentError as exc: return create_error_response(exc) - request_data = request.model_dump() - # vLLM renders the chat template internally; merge the model's default - # kwargs under any per-request values (request wins). - if self.model_config.chat_template_kwargs: - request_data["chat_template_kwargs"] = { - **self.model_config.chat_template_kwargs, - **(request_data.get("chat_template_kwargs") or {}), - } - vllm_request = VllmChatCompletionRequest(**request_data) + vllm_request = engine_ops.build_vllm_request(request, self.model_config.chat_template_kwargs) + + if request.stream: + try: + result = await self.serving_chat.create_chat_completion(vllm_request, cast("Request", raw_request)) + except VLLMValidationError as exc: + return _validation_error(exc) + if isinstance(result, VllmErrorResponse): + return ErrorResponse.model_validate(result.model_dump()) + return cast("AsyncGenerator[str, None]", result) + + return await self._create_chat_completion_no_stream(request, vllm_request, raw_request) + + async def _create_chat_completion_no_stream( + self, + request: ChatCompletionRequest, + vllm_request: VllmChatCompletionRequest, + raw_request: RawRequestProxy, + ) -> ErrorResponse | ChatCompletionResponse: + """Non-stream chat path via `engine_ops`, bypassing `OpenAIServingChat`.""" try: - result = await self.serving_chat.create_chat_completion(vllm_request, cast("Request", raw_request)) + render_result = await engine_ops.render_and_params(self.openai_serving_render, vllm_request) except VLLMValidationError as exc: return _validation_error(exc) - if isinstance(result, VllmErrorResponse): - return ErrorResponse.model_validate(result.model_dump()) - if isinstance(result, VllmChatCompletionResponse): - return ChatCompletionResponse.model_validate(result.model_dump()) - return cast("AsyncGenerator[str, None]", result) + if isinstance(render_result, VllmErrorResponse): + return ErrorResponse.model_validate(render_result.model_dump()) + engine_input, sampling_params = render_result + + tokenizer = self.openai_serving_render.renderer.tokenizer + assert tokenizer is not None, "vllm renderer has no tokenizer (skip_tokenizer_init=True is unsupported here)" + + # Non-streaming reuses one parser instance across every choice (see + # engine_ops.build_choices), so only one is needed here regardless of n. + parser = engine_ops.make_parsers( + self.openai_serving_render, tokenizer, vllm_request, vllm_request.chat_template_kwargs, n=1 + )[0] + prompt_token_ids = engine_ops.extract_prompt_token_ids(self.openai_serving_render, engine_input) + reasoning_ended = engine_ops.derive_reasoning_ended(vllm_request, parser, prompt_token_ids) + + request_id = f"chatcmpl-{base_request_id(raw_request)}" + try: + final_res = await self.run_cancellable( + engine_ops.consume_final_output( + self.engine, + engine_input, + sampling_params, + request_id, + reasoning_ended=reasoning_ended, + parser=parser, + chat_template_kwargs=vllm_request.chat_template_kwargs, + ), + raw_request, + ) + except ClientDisconnectedError: + return create_error_response("Client disconnected") + except VLLMValidationError as exc: + return _validation_error(exc) + + choices, finish_reasons, logprobs_list = engine_ops.build_choices( + final_res, + vllm_request, + parser, + tokenizer, + enable_auto_tools=self._enable_auto_tools, + want_logprobs=bool(request.logprobs), + num_output_top_logprobs=request.top_logprobs, + ) + + assert final_res.prompt_token_ids is not None + prompt_tokens = len(final_res.prompt_token_ids) + completion_tokens = sum(len(output.token_ids) for output in final_res.outputs) + + return build_from_parsed( + request_id=request_id, + model_name=self.model_config.name, + choices=choices, + usage=UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + finish_reasons=finish_reasons, + logprobs=logprobs_list, + ) async def create_embedding( self, request: EmbeddingRequest, raw_request: RawRequestProxy diff --git a/tests/test_base_infer_disconnect.py b/tests/test_base_infer_disconnect.py new file mode 100644 index 0000000..62764bd --- /dev/null +++ b/tests/test_base_infer_disconnect.py @@ -0,0 +1,118 @@ +"""Unit tests for BaseInfer.run_cancellable, the generic non-stream disconnect guard. + +Non-streaming Ray Serve calls don't get a socket to watch (unlike streaming, +where Starlette's own StreamingResponse races disconnect against the body +iterator and cancellation propagates all the way down automatically). +run_cancellable polls RawRequestProxy.is_disconnected() alongside an arbitrary +coroutine and cancels whichever loses, calling the on_generation_aborted() +hook so a loader can free engine-side resources. No GPU/Ray needed — both +race participants are plain asyncio coroutines against a minimal BaseInfer +subclass. +""" + +import asyncio +from unittest.mock import MagicMock + +import pytest + +from modelship.infer.base_infer import _DISCONNECT_POLL_INTERVAL_S, BaseInfer, ClientDisconnectedError + + +class _FakeRawRequest: + def __init__(self, *, disconnect_after: float | None): + self._disconnect_after = disconnect_after + self._start = asyncio.get_event_loop().time() + + async def is_disconnected(self) -> bool: + if self._disconnect_after is None: + return False + return asyncio.get_event_loop().time() - self._start >= self._disconnect_after + + +class _Infer(BaseInfer): + """Minimal concrete BaseInfer — only run_cancellable/on_generation_aborted are exercised.""" + + def __init__(self): + super().__init__(MagicMock()) + self.aborted = False + + def shutdown(self) -> None: + pass + + async def start(self) -> None: + pass + + async def warmup(self) -> None: + pass + + async def on_generation_aborted(self) -> None: + self.aborted = True + + +@pytest.mark.asyncio +async def test_work_finishing_first_returns_result_without_aborting(): + async def fast_work(): + await asyncio.sleep(0.01) + return "done" + + infer = _Infer() + raw_request = _FakeRawRequest(disconnect_after=None) # client never disconnects + + result = await infer.run_cancellable(fast_work(), raw_request) + + assert result == "done" + assert infer.aborted is False + + +@pytest.mark.asyncio +async def test_disconnect_cancels_work_and_calls_abort_hook(): + cancelled = asyncio.Event() + + async def slow_work(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + cancelled.set() + raise + + infer = _Infer() + raw_request = _FakeRawRequest(disconnect_after=0.0) # disconnected immediately + + with pytest.raises(ClientDisconnectedError): + await infer.run_cancellable(slow_work(), raw_request) + + assert cancelled.is_set() + assert infer.aborted is True + + +@pytest.mark.asyncio +async def test_work_exception_propagates_without_aborting(): + async def failing_work(): + raise ValueError("boom") + + infer = _Infer() + raw_request = _FakeRawRequest(disconnect_after=None) + + with pytest.raises(ValueError, match="boom"): + await infer.run_cancellable(failing_work(), raw_request) + + assert infer.aborted is False + + +@pytest.mark.asyncio +async def test_disconnect_poll_interval_does_not_starve_fast_work(): + """A slow poll interval must not delay returning once work finishes — + run_cancellable awaits asyncio.wait with FIRST_COMPLETED, not the poll loop.""" + + async def fast_work(): + return 42 + + infer = _Infer() + raw_request = _FakeRawRequest(disconnect_after=None) + + start = asyncio.get_event_loop().time() + result = await infer.run_cancellable(fast_work(), raw_request) + elapsed = asyncio.get_event_loop().time() - start + + assert result == 42 + assert elapsed < _DISCONNECT_POLL_INTERVAL_S diff --git a/tests/test_integration.py b/tests/test_integration.py index 37d8fdf..3255895 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -693,6 +693,42 @@ def test_response_format_coexists_with_tool_choice_none(self, client): parsed = json.loads(content) assert set(parsed.keys()) == {"city", "country"} + def test_n_greater_than_one_returns_independent_choices(self, client): + """n>1 needs its own parser instance per choice (see + engine_ops.make_parsers) — a shared instance would corrupt state + across choices. Sampling is non-greedy by default so choices should + actually differ; this also guards against a regression that + silently collapses every choice onto the same output. + """ + completion = client.chat.completions.create( + model="chat-capable", + messages=[{"role": "user", "content": "Say one random word."}], + max_tokens=10, + n=3, + ) + assert len(completion.choices) == 3 + assert [c.index for c in completion.choices] == [0, 1, 2] + assert all(c.message.content for c in completion.choices) + + def test_logprobs_returns_choice_logprobs(self, client): + """logprobs must be built from the engine's own RequestOutput.logprobs + (engine_ops.build_chat_logprobs), not silently dropped by the + non-stream rewire.""" + completion = client.chat.completions.create( + model="chat-capable", + messages=[{"role": "user", "content": "Say hello."}], + max_tokens=10, + logprobs=True, + top_logprobs=3, + ) + logprobs = completion.choices[0].logprobs + assert logprobs is not None and logprobs.content + first = logprobs.content[0] + assert isinstance(first.token, str) and first.token + assert isinstance(first.logprob, float) + assert 0 < len(first.top_logprobs) <= 3 + assert all(isinstance(tl.token, str) and isinstance(tl.logprob, float) for tl in first.top_logprobs) + @pytest.mark.integration @pytest.mark.vllm diff --git a/tests/test_vllm_vision.py b/tests/test_vllm_vision.py index a8e95c8..72ea4cf 100644 --- a/tests/test_vllm_vision.py +++ b/tests/test_vllm_vision.py @@ -118,11 +118,14 @@ async def test_image_part_rejected_on_text_only_model_with_400(): @pytest.mark.asyncio async def test_text_only_request_reaches_serving_chat_on_vlm(): """Sanity check: a plain text request on a VLM is not blocked by the - gating layer.""" + gating layer. Streaming, since non-stream no longer calls serving_chat + (see test_vllm_engine_ops.py / test_integration.py::TestChatCapable for + the engine_ops-based non-stream path).""" infer = _make_infer(supports_image=True) request = ChatCompletionRequest( model="qwen-vl", messages=[{"role": "user", "content": "hello"}], + stream=True, ) await infer.create_chat_completion(request, raw_request=MagicMock()) @@ -133,10 +136,10 @@ async def test_text_only_request_reaches_serving_chat_on_vlm(): @pytest.mark.asyncio async def test_model_chat_template_kwargs_merged_into_vllm_request(): """The model's chat_template_kwargs default reaches the vLLM request that - serving_chat renders from.""" + serving_chat renders from. Streaming — see test above for why.""" infer = _make_infer(supports_image=False) infer.model_config.chat_template_kwargs = {"enable_thinking": False} - request = ChatCompletionRequest(model="llm", messages=[{"role": "user", "content": "hi"}]) + request = ChatCompletionRequest(model="llm", messages=[{"role": "user", "content": "hi"}], stream=True) await infer.create_chat_completion(request, raw_request=MagicMock()) From a9458f8d9c867226befea712aeb055cb4325bed5 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:48:32 +0000 Subject: [PATCH 24/35] feat: abort llama_server non-stream requests on client disconnect Routes LlamaServerInfer's non-stream chat completion through BaseInfer.run_cancellable, cancelling the in-flight httpx call to the llama-server subprocess (freeing its slot) instead of running a request to completion for a client that already left. Fixes a latent bug run_cancellable's polling exposed: RawRequestProxy is constructed with registry=None for internal warmup calls across several loaders, and is_disconnected() crashed on that instead of treating it like a dead registry (assume still connected). --- modelship/infer/infer_config.py | 3 ++ .../infer/llama_server/llama_server_infer.py | 12 ++++++- tests/test_llama_server_infer.py | 34 +++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/modelship/infer/infer_config.py b/modelship/infer/infer_config.py index d0811a1..3c37a1b 100644 --- a/modelship/infer/infer_config.py +++ b/modelship/infer/infer_config.py @@ -648,6 +648,9 @@ def __init__(self, registry, headers: dict, request_id: str | None = None): self.request_id = request_id async def is_disconnected(self) -> bool: + if self._registry is None: + # No real registry (e.g. an internal warmup request) — nothing to poll. + return False try: return await self._registry.is_set.remote(self.request_id) except RayActorError: diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 86071d7..2b5c0b1 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -13,7 +13,7 @@ import httpx -from modelship.infer.base_infer import BaseInfer +from modelship.infer.base_infer import BaseInfer, ClientDisconnectedError from modelship.infer.infer_config import LlamaServerConfig, ModelshipModelConfig, ModelUsecase, RawRequestProxy from modelship.logging import TRACE, get_logger from modelship.openai.chat_utils import ( @@ -420,6 +420,16 @@ async def create_chat_completion( if request.stream: return self._stream_chat_completion(payload, request_id) + try: + return await self.run_cancellable(self._send_chat_completion(payload, request_id), raw_request) + except ClientDisconnectedError: + logger.info("chat request %s aborted: client disconnected", request_id) + return create_error_response("Client disconnected") + + async def _send_chat_completion( + self, payload: dict[str, Any], request_id: str + ) -> ErrorResponse | ChatCompletionResponse: + assert self._client is not None try: resp = await self._client.post("/v1/chat/completions", json=payload) resp.raise_for_status() diff --git a/tests/test_llama_server_infer.py b/tests/test_llama_server_infer.py index 985b6c0..7c8195e 100644 --- a/tests/test_llama_server_infer.py +++ b/tests/test_llama_server_infer.py @@ -4,6 +4,7 @@ from __future__ import annotations +import asyncio import contextlib import stat import sys @@ -210,6 +211,19 @@ def _request(**kwargs) -> ChatCompletionRequest: return ChatCompletionRequest(model="test-model", messages=[{"role": "user", "content": "hi"}], **kwargs) +class _DisconnectingRawRequest: + """A RawRequestProxy stand-in whose is_disconnected() flips to True after + `disconnect_after` seconds, for exercising BaseInfer.run_cancellable.""" + + def __init__(self, disconnect_after: float): + self._disconnect_after = disconnect_after + self._start = asyncio.get_event_loop().time() + self.request_id = "req-1" + + async def is_disconnected(self) -> bool: + return asyncio.get_event_loop().time() - self._start >= self._disconnect_after + + class TestNonStreamingProjection: @pytest.mark.asyncio async def test_strips_extension_fields_and_maps_reasoning_and_tools(self): @@ -315,6 +329,26 @@ def handler(request: httpx.Request) -> httpx.Response: assert isinstance(result, ErrorResponse) assert "bad request" in result.error.message + @pytest.mark.asyncio + async def test_disconnect_aborts_inflight_request_without_waiting_for_response(self): + # A client disconnect should cancel the in-flight httpx call to + # llama-server (freeing its GPU/CPU slot) rather than waiting out + # a response nobody will receive. + handler_finished = False + + async def handler(request: httpx.Request) -> httpx.Response: + nonlocal handler_finished + await asyncio.sleep(10) + handler_finished = True + return httpx.Response(200, json={"choices": [], "usage": {}}) + + infer = _infer_with_client(handler) + result = await infer.create_chat_completion(_request(), _DisconnectingRawRequest(disconnect_after=0.05)) + + assert isinstance(result, ErrorResponse) + assert "disconnect" in result.error.message.lower() + assert handler_finished is False + @pytest.mark.asyncio async def test_no_client_falls_back_to_not_supported(self): infer = _infer_with_client(lambda request: httpx.Response(200)) From 643013c81cdbfce53294aa418dc86fef6b2b80d6 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:48:38 +0000 Subject: [PATCH 25/35] fix: replace assert with a defensive check for vLLM prompt_token_ids assert is stripped under python -O; a completed non-stream request with no prompt_token_ids now returns a 502 ErrorResponse instead of being one flag away from a silent AttributeError further down. --- modelship/infer/vllm/vllm_infer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modelship/infer/vllm/vllm_infer.py b/modelship/infer/vllm/vllm_infer.py index 2113e72..6ad8dd9 100644 --- a/modelship/infer/vllm/vllm_infer.py +++ b/modelship/infer/vllm/vllm_infer.py @@ -448,7 +448,8 @@ async def _create_chat_completion_no_stream( num_output_top_logprobs=request.top_logprobs, ) - assert final_res.prompt_token_ids is not None + if final_res.prompt_token_ids is None: + return create_error_response("vllm returned no prompt_token_ids for a completed request", status_code=502) prompt_tokens = len(final_res.prompt_token_ids) completion_tokens = sum(len(output.token_ids) for output in final_res.outputs) From 88b59c5e2c30bc35e570462d857347997772d1fd Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:13:43 +0000 Subject: [PATCH 26/35] feat: rewire vLLM streaming chat onto engine_ops Bypasses OpenAIServingChat for streaming the same way non-stream already was: engine_ops.stream_chat_completion drives Parser.parse_delta per choice and yields modelship's own SSE chunk types directly, so extension fields can no longer leak and the loader stops depending on vLLM's ~800-line monolith for this path. Also adds BaseInfer.run_cancellable_stream, a streaming counterpart to run_cancellable, so a client disconnect mid-stream aborts the engine request via the same explicit polling mechanism non-stream uses instead of relying solely on implicit ASGI-layer cancellation. --- modelship/infer/base_infer.py | 35 ++++++ modelship/infer/vllm/engine_ops.py | 183 ++++++++++++++++++++++++++++ modelship/infer/vllm/vllm_infer.py | 81 ++++++++++-- tests/test_base_infer_disconnect.py | 91 +++++++++++++- tests/test_integration.py | 45 +++++++ tests/test_vllm_vision.py | 20 +-- 6 files changed, 433 insertions(+), 22 deletions(-) diff --git a/modelship/infer/base_infer.py b/modelship/infer/base_infer.py index 748228b..6f0b5de 100644 --- a/modelship/infer/base_infer.py +++ b/modelship/infer/base_infer.py @@ -111,6 +111,41 @@ async def run_cancellable(self, work: Coroutine[Any, Any, T], raw_request: RawRe await self.on_generation_aborted() raise ClientDisconnectedError + async def run_cancellable_stream( + self, work: AsyncGenerator[T, None], raw_request: RawRequestProxy + ) -> AsyncGenerator[T, None]: + """Streaming counterpart of `run_cancellable`. + + Races each pulled item against the same disconnect signal: cancelling + the in-flight `__anext__()` call delivers `CancelledError` straight into + `work`'s currently-suspended frame (and transitively into whatever it's + awaiting, e.g. an engine's own generator), the same way cancelling a + plain task does for `run_cancellable`. `aclose()` afterward is a defensive + no-op when the generator already self-terminated on that exception. + """ + watch = asyncio.ensure_future(self._poll_disconnect(raw_request)) + try: + while True: + next_item = asyncio.ensure_future(work.__anext__()) + done, _pending = await asyncio.wait({next_item, watch}, return_when=asyncio.FIRST_COMPLETED) + if next_item in done: + try: + item = next_item.result() + except StopAsyncIteration: + return + yield item + continue + next_item.cancel() + with contextlib.suppress(asyncio.CancelledError): + await next_item + await work.aclose() + await self.on_generation_aborted() + raise ClientDisconnectedError + finally: + watch.cancel() + with contextlib.suppress(asyncio.CancelledError): + await watch + async def on_generation_aborted(self) -> None: """Hook for loaders whose engine needs cleanup beyond task cancellation when `run_cancellable` aborts a request on client disconnect. No-op by diff --git a/modelship/infer/vllm/engine_ops.py b/modelship/infer/vllm/engine_ops.py index 4124086..e98753e 100644 --- a/modelship/infer/vllm/engine_ops.py +++ b/modelship/infer/vllm/engine_ops.py @@ -15,6 +15,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest as VllmChatCompletionRequest, ) +from vllm.entrypoints.openai.engine.protocol import DeltaMessage as VllmDeltaMessage +from vllm.entrypoints.openai.engine.protocol import DeltaToolCall as VllmDeltaToolCall from vllm.entrypoints.openai.engine.protocol import ErrorResponse as VllmErrorResponse from vllm.entrypoints.openai.engine.protocol import FunctionCall as VllmFunctionCall from vllm.entrypoints.serve.render.serving import OpenAIServingRender @@ -34,8 +36,14 @@ ChatCompletionLogProbs, ChatCompletionLogProbsContent, ChatCompletionRequest, + ChatCompletionResponseStreamChoice, + ChatCompletionStreamResponse, + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, FunctionCall, ToolCall, + UsageInfo, random_uuid, ) @@ -191,6 +199,30 @@ def project_tool_calls(vllm_tool_calls: list[VllmFunctionCall] | None) -> list[T ] +def project_delta_tool_calls(vllm_tool_calls: list[VllmDeltaToolCall]) -> list[DeltaToolCall] | None: + """Project one streaming delta's vLLM tool-call fragments onto modelship's `DeltaToolCall`. + + Unlike `project_tool_calls`, nothing is synthesized here: only the first + delta for a given tool call carries `id`/`type`/`function.name` (per + `Parser.parse_delta`'s own streaming protocol) — later deltas for the same + `index` carry only incremental `function.arguments`, which must pass + through as-is for the client to accumulate correctly. + """ + if not vllm_tool_calls: + return None + return [ + DeltaToolCall( + index=tc.index, + id=tc.id, + type=tc.type, + function=DeltaFunctionCall(name=tc.function.name, arguments=tc.function.arguments) + if tc.function is not None + else None, + ) + for tc in vllm_tool_calls + ] + + def build_chat_logprobs( token_ids: Sequence[int], top_logprobs: Sequence[dict[int, Logprob] | None], @@ -331,3 +363,154 @@ def build_choices( logprobs_list.append(None) return choices, finish_reasons, logprobs_list + + +async def stream_chat_completion( + engine: AsyncLLM, + render: OpenAIServingRender, + vllm_req: VllmChatCompletionRequest, + engine_input: EngineInput, + sampling_params: SamplingParams, + request_id: str, + model_name: str, + tokenizer: TokenizerLike, + *, + enable_auto_tools: bool, + want_logprobs: bool, + num_output_top_logprobs: int | None, +) -> AsyncGenerator[ChatCompletionStreamResponse, None]: + """Drive one streaming chat completion end to end: per-choice parsers, + per-delta parsing via `Parser.parse_delta`, and the OpenAI streaming + chunk lifecycle (role chunk, content/tool/reasoning deltas, finish + chunk, optional usage chunk) — the streaming counterpart of `build_choices`. + + Yields fully-formed modelship chunks; the caller owns SSE encoding and + the trailing `[DONE]` line (symmetric with how `build_choices` leaves + `ChatCompletionResponse` assembly to `chat_utils.build_from_parsed`). + """ + num_choices = vllm_req.n or 1 + parsers = make_parsers(render, tokenizer, vllm_req, vllm_req.chat_template_kwargs, n=num_choices) + prompt_token_ids = extract_prompt_token_ids(render, engine_input) + reasoning_ended = derive_reasoning_ended(vllm_req, parsers[0], prompt_token_ids) + + stream_options = vllm_req.stream_options + include_usage = bool(stream_options and stream_options.include_usage) + include_continuous_usage = include_usage and bool(stream_options and stream_options.continuous_usage_stats) + + previous_num_tokens = [0] * num_choices + finish_reason_sent = [False] * num_choices + tools_streamed = [False] * num_choices + first_iteration = True + num_prompt_tokens = 0 + + async for res in generate( + engine, + engine_input, + sampling_params, + request_id, + reasoning_ended=reasoning_ended, + parser=parsers[0], + chat_template_kwargs=vllm_req.chat_template_kwargs, + ): + if res.prompt_token_ids is not None: + num_prompt_tokens = len(res.prompt_token_ids) + + if first_iteration: + first_iteration = False + role_choices = [ + ChatCompletionResponseStreamChoice(index=i, delta=DeltaMessage(role="assistant", content="")) + for i in range(num_choices) + ] + yield ChatCompletionStreamResponse( + id=request_id, + model=model_name, + choices=role_choices, + usage=_continuous_usage(num_prompt_tokens, 0) if include_continuous_usage else None, + ) + + for output in res.outputs: + i = output.index + if finish_reason_sent[i]: + continue + + delta_text = output.text + if not delta_text and not output.token_ids and not previous_num_tokens[i]: + # Chunked prefill: nothing new to emit yet. + continue + + parser = parsers[i] + if parser is not None: + vllm_delta = parser.parse_delta( + delta_text=delta_text, + delta_token_ids=list(output.token_ids), + request=vllm_req, + prompt_token_ids=res.prompt_token_ids, + finished=output.finish_reason is not None, + ) + if vllm_delta is not None and vllm_delta.tool_calls: + tools_streamed[i] = True + else: + vllm_delta = VllmDeltaMessage(content=delta_text) + + previous_num_tokens[i] += len(output.token_ids) + + if vllm_delta is None: + # Parser swallowed a control token (e.g. a `` marker) with + # nothing yet emittable — skip unless this is the final delta, + # which still needs a (possibly empty) delta to carry finish_reason. + if output.finish_reason is None: + continue + vllm_delta = VllmDeltaMessage() + + delta_message = DeltaMessage( + role=vllm_delta.role, + content=vllm_delta.content, + reasoning=vllm_delta.reasoning, + tool_calls=project_delta_tool_calls(vllm_delta.tool_calls), + ) + + logprobs = None + if want_logprobs and output.logprobs is not None: + logprobs = build_chat_logprobs(output.token_ids, output.logprobs, tokenizer, num_output_top_logprobs) + + if output.finish_reason is None: + choice = ChatCompletionResponseStreamChoice(index=i, delta=delta_message, logprobs=logprobs) + else: + finish_reason_sent[i] = True + choice = ChatCompletionResponseStreamChoice( + index=i, + delta=delta_message, + logprobs=logprobs, + finish_reason=_finish_reason_for_choice(vllm_req, tools_streamed[i], output.finish_reason), + stop_reason=output.stop_reason, + ) + + yield ChatCompletionStreamResponse( + id=request_id, + model=model_name, + choices=[choice], + usage=_continuous_usage(num_prompt_tokens, previous_num_tokens[i]) + if include_continuous_usage + else None, + ) + + if include_usage: + completion_tokens = sum(previous_num_tokens) + yield ChatCompletionStreamResponse( + id=request_id, + model=model_name, + choices=[], + usage=UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=num_prompt_tokens + completion_tokens, + ), + ) + + +def _continuous_usage(prompt_tokens: int, completion_tokens: int) -> UsageInfo: + return UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) diff --git a/modelship/infer/vllm/vllm_infer.py b/modelship/infer/vllm/vllm_infer.py index 6ad8dd9..87c5f8c 100644 --- a/modelship/infer/vllm/vllm_infer.py +++ b/modelship/infer/vllm/vllm_infer.py @@ -1,4 +1,5 @@ import io +import json from collections.abc import AsyncGenerator from http import HTTPStatus from typing import Any, ClassVar, cast @@ -51,7 +52,7 @@ from modelship.infer.infer_config import ModelshipModelConfig, ModelUsecase, RawRequestProxy, VllmEngineConfig from modelship.infer.vllm import engine_ops from modelship.infer.vllm.capabilities import VllmCapabilities -from modelship.logging import get_logger +from modelship.logging import TRACE, get_logger from modelship.metrics import _ENABLED as _METRICS_ENABLED from modelship.openai.chat_utils import UnsupportedContentError, build_from_parsed, normalize_chat_messages from modelship.openai.parsers.reasoning import resolve_active_reasoning_parser @@ -59,6 +60,7 @@ from modelship.openai.protocol import ( ChatCompletionRequest, ChatCompletionResponse, + ChatCompletionStreamResponse, EmbeddingCompletionRequest, EmbeddingRequest, ErrorResponse, @@ -77,6 +79,14 @@ logger = get_logger("infer.vllm") +def _encode_chunk(chunk: ChatCompletionStreamResponse) -> str: + return f"data: {json.dumps(chunk.model_dump(mode='json'))}\n\n" + + +def _encode_error(error: ErrorResponse) -> str: + return f"data: {json.dumps(error.model_dump(mode='json'))}\n\n" + + def _validation_error(exc: VLLMValidationError) -> ErrorResponse: # VLLMValidationError.__str__ appends "(parameter=..., value=...)"; keep the # original message and surface the offending field via the OpenAI `param` slot. @@ -383,16 +393,71 @@ async def create_chat_completion( vllm_request = engine_ops.build_vllm_request(request, self.model_config.chat_template_kwargs) if request.stream: - try: - result = await self.serving_chat.create_chat_completion(vllm_request, cast("Request", raw_request)) - except VLLMValidationError as exc: - return _validation_error(exc) - if isinstance(result, VllmErrorResponse): - return ErrorResponse.model_validate(result.model_dump()) - return cast("AsyncGenerator[str, None]", result) + return self._create_chat_completion_stream(request, vllm_request, raw_request) return await self._create_chat_completion_no_stream(request, vllm_request, raw_request) + async def _create_chat_completion_stream( + self, + request: ChatCompletionRequest, + vllm_request: VllmChatCompletionRequest, + raw_request: RawRequestProxy, + ) -> AsyncGenerator[str, None]: + """Streaming chat path via `engine_ops`, bypassing `OpenAIServingChat`.""" + request_id = f"chatcmpl-{base_request_id(raw_request)}" + try: + render_result = await engine_ops.render_and_params(self.openai_serving_render, vllm_request) + except VLLMValidationError as exc: + yield _encode_error(_validation_error(exc)) + yield "data: [DONE]\n\n" + return + if isinstance(render_result, VllmErrorResponse): + yield _encode_error(ErrorResponse.model_validate(render_result.model_dump())) + yield "data: [DONE]\n\n" + return + engine_input, sampling_params = render_result + + tokenizer = self.openai_serving_render.renderer.tokenizer + assert tokenizer is not None, "vllm renderer has no tokenizer (skip_tokenizer_init=True is unsupported here)" + + stream = engine_ops.stream_chat_completion( + self.engine, + self.openai_serving_render, + vllm_request, + engine_input, + sampling_params, + request_id, + self.model_config.name, + tokenizer, + enable_auto_tools=self._enable_auto_tools, + want_logprobs=bool(request.logprobs), + num_output_top_logprobs=request.top_logprobs, + ) + buffered: list[str] = [] + try: + async for chunk in self.run_cancellable_stream(stream, raw_request): + for choice in chunk.choices: + if choice.delta.content: + buffered.append(choice.delta.content) + yield _encode_chunk(chunk) + except ClientDisconnectedError: + logger.info("chat request %s aborted: client disconnected", request_id) + return + except VLLMValidationError as exc: + yield _encode_error(_validation_error(exc)) + yield "data: [DONE]\n\n" + return + except Exception: + logger.exception("chat request %s failed mid-stream", request_id) + yield _encode_error( + create_error_response("Internal error during generation", err_type="api_error", status_code=500) + ) + yield "data: [DONE]\n\n" + return + finally: + logger.log(TRACE, "chat response %s (stream): %r", request_id, "".join(buffered)) + yield "data: [DONE]\n\n" + async def _create_chat_completion_no_stream( self, request: ChatCompletionRequest, diff --git a/tests/test_base_infer_disconnect.py b/tests/test_base_infer_disconnect.py index 62764bd..0f30476 100644 --- a/tests/test_base_infer_disconnect.py +++ b/tests/test_base_infer_disconnect.py @@ -1,13 +1,17 @@ -"""Unit tests for BaseInfer.run_cancellable, the generic non-stream disconnect guard. +"""Unit tests for BaseInfer.run_cancellable / run_cancellable_stream, the +generic disconnect guards. Non-streaming Ray Serve calls don't get a socket to watch (unlike streaming, where Starlette's own StreamingResponse races disconnect against the body iterator and cancellation propagates all the way down automatically). run_cancellable polls RawRequestProxy.is_disconnected() alongside an arbitrary coroutine and cancels whichever loses, calling the on_generation_aborted() -hook so a loader can free engine-side resources. No GPU/Ray needed — both -race participants are plain asyncio coroutines against a minimal BaseInfer -subclass. +hook so a loader can free engine-side resources. run_cancellable_stream does +the same thing per-item for an async generator, so a loader can opt into +explicit disconnect handling for streaming too instead of relying solely on +the ASGI layer's own cancellation-on-disconnect. No GPU/Ray needed — both +race participants are plain asyncio coroutines/generators against a minimal +BaseInfer subclass. """ import asyncio @@ -116,3 +120,82 @@ async def fast_work(): assert result == 42 assert elapsed < _DISCONNECT_POLL_INTERVAL_S + + +@pytest.mark.asyncio +async def test_stream_yields_items_without_aborting(): + async def gen(): + for i in range(3): + yield i + + infer = _Infer() + raw_request = _FakeRawRequest(disconnect_after=None) + + items = [item async for item in infer.run_cancellable_stream(gen(), raw_request)] + + assert items == [0, 1, 2] + assert infer.aborted is False + + +@pytest.mark.asyncio +async def test_stream_disconnect_cancels_work_and_calls_abort_hook(): + cancelled = asyncio.Event() + + async def gen(): + yield "first" + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + cancelled.set() + raise + yield "unreachable" # pragma: no cover - never reached + + infer = _Infer() + raw_request = _FakeRawRequest(disconnect_after=0.05) + + received = [] + with pytest.raises(ClientDisconnectedError): + async for item in infer.run_cancellable_stream(gen(), raw_request): + received.append(item) + + assert received == ["first"] + assert cancelled.is_set() + assert infer.aborted is True + + +@pytest.mark.asyncio +async def test_stream_exception_propagates_without_aborting(): + async def gen(): + yield "first" + raise ValueError("boom") + + infer = _Infer() + raw_request = _FakeRawRequest(disconnect_after=None) + + received = [] + with pytest.raises(ValueError, match="boom"): + async for item in infer.run_cancellable_stream(gen(), raw_request): + received.append(item) + + assert received == ["first"] + assert infer.aborted is False + + +@pytest.mark.asyncio +async def test_stream_disconnect_poll_interval_does_not_starve_fast_stream(): + """A slow poll interval must not delay a fast stream — each `__anext__()` + races `asyncio.wait` with FIRST_COMPLETED, not the poll loop.""" + + async def gen(): + for i in range(5): + yield i + + infer = _Infer() + raw_request = _FakeRawRequest(disconnect_after=None) + + start = asyncio.get_event_loop().time() + items = [item async for item in infer.run_cancellable_stream(gen(), raw_request)] + elapsed = asyncio.get_event_loop().time() - start + + assert items == [0, 1, 2, 3, 4] + assert elapsed < _DISCONNECT_POLL_INTERVAL_S diff --git a/tests/test_integration.py b/tests/test_integration.py index 3255895..ebd691a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -729,6 +729,51 @@ def test_logprobs_returns_choice_logprobs(self, client): assert 0 < len(first.top_logprobs) <= 3 assert all(isinstance(tl.token, str) and isinstance(tl.logprob, float) for tl in first.top_logprobs) + def test_streaming_n_greater_than_one_returns_independent_choices(self, client): + """Streaming counterpart of the n>1 test above — each choice needs its + own `Parser` instance (`engine_ops.stream_chat_completion`'s per-choice + `make_parsers` call), or every choice after the first corrupts onto a + shared stream state.""" + stream = client.chat.completions.create( + model="chat-capable", + messages=[{"role": "user", "content": "Say one random word."}], + max_tokens=10, + n=3, + stream=True, + ) + content_by_index: dict[int, str] = {0: "", 1: "", 2: ""} + finish_reasons: dict[int, str | None] = {} + for chunk in stream: + for choice in chunk.choices: + if choice.delta.content: + content_by_index[choice.index] += choice.delta.content + if choice.finish_reason: + finish_reasons[choice.index] = choice.finish_reason + assert set(finish_reasons) == {0, 1, 2} + assert all(content_by_index[i] for i in range(3)) + + def test_streaming_logprobs_returns_choice_logprobs(self, client): + """Streaming counterpart of the logprobs test above — logprobs must be + built per-delta from `RequestOutput.logprobs`, not just on the final + non-streamed response.""" + stream = client.chat.completions.create( + model="chat-capable", + messages=[{"role": "user", "content": "Say hello."}], + max_tokens=10, + logprobs=True, + top_logprobs=3, + stream=True, + ) + seen_logprobs = [] + for chunk in stream: + if chunk.choices and chunk.choices[0].logprobs and chunk.choices[0].logprobs.content: + seen_logprobs.extend(chunk.choices[0].logprobs.content) + assert seen_logprobs + first = seen_logprobs[0] + assert isinstance(first.token, str) and first.token + assert isinstance(first.logprob, float) + assert 0 < len(first.top_logprobs) <= 3 + @pytest.mark.integration @pytest.mark.vllm diff --git a/tests/test_vllm_vision.py b/tests/test_vllm_vision.py index 72ea4cf..6776712 100644 --- a/tests/test_vllm_vision.py +++ b/tests/test_vllm_vision.py @@ -9,7 +9,7 @@ out — the goal there is the 400-rejection path, not the inference call. """ -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest as VllmChatCompletionRequest @@ -84,7 +84,7 @@ def _make_infer(*, supports_image: bool) -> VllmInfer: infer._caps = VllmCapabilities(supports_image=supports_image) # type: ignore[attr-defined] infer.model_config = MagicMock(chat_template_kwargs={}) infer.serving_chat = MagicMock() - infer.serving_chat.create_chat_completion = AsyncMock(return_value=MagicMock()) + infer._create_chat_completion_stream = MagicMock(return_value=MagicMock()) return infer @@ -112,15 +112,15 @@ async def test_image_part_rejected_on_text_only_model_with_400(): assert result._http_status == 400 assert "image" in result.error.message.lower() # The reject must happen before we hand off to vLLM. - infer.serving_chat.create_chat_completion.assert_not_awaited() + infer._create_chat_completion_stream.assert_not_called() @pytest.mark.asyncio -async def test_text_only_request_reaches_serving_chat_on_vlm(): +async def test_text_only_request_reaches_streaming_path_on_vlm(): """Sanity check: a plain text request on a VLM is not blocked by the - gating layer. Streaming, since non-stream no longer calls serving_chat + gating layer. Streaming, since non-stream no longer calls `_create_chat_completion_stream` (see test_vllm_engine_ops.py / test_integration.py::TestChatCapable for - the engine_ops-based non-stream path).""" + the engine_ops-based non-stream path; TestChatStreamingCapable for streaming).""" infer = _make_infer(supports_image=True) request = ChatCompletionRequest( model="qwen-vl", @@ -130,18 +130,18 @@ async def test_text_only_request_reaches_serving_chat_on_vlm(): await infer.create_chat_completion(request, raw_request=MagicMock()) - infer.serving_chat.create_chat_completion.assert_awaited_once() + infer._create_chat_completion_stream.assert_called_once() @pytest.mark.asyncio async def test_model_chat_template_kwargs_merged_into_vllm_request(): - """The model's chat_template_kwargs default reaches the vLLM request that - serving_chat renders from. Streaming — see test above for why.""" + """The model's chat_template_kwargs default reaches the vLLM request built + for the streaming path.""" infer = _make_infer(supports_image=False) infer.model_config.chat_template_kwargs = {"enable_thinking": False} request = ChatCompletionRequest(model="llm", messages=[{"role": "user", "content": "hi"}], stream=True) await infer.create_chat_completion(request, raw_request=MagicMock()) - vllm_request = infer.serving_chat.create_chat_completion.await_args.args[0] + vllm_request = infer._create_chat_completion_stream.call_args.args[1] assert vllm_request.chat_template_kwargs == {"enable_thinking": False} From 3e816b8316db87842351155d89b7e096212e38e4 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:25:08 +0000 Subject: [PATCH 27/35] fix: guard against out-of-bounds top_logprobs index in vLLM logprobs projection Prevents an IndexError in build_chat_logprobs if token_ids and top_logprobs sequences are ever mismatched in length. --- modelship/infer/vllm/engine_ops.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modelship/infer/vllm/engine_ops.py b/modelship/infer/vllm/engine_ops.py index e98753e..710e81a 100644 --- a/modelship/infer/vllm/engine_ops.py +++ b/modelship/infer/vllm/engine_ops.py @@ -237,7 +237,9 @@ def build_chat_logprobs( """ content: list[ChatCompletionLogProbsContent] = [] for i, token_id in enumerate(token_ids): - step_top_logprobs = top_logprobs[i] + # Defensive: token_ids and top_logprobs should be the same length, but + # guard against a mismatched pair rather than risk an IndexError. + step_top_logprobs = top_logprobs[i] if i < len(top_logprobs) else None chosen = step_top_logprobs.get(token_id) if step_top_logprobs else None if chosen is None: token = tokenizer.decode(token_id) From ee4c151a7094bb685345e0241b46c6b8169746b6 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:25:41 +0000 Subject: [PATCH 28/35] fix: wire llama_server streaming chat onto client disconnect and stamp chunk ids Ray Serve doesn't propagate client disconnects across the process boundary, so the streaming path now races the SSE consumption against the same cross-process disconnect signal run_cancellable already uses for non-stream requests, closing the httpx stream (and freeing the llama-server --parallel slot) as soon as the client is gone. Also stamps each ChatCompletionStreamResponse chunk with the actual completion id instead of a fresh random uuid per chunk, matching the non-stream path's id=data.get("id") or request_id convention. --- .../infer/llama_server/llama_server_infer.py | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 2b5c0b1..b3d5d9e 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -418,7 +418,7 @@ async def create_chat_completion( payload = _build_payload(request, messages, model_name=self.model_config.name) if request.stream: - return self._stream_chat_completion(payload, request_id) + return self._stream_chat_completion(payload, request_id, raw_request) try: return await self.run_cancellable(self._send_chat_completion(payload, request_id), raw_request) @@ -453,7 +453,25 @@ async def _send_chat_completion( return create_error_response(message or "Unknown error returned from llama-server", status_code=502) return _project_chat_response(data, model_name=self.model_config.name, request_id=request_id) - async def _stream_chat_completion(self, payload: dict[str, Any], request_id: str) -> AsyncGenerator[str, None]: + async def _stream_chat_completion( + self, payload: dict[str, Any], request_id: str, raw_request: RawRequestProxy + ) -> AsyncGenerator[str, None]: + """Race the llama-server SSE stream against the client's disconnect signal. + + Ray Serve gives replicas no automatic disconnect propagation across the + process boundary, so without this a gone client would leave `_stream_chat_completion_body` + (and its `httpx` stream against llama-server) running to completion, + holding a `--parallel` slot until the response finishes on its own. + """ + stream = self._stream_chat_completion_body(payload, request_id) + try: + async for chunk in self.run_cancellable_stream(stream, raw_request): + yield chunk + except ClientDisconnectedError: + logger.info("chat request %s aborted: client disconnected", request_id) + return + + async def _stream_chat_completion_body(self, payload: dict[str, Any], request_id: str) -> AsyncGenerator[str, None]: assert self._client is not None buffered: list[str] = [] try: @@ -489,7 +507,7 @@ async def _stream_chat_completion(self, payload: dict[str, Any], request_id: str yield _encode_error(message or "Unknown mid-stream error from llama-server") yield "data: [DONE]\n\n" return - chunk = _project_stream_chunk(data, model_name=self.model_config.name) + chunk = _project_stream_chunk(data, model_name=self.model_config.name, request_id=request_id) for choice in chunk.choices: if choice.delta.content: buffered.append(choice.delta.content) @@ -610,7 +628,7 @@ def _project_chat_response(data: dict, *, model_name: str, request_id: str) -> C ) -def _project_stream_chunk(data: dict, *, model_name: str) -> ChatCompletionStreamResponse: +def _project_stream_chunk(data: dict, *, model_name: str, request_id: str) -> ChatCompletionStreamResponse: choices = [] for choice in data.get("choices", []): delta = choice.get("delta") or {} @@ -647,6 +665,7 @@ def _project_stream_chunk(data: dict, *, model_name: str) -> ChatCompletionStrea ) ) return ChatCompletionStreamResponse( + id=data.get("id") or request_id, model=model_name, choices=choices, usage=_project_usage(data["usage"]) if data.get("usage") else None, From 59d3ebc97adf9feed92b95318e58342fd4c26676 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:32:33 +0000 Subject: [PATCH 29/35] refactor: delete vLLM OpenAIServingChat monolith usage C3/C4 already moved all real chat work onto engine_ops; OpenAIServingChat was only being constructed to serve as a truthy/None capability gate, never actually called. Drop its import/construction and gate on hasattr(self, "openai_serving_render") instead, matching the transformers loader's existing gating pattern. --- modelship/infer/vllm/vllm_infer.py | 29 ++++++++--------------------- tests/test_vllm_vision.py | 6 +++--- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/modelship/infer/vllm/vllm_infer.py b/modelship/infer/vllm/vllm_infer.py index 87c5f8c..ea85a32 100644 --- a/modelship/infer/vllm/vllm_infer.py +++ b/modelship/infer/vllm/vllm_infer.py @@ -12,7 +12,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest as VllmChatCompletionRequest, ) -from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse as VllmErrorResponse, ) @@ -217,7 +216,7 @@ async def start(self): self._caps = VllmCapabilities.detect(self.vllm_config.model_config) logger.info("vllm capabilities for '%s': %s", self.model_config.name, self._caps) - self.serving_chat = await self.init_serving_chat() + await self.init_serving_chat() self.serving_embedding = await self.init_serving_embeding() self.serving_transcription = await self.init_serving_transcription() self.serving_translation = await self.init_serving_translation() @@ -226,7 +225,7 @@ async def warmup(self) -> None: logger.info("Warming up vllm model: %s", self.model_config.name) dummy_proxy = RawRequestProxy(None, {}) - if self.serving_chat is not None: + if hasattr(self, "openai_serving_render"): request = ChatCompletionRequest( model=self.model_config.name, messages=[{"role": "user", "content": "warmup"}], max_tokens=1, seed=-1 ) @@ -266,10 +265,13 @@ async def warmup(self) -> None: pass logger.info("Warmup translation done for %s", self.model_config.name) - async def init_serving_chat(self) -> OpenAIServingChat | None: + async def init_serving_chat(self) -> None: + """Sets up the render/parse pipeline `create_chat_completion` drives directly + (see engine_ops), if the model supports it. Leaves `openai_serving_render` + unset otherwise — callers gate on `hasattr(self, "openai_serving_render")`.""" logger.info("init_serving_chat: %s, %s", self.supported_tasks, self.model_config.usecase) if not (self.model_config.usecase is ModelUsecase.generate and "generate" in self.supported_tasks): - return None + return models = OpenAIServingModels( engine_client=self.engine, @@ -293,8 +295,6 @@ async def init_serving_chat(self) -> OpenAIServingChat | None: or "" ) - # Stashed for the non-stream path (engine_ops pipeline), which renders - # and parses directly rather than going through OpenAIServingChat. self._enable_auto_tools = enable_tools self.openai_serving_render = OpenAIServingRender( model_config=self.engine.model_config, @@ -308,19 +308,6 @@ async def init_serving_chat(self) -> OpenAIServingChat | None: reasoning_parser=reasoning_parser_name, ) - return OpenAIServingChat( - engine_client=self.engine, - models=models, - openai_serving_render=self.openai_serving_render, - response_role="assistant", - request_logger=RequestLogger(max_log_len=None), - chat_template=None, - chat_template_content_format=self.vllm_engine_kwargs.chat_template_content_format, - enable_auto_tools=enable_tools, - tool_parser=tool_parser_name, - reasoning_parser=reasoning_parser_name, - ) - async def init_serving_embeding(self) -> ServingEmbedding | None: logger.info("init_serving_embeding: %s, %s", self.supported_tasks, self.model_config.usecase) return ( @@ -380,7 +367,7 @@ async def init_serving_translation(self) -> OpenAIServingTranslation | None: async def create_chat_completion( self, request: ChatCompletionRequest, raw_request: RawRequestProxy ) -> ErrorResponse | ChatCompletionResponse | AsyncGenerator[str, None]: - if self.serving_chat is None: + if not hasattr(self, "openai_serving_render"): return await super().create_chat_completion(request, raw_request) try: request.messages = normalize_chat_messages( diff --git a/tests/test_vllm_vision.py b/tests/test_vllm_vision.py index 6776712..e570d54 100644 --- a/tests/test_vllm_vision.py +++ b/tests/test_vllm_vision.py @@ -5,8 +5,8 @@ (vLLM's own pydantic model). If image parts survive that, vLLM's downstream multimodal preprocessing runs against the same shape it does in upstream deployments. Wrapper-level concerns (normalize_chat_messages gating) are -tested via :class:`VllmInfer.create_chat_completion` with serving_chat stubbed -out — the goal there is the 400-rejection path, not the inference call. +tested via :class:`VllmInfer.create_chat_completion` with the streaming path +stubbed out — the goal there is the 400-rejection path, not the inference call. """ from unittest.mock import MagicMock @@ -83,7 +83,7 @@ def _make_infer(*, supports_image: bool) -> VllmInfer: infer = VllmInfer.__new__(VllmInfer) infer._caps = VllmCapabilities(supports_image=supports_image) # type: ignore[attr-defined] infer.model_config = MagicMock(chat_template_kwargs={}) - infer.serving_chat = MagicMock() + infer.openai_serving_render = MagicMock() infer._create_chat_completion_stream = MagicMock(return_value=MagicMock()) return infer From 86226deab9b37e057d57a7dba68076981f87f828 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Sat, 4 Jul 2026 07:32:26 +0000 Subject: [PATCH 30/35] feat: shape /v1/responses natively from ParsedChatOutput for vllm and llama_server Adds a dedicated BaseInfer.create_response hook so vllm/llama_server build Responses items directly from the parsed DTO instead of round-tripping through a ChatCompletionResponse; the gateway route now just dispatches the original request to a new ModelDeployment.respond() method. Non-streaming only for now (falls back to unsupported for stream=True and every other loader) pending native streaming support. --- modelship/infer/base_infer.py | 7 + .../infer/llama_server/llama_server_infer.py | 103 +++++++++++++- modelship/infer/model_deployment.py | 22 +++ modelship/infer/vllm/vllm_infer.py | 128 +++++++++++++++++- modelship/openai/api.py | 24 +--- modelship/openai/chat_utils.py | 31 +++++ tests/test_chat_utils.py | 24 ++++ tests/test_responses_api.py | 86 +++++------- 8 files changed, 346 insertions(+), 79 deletions(-) diff --git a/modelship/infer/base_infer.py b/modelship/infer/base_infer.py index 6f0b5de..0ad2965 100644 --- a/modelship/infer/base_infer.py +++ b/modelship/infer/base_infer.py @@ -18,6 +18,8 @@ ImageGenerationResponse, ImageVariationRequest, RawSpeechResponse, + ResponseObject, + ResponsesRequest, SpeechRequest, TranscriptionRequest, TranscriptionResponse, @@ -186,6 +188,11 @@ async def create_chat_completion( ) -> ErrorResponse | ChatCompletionResponse | AsyncGenerator[str, None]: return _NOT_SUPPORTED + async def create_response( + self, request: ResponsesRequest, raw_request: RawRequestProxy + ) -> ErrorResponse | ResponseObject | AsyncGenerator[str, None]: + return _NOT_SUPPORTED + async def create_embedding(self, request: EmbeddingRequest, raw_request: RawRequestProxy) -> ErrorResponse: return _NOT_SUPPORTED diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index b3d5d9e..4c6ab86 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -12,6 +12,7 @@ from typing import Any import httpx +from pydantic import ValidationError from modelship.infer.base_infer import BaseInfer, ClientDisconnectedError from modelship.infer.infer_config import LlamaServerConfig, ModelshipModelConfig, ModelUsecase, RawRequestProxy @@ -20,6 +21,7 @@ ParsedChatOutput, UnsupportedContentError, build_from_parsed, + build_responses_items_from_parsed, normalize_chat_messages, ) from modelship.openai.protocol import ( @@ -36,10 +38,19 @@ EmbeddingResponseData, ErrorResponse, FunctionCall, + ResponseObject, + ResponsesRequest, ToolCall, UsageInfo, create_error_response, ) +from modelship.openai.protocol.responses.adapter import ( + UnsupportedResponsesFeatureError, + _status_for, + _usage_from_chat, + build_response_object, + responses_request_to_chat, +) from modelship.preflight import discover_hardware, merge_with_user_overrides, run_preflight from modelship.utils import base_request_id, random_uuid @@ -429,6 +440,55 @@ async def create_chat_completion( async def _send_chat_completion( self, payload: dict[str, Any], request_id: str ) -> ErrorResponse | ChatCompletionResponse: + data = await self._post_chat(payload, request_id) + if isinstance(data, ErrorResponse): + return data + return _project_chat_response(data, model_name=self.model_config.name, request_id=request_id) + + async def create_response( + self, request: ResponsesRequest, raw_request: RawRequestProxy + ) -> ErrorResponse | ResponseObject | AsyncGenerator[str, None]: + if self._client is None: + return await super().create_response(request, raw_request) + if request.stream: + # Native streaming lands in Stage D part 2 (needs a delta-level DTO + # this loader doesn't build yet); fall back to the chat-adapted default. + return await super().create_response(request, raw_request) + + try: + chat_request = responses_request_to_chat(request) + except UnsupportedResponsesFeatureError as e: + return create_error_response(e) + except ValidationError as e: + return _responses_validation_error(e) + + request_id = f"resp-{base_request_id(raw_request)}" + supports_image = bool(self.config.mmproj) + try: + messages = normalize_chat_messages( + chat_request.messages, supports_image=supports_image, supports_audio=False + ) + except UnsupportedContentError as e: + logger.warning("responses request %s rejected: %s", request_id, e) + return create_error_response(e) + + payload = _build_payload(chat_request, messages, model_name=self.model_config.name) + try: + return await self.run_cancellable(self._send_response(payload, request, request_id), raw_request) + except ClientDisconnectedError: + logger.info("responses request %s aborted: client disconnected", request_id) + return create_error_response("Client disconnected") + + async def _send_response( + self, payload: dict[str, Any], request: ResponsesRequest, request_id: str + ) -> ErrorResponse | ResponseObject: + data = await self._post_chat(payload, request_id) + if isinstance(data, ErrorResponse): + return data + return _project_response(data, request, model_name=self.model_config.name) + + async def _post_chat(self, payload: dict[str, Any], request_id: str) -> ErrorResponse | dict: + """POST `/v1/chat/completions` and return the parsed JSON body, or an `ErrorResponse`.""" assert self._client is not None try: resp = await self._client.post("/v1/chat/completions", json=payload) @@ -451,7 +511,7 @@ async def _send_chat_completion( message = error_data.get("message") if isinstance(error_data, dict) else str(error_data) logger.warning("chat request %s failed with inline error: %s", request_id, message) return create_error_response(message or "Unknown error returned from llama-server", status_code=502) - return _project_chat_response(data, model_name=self.model_config.name, request_id=request_id) + return data async def _stream_chat_completion( self, payload: dict[str, Any], request_id: str, raw_request: RawRequestProxy @@ -527,6 +587,13 @@ async def _stream_chat_completion_body(self, payload: dict[str, Any], request_id # --------------------------------------------------------------------------- +def _responses_validation_error(exc: ValidationError) -> ErrorResponse: + # Pydantic ValidationErrors surfaced by responses_request_to_chat (e.g. a + # bad reasoning.effort value) — same 400 shape as every other rejection here. + base = exc.args[0] if exc.args else str(exc) + return create_error_response(message=base, err_type="invalid_request_error") + + def _redact(args: list[str]) -> list[str]: redacted = list(args) for i, arg in enumerate(redacted): @@ -595,10 +662,18 @@ def _project_usage(raw_usage: dict | None) -> UsageInfo: ) -def _project_chat_response(data: dict, *, model_name: str, request_id: str) -> ChatCompletionResponse: - choices = [] - finish_reasons = [] - logprobs_list = [] +def _parsed_choices_from_data( + data: dict, +) -> tuple[list[ParsedChatOutput], list[str | None], list[ChatCompletionLogProbs | None]]: + """Parse every `choices[]` entry in a llama-server response into `ParsedChatOutput`. + + Shared by both `/v1/chat/completions` and `/v1/responses` projection — + llama-server's own parsers already did the reasoning/tool-call parsing, + this just extracts it into modelship's loader-agnostic DTO. + """ + choices: list[ParsedChatOutput] = [] + finish_reasons: list[str | None] = [] + logprobs_list: list[ChatCompletionLogProbs | None] = [] for choice in data.get("choices", []): message = choice.get("message") or {} dto = ParsedChatOutput( @@ -617,6 +692,11 @@ def _project_chat_response(data: dict, *, model_name: str, request_id: str) -> C logger.warning("Failed to validate choice logprobs: %s", e) logprobs_list.append(choice_logprobs) + return choices, finish_reasons, logprobs_list + + +def _project_chat_response(data: dict, *, model_name: str, request_id: str) -> ChatCompletionResponse: + choices, finish_reasons, logprobs_list = _parsed_choices_from_data(data) return build_from_parsed( request_id=data.get("id") or request_id, model_name=model_name, @@ -628,6 +708,19 @@ def _project_chat_response(data: dict, *, model_name: str, request_id: str) -> C ) +def _project_response(data: dict, request: ResponsesRequest, *, model_name: str) -> ResponseObject: + choices, finish_reasons, _logprobs_list = _parsed_choices_from_data(data) + status, incomplete = _status_for(finish_reasons[0] if finish_reasons else None) + return build_response_object( + request, + status=status, + output=build_responses_items_from_parsed(choices[0]) if choices else [], + usage=_usage_from_chat(_project_usage(data.get("usage"))), + incomplete=incomplete, + model=model_name, + ) + + def _project_stream_chunk(data: dict, *, model_name: str, request_id: str) -> ChatCompletionStreamResponse: choices = [] for choice in data.get("choices", []): diff --git a/modelship/infer/model_deployment.py b/modelship/infer/model_deployment.py index fa3382e..fc04cd8 100644 --- a/modelship/infer/model_deployment.py +++ b/modelship/infer/model_deployment.py @@ -29,6 +29,7 @@ ImageEditRequest, ImageGenerationRequest, ImageVariationRequest, + ResponsesRequest, SpeechRequest, TranscriptionRequest, TranslationRequest, @@ -252,6 +253,27 @@ async def generate( GENERATION_DURATION_SECONDS.observe(time.monotonic() - start, tags={"model": self.config.name}) yield result + async def respond( + self, + request: ResponsesRequest, + request_headers: dict[str, str], + disconnect_registry: Any, + request_id: str | None = None, + ): + self._set_request_id(request_id) + proxy = RawRequestProxy(disconnect_registry, request_headers, request_id) + start = time.monotonic() + result = await self.infer.create_response(request, proxy) + if isinstance(result, AsyncGenerator): + try: + async for chunk in result: + yield chunk + finally: + GENERATION_DURATION_SECONDS.observe(time.monotonic() - start, tags={"model": self.config.name}) + else: + GENERATION_DURATION_SECONDS.observe(time.monotonic() - start, tags={"model": self.config.name}) + yield result + async def embed( self, request: EmbeddingRequest, diff --git a/modelship/infer/vllm/vllm_infer.py b/modelship/infer/vllm/vllm_infer.py index ea85a32..a9b6700 100644 --- a/modelship/infer/vllm/vllm_infer.py +++ b/modelship/infer/vllm/vllm_infer.py @@ -5,6 +5,7 @@ from typing import Any, ClassVar, cast from fastapi import UploadFile +from pydantic import ValidationError from starlette.requests import Request from starlette.responses import Response from vllm.config.model import ModelDType @@ -53,7 +54,12 @@ from modelship.infer.vllm.capabilities import VllmCapabilities from modelship.logging import TRACE, get_logger from modelship.metrics import _ENABLED as _METRICS_ENABLED -from modelship.openai.chat_utils import UnsupportedContentError, build_from_parsed, normalize_chat_messages +from modelship.openai.chat_utils import ( + UnsupportedContentError, + build_from_parsed, + build_responses_items_from_parsed, + normalize_chat_messages, +) from modelship.openai.parsers.reasoning import resolve_active_reasoning_parser from modelship.openai.parsers.utils import render_generation_prompt from modelship.openai.protocol import ( @@ -63,6 +69,8 @@ EmbeddingCompletionRequest, EmbeddingRequest, ErrorResponse, + ResponseObject, + ResponsesRequest, TranscriptionRequest, TranscriptionResponse, TranscriptionResponseVerbose, @@ -72,6 +80,13 @@ UsageInfo, create_error_response, ) +from modelship.openai.protocol.responses.adapter import ( + UnsupportedResponsesFeatureError, + _status_for, + _usage_from_chat, + build_response_object, + responses_request_to_chat, +) from modelship.preflight import discover_hardware, merge_with_user_overrides, run_preflight from modelship.utils import base_request_id @@ -98,6 +113,13 @@ def _validation_error(exc: VLLMValidationError) -> ErrorResponse: ) +def _responses_validation_error(exc: ValidationError) -> ErrorResponse: + # Same shape as _validation_error, for pydantic ValidationErrors surfaced by + # responses_request_to_chat (e.g. a bad reasoning.effort value). + base = exc.args[0] if exc.args else str(exc) + return create_error_response(message=base, err_type="invalid_request_error", status_code=HTTPStatus.BAD_REQUEST) + + class VllmInfer(BaseInfer): _vllm_usecases: ClassVar[list[ModelUsecase]] = [ ModelUsecase.generate, @@ -518,6 +540,110 @@ async def _create_chat_completion_no_stream( logprobs=logprobs_list, ) + async def create_response( + self, request: ResponsesRequest, raw_request: RawRequestProxy + ) -> ErrorResponse | ResponseObject | AsyncGenerator[str, None]: + if not hasattr(self, "openai_serving_render"): + return await super().create_response(request, raw_request) + if request.stream: + # Native streaming lands in Stage D part 2 (needs a delta-level DTO + # this loader doesn't build yet); fall back to the chat-adapted default. + return await super().create_response(request, raw_request) + + try: + chat_request = responses_request_to_chat(request) + except UnsupportedResponsesFeatureError as e: + return create_error_response(e) + except ValidationError as e: + return _responses_validation_error(e) + + try: + chat_request.messages = normalize_chat_messages( + chat_request.messages, + supports_image=self._caps.supports_image, + supports_audio=self._caps.supports_audio, + ) + except UnsupportedContentError as exc: + return create_error_response(exc) + vllm_request = engine_ops.build_vllm_request(chat_request, self.model_config.chat_template_kwargs) + + return await self._create_response_no_stream(request, vllm_request, raw_request) + + async def _create_response_no_stream( + self, + request: ResponsesRequest, + vllm_request: VllmChatCompletionRequest, + raw_request: RawRequestProxy, + ) -> ErrorResponse | ResponseObject: + """Non-stream Responses path via `engine_ops`, shaping items directly from + `ParsedChatOutput` instead of round-tripping through a `ChatCompletionResponse`.""" + try: + render_result = await engine_ops.render_and_params(self.openai_serving_render, vllm_request) + except VLLMValidationError as exc: + return _validation_error(exc) + if isinstance(render_result, VllmErrorResponse): + return ErrorResponse.model_validate(render_result.model_dump()) + engine_input, sampling_params = render_result + + tokenizer = self.openai_serving_render.renderer.tokenizer + assert tokenizer is not None, "vllm renderer has no tokenizer (skip_tokenizer_init=True is unsupported here)" + + parser = engine_ops.make_parsers( + self.openai_serving_render, tokenizer, vllm_request, vllm_request.chat_template_kwargs, n=1 + )[0] + prompt_token_ids = engine_ops.extract_prompt_token_ids(self.openai_serving_render, engine_input) + reasoning_ended = engine_ops.derive_reasoning_ended(vllm_request, parser, prompt_token_ids) + + request_id = f"resp-{base_request_id(raw_request)}" + try: + final_res = await self.run_cancellable( + engine_ops.consume_final_output( + self.engine, + engine_input, + sampling_params, + request_id, + reasoning_ended=reasoning_ended, + parser=parser, + chat_template_kwargs=vllm_request.chat_template_kwargs, + ), + raw_request, + ) + except ClientDisconnectedError: + return create_error_response("Client disconnected") + except VLLMValidationError as exc: + return _validation_error(exc) + + choices, finish_reasons, _logprobs_list = engine_ops.build_choices( + final_res, + vllm_request, + parser, + tokenizer, + enable_auto_tools=self._enable_auto_tools, + want_logprobs=False, + num_output_top_logprobs=None, + ) + + if final_res.prompt_token_ids is None: + return create_error_response("vllm returned no prompt_token_ids for a completed request", status_code=502) + prompt_tokens = len(final_res.prompt_token_ids) + completion_tokens = sum(len(output.token_ids) for output in final_res.outputs) + + status, incomplete = _status_for(finish_reasons[0]) + return build_response_object( + request, + status=status, + output=build_responses_items_from_parsed(choices[0]), + usage=_usage_from_chat( + UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + ), + incomplete=incomplete, + model=self.model_config.name, + ) + async def create_embedding( self, request: EmbeddingRequest, raw_request: RawRequestProxy ) -> ErrorResponse | Response: diff --git a/modelship/openai/api.py b/modelship/openai/api.py index ad11270..ccbb10d 100644 --- a/modelship/openai/api.py +++ b/modelship/openai/api.py @@ -53,12 +53,9 @@ TranslationResponse, create_error_response, ) -from modelship.openai.protocol.chat import StreamOptions from modelship.openai.protocol.responses import ( UnsupportedResponsesFeatureError, - responses_from_chat, responses_request_to_chat, - responses_stream_from_chat, ) from modelship.utils import random_uuid @@ -544,7 +541,10 @@ async def create_response(self, request: ResponsesRequest, raw_request: Request) self._set_request_id(req_id) model = request.model or "" try: - chat_request = responses_request_to_chat(request) + # Fail fast on unsupported features before touching Ray. The loader + # re-derives its own ChatCompletionRequest from `request` — this call's + # result is discarded, it's here purely for the early validation. + responses_request_to_chat(request) except UnsupportedResponsesFeatureError as e: return _error_response(create_error_response(e)) except ValidationError as e: @@ -553,13 +553,6 @@ async def create_response(self, request: ResponsesRequest, raw_request: Request) # which is not a ValueError — return a 400 rather than a generic 500. return _error_response(_validation_error_from_cause(e)) - if request.stream: - # Drive the chat pipeline in streaming mode and translate its SSE - # chunks into the Responses event protocol. include_usage so the - # terminal response.completed event carries token counts. - chat_request.stream = True - chat_request.stream_options = StreamOptions(include_usage=True) - handle = self._get_handle(request.model) watcher = RequestWatcher(raw_request, req_id, model=model, endpoint="create_response") headers = dict(raw_request.headers) @@ -575,14 +568,9 @@ async def create_response(self, request: ResponsesRequest, raw_request: Request) # doesn't overload on the stream literal, so narrow it explicitly. response_gen = cast( "DeploymentResponseGenerator[Any]", - handle.generate.options(stream=True).remote(chat_request, headers, watcher.registry, req_id), - ) - adapted = ( - responses_stream_from_chat(response_gen, request) - if request.stream - else responses_from_chat(response_gen, request) + handle.respond.options(stream=True).remote(request, headers, watcher.registry, req_id), ) - return await self._handle_response(adapted, watcher, model, "create_response") + return await self._handle_response(response_gen, watcher, model, "create_response") @app.post("/v1/embeddings") async def create_embeddings(self, request: EmbeddingRequest, raw_request: Request): diff --git a/modelship/openai/chat_utils.py b/modelship/openai/chat_utils.py index e9fa61b..efa8633 100644 --- a/modelship/openai/chat_utils.py +++ b/modelship/openai/chat_utils.py @@ -12,6 +12,14 @@ ToolCall, UsageInfo, ) +from modelship.openai.protocol.responses.schemas import ( + ResponseFunctionToolCall, + ResponseOutputItem, + ResponseOutputMessage, + ResponseOutputText, + ResponseReasoningItem, + ResponseReasoningSummary, +) logger = get_logger("openai.chat_utils") @@ -270,3 +278,26 @@ def build_from_parsed( usage=usage, created=created, ) + + +def build_responses_items_from_parsed(parsed: ParsedChatOutput) -> list[ResponseOutputItem]: + """Shape one parsed choice into Responses ``output[]`` items. + + Sibling to `build_from_parsed`: same DTO in, Responses items out instead + of a `ChatCompletionResponse`. Order matches OpenAI's own: reasoning + first, then the assistant message, then one `function_call` per tool call. + """ + output: list[ResponseOutputItem] = [] + if parsed.reasoning: + output.append(ResponseReasoningItem(summary=[ResponseReasoningSummary(text=parsed.reasoning)])) + if parsed.content: + output.append(ResponseOutputMessage(content=[ResponseOutputText(text=parsed.content)])) + for call in parsed.tool_calls: + output.append( + ResponseFunctionToolCall( + call_id=call.id, + name=call.function.name, + arguments=call.function.arguments, + ) + ) + return output diff --git a/tests/test_chat_utils.py b/tests/test_chat_utils.py index 05a26b2..9e9a815 100644 --- a/tests/test_chat_utils.py +++ b/tests/test_chat_utils.py @@ -3,6 +3,7 @@ from modelship.openai.chat_utils import ( ParsedChatOutput, build_from_parsed, + build_responses_items_from_parsed, normalize_chat_messages, ) from modelship.openai.protocol import FunctionCall, ToolCall, UsageInfo @@ -174,3 +175,26 @@ def test_build_from_parsed_multi_choice_and_dto(): ) assert res3.choices[0].finish_reason == "length" assert res3.choices[1].finish_reason == "stop" + + +def test_build_responses_items_from_parsed_orders_reasoning_message_tools(): + parsed = ParsedChatOutput( + content="Hello!", + reasoning="Thinking...", + tool_calls=[ToolCall(id="call_1", type="function", function=FunctionCall(name="get_weather", arguments="{}"))], + ) + + output = build_responses_items_from_parsed(parsed) + + assert [item.type for item in output] == ["reasoning", "message", "function_call"] + assert output[0].summary[0].text == "Thinking..." + assert output[1].content[0].text == "Hello!" + assert output[2].call_id == "call_1" + assert output[2].name == "get_weather" + assert output[2].arguments == "{}" + + +def test_build_responses_items_from_parsed_skips_empty_fields(): + parsed = ParsedChatOutput(content=None, reasoning=None, tool_calls=[]) + + assert build_responses_items_from_parsed(parsed) == [] diff --git a/tests/test_responses_api.py b/tests/test_responses_api.py index fd49427..0d84b80 100644 --- a/tests/test_responses_api.py +++ b/tests/test_responses_api.py @@ -1,4 +1,11 @@ -"""Route-level tests for /v1/responses (Phase A, non-streaming).""" +"""Route-level tests for /v1/responses. + +Since Stage D, the route does no chat<->Responses translation itself (that +now lives on `BaseInfer.create_response`, either the default Phase A shim or +a loader's native override) — it only fails fast on unsupported features +before touching Ray, then hands the original `ResponsesRequest` straight to +`handle.respond` and threads the result through the shared `_handle_response`. +""" import json from unittest.mock import AsyncMock, MagicMock, patch @@ -6,17 +13,8 @@ import pytest from modelship.openai.api import ModelshipAPI -from modelship.openai.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, - ChatCompletionResponseChoice, - ChatCompletionResponseStreamChoice, - ChatCompletionStreamResponse, - ChatMessage, - DeltaMessage, - ResponsesRequest, - UsageInfo, -) +from modelship.openai.protocol import ResponsesRequest +from modelship.openai.protocol.responses import ResponseObject, ResponseOutputMessage, ResponseOutputText, ResponseUsage _ModelshipAPI = ModelshipAPI.func_or_class @@ -45,9 +43,9 @@ def _raw_request(): class TestResponsesRoute: @pytest.mark.asyncio - async def test_adapts_request_and_reuses_handle_response(self, api): + async def test_dispatches_original_request_to_respond(self, api): handle = MagicMock() - remote = handle.generate.options.return_value.remote + remote = handle.respond.options.return_value.remote api.models = {"m": {"m-a1b2c": handle}} api._round_robin = {"m": 0} @@ -57,38 +55,25 @@ async def test_adapts_request_and_reuses_handle_response(self, api): result = await api.create_response(request, _raw_request()) assert result == "OK" - # The chat request handed to the actor must be the translated shape. - chat_request = remote.call_args.args[0] - assert isinstance(chat_request, ChatCompletionRequest) - assert chat_request.messages == [ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"}, - ] - assert chat_request.stream is False + # No gateway-side translation: the original ResponsesRequest is handed + # straight to the deployment, which owns chat<->Responses shaping now. + assert remote.call_args.args[0] is request hr.assert_awaited_once() # endpoint label flows through to _handle_response for metrics. assert hr.call_args.args[3] == "create_response" @pytest.mark.asyncio - async def test_stream_true_drives_streaming_translation(self, api): - # stream=True sets stream + include_usage on the chat request and routes - # the chat SSE chunks through the Responses event translator, yielding a - # text/event-stream of Responses events. + async def test_stream_true_dispatches_and_returns_event_stream(self, api): + # Streaming has no gateway-side translation anymore: the deployment + # (BaseInfer.create_response) is responsible for producing Responses SSE + # events directly. The route only needs to thread them through. handle = MagicMock() async def gen(): - chunk = ChatCompletionStreamResponse( - model="m", - choices=[ - ChatCompletionResponseStreamChoice( - index=0, delta=DeltaMessage(content="hello!"), finish_reason="stop" - ) - ], - ) - yield f"data: {json.dumps(chunk.model_dump(mode='json'))}\n\n" - yield "data: [DONE]\n\n" + yield 'event: response.created\ndata: {"type": "response.created"}\n\n' + yield 'event: response.completed\ndata: {"type": "response.completed"}\n\n' - handle.generate.options.return_value.remote.return_value = gen() + handle.respond.options.return_value.remote.return_value = gen() api.models = {"m": {"m-a1b2c": handle}} api._round_robin = {"m": 0} @@ -96,14 +81,10 @@ async def gen(): result = await api.create_response(request, _raw_request()) assert result.media_type == "text/event-stream" - # The chat request handed to the loader must be streaming with usage on. - chat_request = handle.generate.options.return_value.remote.call_args.args[0] - assert chat_request.stream is True - assert chat_request.stream_options is not None and chat_request.stream_options.include_usage is True + assert handle.respond.options.return_value.remote.call_args.args[0] is request body = "".join([chunk async for chunk in result.body_iterator]) assert "event: response.created" in body - assert "event: response.output_text.delta" in body assert "event: response.completed" in body @pytest.mark.asyncio @@ -126,25 +107,20 @@ async def test_invalid_param_returns_400_not_500(self, api): assert body["error"]["type"] == "invalid_request_error" @pytest.mark.asyncio - async def test_end_to_end_adaptation_through_handle_response(self, api): - # Drive the real _handle_response with a mock generator yielding a chat - # response, and assert the body comes back in Responses shape. + async def test_end_to_end_through_handle_response(self, api): + # Drive the real _handle_response with a mock generator yielding an + # already-built ResponseObject (what the deployment now returns + # directly, with no gateway-side chat->Responses translation left). handle = MagicMock() async def gen(): - yield ChatCompletionResponse( + yield ResponseObject( model="m", - choices=[ - ChatCompletionResponseChoice( - index=0, - message=ChatMessage(role="assistant", content="hello!"), - finish_reason="stop", - ) - ], - usage=UsageInfo(prompt_tokens=1, completion_tokens=2, total_tokens=3), + output=[ResponseOutputMessage(content=[ResponseOutputText(text="hello!")])], + usage=ResponseUsage(input_tokens=1, output_tokens=2, total_tokens=3), ) - handle.generate.options.return_value.remote.return_value = gen() + handle.respond.options.return_value.remote.return_value = gen() api.models = {"m": {"m-a1b2c": handle}} api._round_robin = {"m": 0} From 7e96a6ece6f33d2a44bc1fb26f0bc137cb8191a9 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Sat, 4 Jul 2026 07:51:15 +0000 Subject: [PATCH 31/35] feat: add native streaming support to /v1/responses for vllm and llama_server Feeds ResponsesStreamTranslator directly from each loader's typed stream chunks (engine_ops.stream_chat_completion for vllm, a newly-shared _raw_stream_chunks for llama_server) instead of going through the chat SSE text round trip, with a new translator.fail() path for mid-stream errors. Also fixes a latent bug where responses_request_to_chat's hardcoded stream=False was never flipped back for streaming requests, so llama_server would have answered a streaming call with a single JSON object. --- .../infer/llama_server/llama_server_infer.py | 151 +++++++++++++----- modelship/infer/vllm/vllm_infer.py | 79 ++++++++- .../openai/protocol/responses/adapter.py | 5 + .../openai/protocol/responses/streaming.py | 43 +++-- tests/test_llama_server_infer.py | 143 ++++++++++++++++- tests/test_responses_streaming.py | 60 ++++++- tests/test_vllm_responses.py | 147 +++++++++++++++++ 7 files changed, 561 insertions(+), 67 deletions(-) create mode 100644 tests/test_vllm_responses.py diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 4c6ab86..33e7c78 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -51,6 +51,7 @@ build_response_object, responses_request_to_chat, ) +from modelship.openai.protocol.responses.streaming import ResponsesStreamTranslator from modelship.preflight import discover_hardware, merge_with_user_overrides, run_preflight from modelship.utils import base_request_id, random_uuid @@ -450,10 +451,6 @@ async def create_response( ) -> ErrorResponse | ResponseObject | AsyncGenerator[str, None]: if self._client is None: return await super().create_response(request, raw_request) - if request.stream: - # Native streaming lands in Stage D part 2 (needs a delta-level DTO - # this loader doesn't build yet); fall back to the chat-adapted default. - return await super().create_response(request, raw_request) try: chat_request = responses_request_to_chat(request) @@ -472,7 +469,12 @@ async def create_response( logger.warning("responses request %s rejected: %s", request_id, e) return create_error_response(e) + chat_request.stream = bool(request.stream) payload = _build_payload(chat_request, messages, model_name=self.model_config.name) + + if request.stream: + return self._stream_response(payload, request, request_id, raw_request) + try: return await self.run_cancellable(self._send_response(payload, request, request_id), raw_request) except ClientDisconnectedError: @@ -535,44 +537,17 @@ async def _stream_chat_completion_body(self, payload: dict[str, Any], request_id assert self._client is not None buffered: list[str] = [] try: - async with self._client.stream("POST", "/v1/chat/completions", json=payload) as resp: - try: - resp.raise_for_status() - except httpx.HTTPStatusError as e: - await e.response.aread() - detail = _extract_error_detail(e.response) - logger.warning("chat request %s failed: %s", request_id, detail) - yield _encode_error(detail) - yield "data: [DONE]\n\n" - return - - async for line in resp.aiter_lines(): - if not line.startswith("data:"): - continue - data_str = line[len("data:") :].strip() - if data_str == "[DONE]": - break - try: - data = json.loads(data_str) - except json.JSONDecodeError: - logger.warning("chat request %s: unparseable stream chunk: %r", request_id, data_str) - continue - if not isinstance(data, dict): - logger.warning("chat request %s: skipping non-object stream chunk: %r", request_id, data) - continue - if "error" in data: - error_data = data["error"] or {} - message = error_data.get("message") if isinstance(error_data, dict) else str(error_data) - logger.warning("chat request %s failed mid-stream with error: %s", request_id, message) - yield _encode_error(message or "Unknown mid-stream error from llama-server") - yield "data: [DONE]\n\n" - return - chunk = _project_stream_chunk(data, model_name=self.model_config.name, request_id=request_id) - for choice in chunk.choices: - if choice.delta.content: - buffered.append(choice.delta.content) - yield _encode_chunk(chunk) - yield "data: [DONE]\n\n" + async for chunk in _raw_stream_chunks( + self._client, payload, model_name=self.model_config.name, request_id=request_id + ): + for choice in chunk.choices: + if choice.delta.content: + buffered.append(choice.delta.content) + yield _encode_chunk(chunk) + yield "data: [DONE]\n\n" + except _LlamaServerStreamError as e: + yield _encode_error(str(e)) + yield "data: [DONE]\n\n" except httpx.HTTPError as e: logger.warning("chat request %s failed mid-stream: %s", request_id, e) yield _encode_error(f"llama-server request failed: {e}") @@ -580,6 +555,47 @@ async def _stream_chat_completion_body(self, payload: dict[str, Any], request_id finally: logger.log(TRACE, "chat response %s (stream): %r", request_id, "".join(buffered)) + async def _stream_response( + self, payload: dict[str, Any], request: ResponsesRequest, request_id: str, raw_request: RawRequestProxy + ) -> AsyncGenerator[str, None]: + """Race the llama-server SSE stream against the client's disconnect signal — + the Responses counterpart of `_stream_chat_completion`.""" + stream = self._stream_response_body(payload, request, request_id) + try: + async for chunk in self.run_cancellable_stream(stream, raw_request): + yield chunk + except ClientDisconnectedError: + logger.info("responses request %s aborted: client disconnected", request_id) + return + + async def _stream_response_body( + self, payload: dict[str, Any], request: ResponsesRequest, request_id: str + ) -> AsyncGenerator[str, None]: + """Native streaming Responses path: feeds `ResponsesStreamTranslator` directly + from `_raw_stream_chunks`'s typed chunks, same source `_stream_chat_completion_body` + uses — no chat SSE text round trip like the (unused-by-this-loader) `BaseInfer` default.""" + assert self._client is not None + translator = ResponsesStreamTranslator(request) + for event in translator.start(): + yield event + try: + async for chunk in _raw_stream_chunks( + self._client, payload, model_name=self.model_config.name, request_id=request_id + ): + for event in translator.process(chunk): + yield event + except _LlamaServerStreamError as e: + for event in translator.fail(str(e)): + yield event + return + except httpx.HTTPError as e: + logger.warning("responses request %s failed mid-stream: %s", request_id, e) + for event in translator.fail(f"llama-server request failed: {e}"): + yield event + return + for event in translator.finish(): + yield event + # --------------------------------------------------------------------------- # Request / response projection — never relay llama-server's JSON verbatim, @@ -765,6 +781,57 @@ def _project_stream_chunk(data: dict, *, model_name: str, request_id: str) -> Ch ) +class _LlamaServerStreamError(Exception): + """A llama-server streaming request failed after the response started. + + Raised generically so both the chat and Responses streaming wrappers can + encode it in their own wire format (chat SSE error chunk vs a Responses + `response.failed` event). + """ + + +async def _raw_stream_chunks( + client: httpx.AsyncClient, payload: dict[str, Any], *, model_name: str, request_id: str +) -> AsyncGenerator[ChatCompletionStreamResponse, None]: + """Open the llama-server SSE stream and yield typed chunks. + + Shared by `_stream_chat_completion_body` (chat) and `_stream_response_body` + (Responses) — both need the same parsed chunks, just encoded differently. + Raises `_LlamaServerStreamError` for a failure after the connection opens + (HTTP error status, inline `error` field); `httpx.HTTPError` propagates + uncaught for transport-level failures, same as before this was factored out. + """ + async with client.stream("POST", "/v1/chat/completions", json=payload) as resp: + try: + resp.raise_for_status() + except httpx.HTTPStatusError as e: + await e.response.aread() + detail = _extract_error_detail(e.response) + logger.warning("chat request %s failed: %s", request_id, detail) + raise _LlamaServerStreamError(detail) from e + + async for line in resp.aiter_lines(): + if not line.startswith("data:"): + continue + data_str = line[len("data:") :].strip() + if data_str == "[DONE]": + return + try: + data = json.loads(data_str) + except json.JSONDecodeError: + logger.warning("chat request %s: unparseable stream chunk: %r", request_id, data_str) + continue + if not isinstance(data, dict): + logger.warning("chat request %s: skipping non-object stream chunk: %r", request_id, data) + continue + if "error" in data: + error_data = data["error"] or {} + message = error_data.get("message") if isinstance(error_data, dict) else str(error_data) + logger.warning("chat request %s failed mid-stream with error: %s", request_id, message) + raise _LlamaServerStreamError(message or "Unknown mid-stream error from llama-server") + yield _project_stream_chunk(data, model_name=model_name, request_id=request_id) + + def _project_embedding_response(data: dict, *, model_name: str) -> EmbeddingResponse: raw_data_list = data.get("data", []) projected_data = [] diff --git a/modelship/infer/vllm/vllm_infer.py b/modelship/infer/vllm/vllm_infer.py index a9b6700..51bb01e 100644 --- a/modelship/infer/vllm/vllm_infer.py +++ b/modelship/infer/vllm/vllm_infer.py @@ -45,6 +45,8 @@ ) from vllm.entrypoints.speech_to_text.translation.serving import OpenAIServingTranslation from vllm.exceptions import VLLMValidationError +from vllm.inputs import EngineInput +from vllm.sampling_params import SamplingParams from vllm.usage.usage_lib import UsageContext from vllm.v1.engine.async_llm import AsyncLLM @@ -87,6 +89,7 @@ build_response_object, responses_request_to_chat, ) +from modelship.openai.protocol.responses.streaming import ResponsesStreamTranslator from modelship.preflight import discover_hardware, merge_with_user_overrides, run_preflight from modelship.utils import base_request_id @@ -545,10 +548,6 @@ async def create_response( ) -> ErrorResponse | ResponseObject | AsyncGenerator[str, None]: if not hasattr(self, "openai_serving_render"): return await super().create_response(request, raw_request) - if request.stream: - # Native streaming lands in Stage D part 2 (needs a delta-level DTO - # this loader doesn't build yet); fall back to the chat-adapted default. - return await super().create_response(request, raw_request) try: chat_request = responses_request_to_chat(request) @@ -565,10 +564,31 @@ async def create_response( ) except UnsupportedContentError as exc: return create_error_response(exc) + chat_request.stream = request.stream or False vllm_request = engine_ops.build_vllm_request(chat_request, self.model_config.chat_template_kwargs) + if request.stream: + return await self._create_response_stream_or_error(request, vllm_request, raw_request) return await self._create_response_no_stream(request, vllm_request, raw_request) + async def _create_response_stream_or_error( + self, + request: ResponsesRequest, + vllm_request: VllmChatCompletionRequest, + raw_request: RawRequestProxy, + ) -> ErrorResponse | AsyncGenerator[str, None]: + """Render + derive sampling params before committing to a Responses event + stream, so a pre-generation failure (e.g. context overflow) can still be + returned as a plain `ErrorResponse` instead of a mid-stream `response.failed`.""" + try: + render_result = await engine_ops.render_and_params(self.openai_serving_render, vllm_request) + except VLLMValidationError as exc: + return _validation_error(exc) + if isinstance(render_result, VllmErrorResponse): + return ErrorResponse.model_validate(render_result.model_dump()) + engine_input, sampling_params = render_result + return self._create_response_stream(request, vllm_request, engine_input, sampling_params, raw_request) + async def _create_response_no_stream( self, request: ResponsesRequest, @@ -644,6 +664,57 @@ async def _create_response_no_stream( model=self.model_config.name, ) + async def _create_response_stream( + self, + request: ResponsesRequest, + vllm_request: VllmChatCompletionRequest, + engine_input: EngineInput, + sampling_params: SamplingParams, + raw_request: RawRequestProxy, + ) -> AsyncGenerator[str, None]: + """Native streaming Responses path: feeds `ResponsesStreamTranslator` directly + from `engine_ops.stream_chat_completion`'s typed chunks — no chat SSE text + round trip like the (unused-by-this-loader) `BaseInfer` default.""" + request_id = f"resp-{base_request_id(raw_request)}" + tokenizer = self.openai_serving_render.renderer.tokenizer + assert tokenizer is not None, "vllm renderer has no tokenizer (skip_tokenizer_init=True is unsupported here)" + + stream = engine_ops.stream_chat_completion( + self.engine, + self.openai_serving_render, + vllm_request, + engine_input, + sampling_params, + request_id, + self.model_config.name, + tokenizer, + enable_auto_tools=self._enable_auto_tools, + want_logprobs=False, + num_output_top_logprobs=None, + ) + translator = ResponsesStreamTranslator(request) + for event in translator.start(): + yield event + try: + async for chunk in self.run_cancellable_stream(stream, raw_request): + for event in translator.process(chunk): + yield event + except ClientDisconnectedError: + logger.info("responses request %s aborted: client disconnected", request_id) + return + except VLLMValidationError as exc: + base = exc.args[0] if exc.args else str(exc) + for event in translator.fail(str(base)): + yield event + return + except Exception: + logger.exception("responses request %s failed mid-stream", request_id) + for event in translator.fail("Internal error during generation"): + yield event + return + for event in translator.finish(): + yield event + async def create_embedding( self, request: EmbeddingRequest, raw_request: RawRequestProxy ) -> ErrorResponse | Response: diff --git a/modelship/openai/protocol/responses/adapter.py b/modelship/openai/protocol/responses/adapter.py index 2037191..d03fbe6 100644 --- a/modelship/openai/protocol/responses/adapter.py +++ b/modelship/openai/protocol/responses/adapter.py @@ -289,6 +289,7 @@ def build_response_object( model: str | None = None, response_id: str | None = None, created_at: int | None = None, + error: Any | None = None, ) -> ResponseObject: """Build a ``ResponseObject``, echoing the request settings OpenAI returns. @@ -296,6 +297,8 @@ def build_response_object( ``response.created`` / ``response.completed`` envelopes and the non-streaming body carry an identical shape. ``response_id`` / ``created_at`` let the streaming translator keep one stable id across all of its events. + ``error`` is set only for a ``status="failed"`` terminal event (see + ``ResponsesStreamTranslator.fail``); every other caller leaves it ``None``. """ kwargs: dict[str, Any] = { "model": model or request.model or "", @@ -319,6 +322,8 @@ def build_response_object( kwargs["id"] = response_id if created_at is not None: kwargs["created_at"] = created_at + if error is not None: + kwargs["error"] = error return ResponseObject(**kwargs) diff --git a/modelship/openai/protocol/responses/streaming.py b/modelship/openai/protocol/responses/streaming.py index fa4c189..a1a59a0 100644 --- a/modelship/openai/protocol/responses/streaming.py +++ b/modelship/openai/protocol/responses/streaming.py @@ -119,7 +119,9 @@ def _take_oi(self) -> int: self._next_oi += 1 return oi - def _envelope(self, event_type: str, status: str, output: list[Any], usage, incomplete) -> str: + def _envelope( + self, event_type: str, status: str, output: list[Any], usage, incomplete, error: Any | None = None + ) -> str: response = build_response_object( self.request, status=status, @@ -129,11 +131,28 @@ def _envelope(self, event_type: str, status: str, output: list[Any], usage, inco model=self.model, response_id=self.response_id, created_at=self.created_at, + error=error, ) # Pin created_at after the first build so every envelope is identical. self.created_at = response.created_at return self._event(event_type, {"response": response.model_dump(mode="json")}) + def _close_all(self) -> Iterator[str]: + # Close every open item, in output-index (first-seen) order. + yield from self._close_reasoning() + yield from self._close_message() + yield from self._close_tools() + + def _collect_output(self) -> list[Any]: + output: list[Any] = [] + if self._reasoning is not None: + output.append(self._reasoning) + if self._message is not None: + output.append(self._message) + for idx in sorted(self._tools): + output.append(self._tools[idx]) + return output + # -- lifecycle ---------------------------------------------------------- def start(self) -> Iterator[str]: @@ -157,24 +176,22 @@ def process(self, chunk: ChatCompletionStreamResponse) -> Iterator[str]: self.finish_reason = choice.finish_reason def finish(self) -> Iterator[str]: - # Close every open item, in output-index (first-seen) order. - yield from self._close_reasoning() - yield from self._close_message() - yield from self._close_tools() - - output: list[Any] = [] - if self._reasoning is not None: - output.append(self._reasoning) - if self._message is not None: - output.append(self._message) - for idx in sorted(self._tools): - output.append(self._tools[idx]) + yield from self._close_all() + output = self._collect_output() status, incomplete = _status_for(self.finish_reason) usage = _usage_from_chat(self.usage) if self.usage is not None else None terminal = "response.incomplete" if status == "incomplete" else "response.completed" yield self._envelope(terminal, status, output, usage, incomplete) + def fail(self, message: str) -> Iterator[str]: + """Terminal ``response.failed`` event for a mid-stream error (a loader + exception, not a normal completion). Still closes any already-open item + brackets so a client sees whatever partial content was generated.""" + yield from self._close_all() + output = self._collect_output() + yield self._envelope("response.failed", "failed", output, None, None, error={"message": message}) + # -- reasoning channel -------------------------------------------------- def _on_reasoning(self, text: str) -> Iterator[str]: diff --git a/tests/test_llama_server_infer.py b/tests/test_llama_server_infer.py index 7c8195e..08977c9 100644 --- a/tests/test_llama_server_infer.py +++ b/tests/test_llama_server_infer.py @@ -22,7 +22,13 @@ RawRequestProxy, ) from modelship.infer.llama_server.llama_server_infer import LlamaServerInfer -from modelship.openai.protocol import ChatCompletionRequest, EmbeddingRequest, EmbeddingResponse, ErrorResponse +from modelship.openai.protocol import ( + ChatCompletionRequest, + EmbeddingRequest, + EmbeddingResponse, + ErrorResponse, + ResponsesRequest, +) # --------------------------------------------------------------------------- # Fake llama-server executables (plain scripts; no real binary needed in CI) @@ -678,4 +684,137 @@ def handler(request: httpx.Request) -> httpx.Response: # First 2 are normal choices, then the error chunk, then [DONE] assert len(chunks) == 4 assert "inline mid-stream error detail" in chunks[2] - assert chunks[-1] == "data: [DONE]\n\n" + + +def _responses_request(**kwargs) -> ResponsesRequest: + return ResponsesRequest(model="test-model", input="hi", **kwargs) + + +class TestResponsesProjection: + """Stage D: LlamaServerInfer.create_response — the native Responses path + shaping items directly from llama-server's own parsed reasoning/tool_calls, + same as TestNonStreamingProjection/TestStreamingProjection do for chat.""" + + @pytest.mark.asyncio + async def test_non_stream_maps_reasoning_and_tools_to_responses_items(self): + raw = { + "id": "chatcmpl-xyz", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "reasoning_content": "thinking...", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14}, + "timings": {"predicted_ms": 42.0}, # llama.cpp extension — must not leak + } + + infer = _infer_with_client(lambda request: httpx.Response(200, json=raw)) + result = await infer.create_response(_responses_request(), RawRequestProxy(None, {})) + + assert not isinstance(result, ErrorResponse) + assert result.object == "response" + assert [item.type for item in result.output] == ["reasoning", "function_call"] + assert result.output[0].summary[0].text == "thinking..." + assert result.output[1].name == "get_weather" + assert result.usage.input_tokens == 10 + + @pytest.mark.asyncio + async def test_previous_response_id_rejected_before_any_http_call(self): + called = False + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal called + called = True + return httpx.Response(200, json={"choices": [], "usage": {}}) + + infer = _infer_with_client(handler) + result = await infer.create_response( + _responses_request(previous_response_id="resp_1"), RawRequestProxy(None, {}) + ) + + assert isinstance(result, ErrorResponse) + assert result._http_status == 400 + assert called is False + + @pytest.mark.asyncio + async def test_no_client_falls_back_to_not_supported(self): + infer = _infer_with_client(lambda request: httpx.Response(200)) + infer._client = None + result = await infer.create_response(_responses_request(), RawRequestProxy(None, {})) + assert isinstance(result, ErrorResponse) + + @pytest.mark.asyncio + async def test_stream_events_sequence_and_forces_stream_true_on_wire(self): + captured = {} + sse_body = ( + 'data: {"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n' + 'data: {"choices":[{"index":0,"delta":{"content":"Hel"},"finish_reason":null}]}\n\n' + 'data: {"choices":[{"index":0,"delta":{"content":"lo"},"finish_reason":null}]}\n\n' + 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],' + '"usage":{"prompt_tokens":5,"completion_tokens":2,"total_tokens":7}}\n\n' + "data: [DONE]\n\n" + ) + + def handler(request: httpx.Request) -> httpx.Response: + import json + + captured["payload"] = json.loads(request.content) + return httpx.Response(200, content=sse_body.encode(), headers={"content-type": "text/event-stream"}) + + infer = _infer_with_client(handler) + result = await infer.create_response(_responses_request(stream=True), RawRequestProxy(None, {})) + body = "".join([chunk async for chunk in result]) + + # responses_request_to_chat hardcodes stream=False; create_response must + # flip it back to True before this reaches the wire, or llama-server + # would answer with a single JSON object instead of an SSE stream. + assert captured["payload"]["stream"] is True + assert "event: response.created" in body + assert "event: response.output_text.delta" in body + assert "event: response.completed" in body + + @pytest.mark.asyncio + async def test_stream_mid_stream_error_emits_failed_event(self): + sse_body = ( + 'data: {"choices":[{"index":0,"delta":{"content":"Hel"},"finish_reason":null}]}\n\n' + 'data: {"error": {"message": "inline mid-stream error detail"}}\n\n' + ) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=sse_body.encode(), headers={"content-type": "text/event-stream"}) + + infer = _infer_with_client(handler) + result = await infer.create_response(_responses_request(stream=True), RawRequestProxy(None, {})) + body = "".join([chunk async for chunk in result]) + + assert "event: response.failed" in body + assert "inline mid-stream error detail" in body + assert "event: response.completed" not in body + + @pytest.mark.asyncio + async def test_stream_disconnect_ends_without_hanging(self): + async def handler(request: httpx.Request) -> httpx.Response: + await asyncio.sleep(10) + return httpx.Response(200, content=b"data: [DONE]\n\n") # pragma: no cover - never reached + + infer = _infer_with_client(handler) + result = await infer.create_response( + _responses_request(stream=True), _DisconnectingRawRequest(disconnect_after=0.05) + ) + + # translator.start()'s two events are synchronous and yield immediately; + # the disconnect only bites once the generator would block on the (never + # answered) HTTP call — no response.completed/failed ever follows. + body = "".join([chunk async for chunk in result]) + assert "event: response.created" in body + assert "event: response.completed" not in body + assert "event: response.failed" not in body diff --git a/tests/test_responses_streaming.py b/tests/test_responses_streaming.py index 54cd794..2a17b0d 100644 --- a/tests/test_responses_streaming.py +++ b/tests/test_responses_streaming.py @@ -40,6 +40,15 @@ def _chunk(delta: DeltaMessage | None = None, *, finish_reason=None, usage=None) return ChatCompletionStreamResponse(model="m", choices=choices, usage=usage) +def _decode(raw: list[str]) -> list[dict]: + out = [] + for line in raw: + # each is "event: \ndata: {json}\n\n" + data_line = next(ln for ln in line.splitlines() if ln.startswith("data:")) + out.append(json.loads(data_line[len("data:") :].strip())) + return out + + def _events(translator: ResponsesStreamTranslator, chunks) -> list[dict]: """Run start -> process(chunks) -> finish and return parsed event payloads.""" raw: list[str] = [] @@ -47,12 +56,17 @@ def _events(translator: ResponsesStreamTranslator, chunks) -> list[dict]: for c in chunks: raw.extend(translator.process(c)) raw.extend(translator.finish()) - out = [] - for line in raw: - # each is "event: \ndata: {json}\n\n" - data_line = next(ln for ln in line.splitlines() if ln.startswith("data:")) - out.append(json.loads(data_line[len("data:") :].strip())) - return out + return _decode(raw) + + +def _events_then_fail(translator: ResponsesStreamTranslator, chunks, message: str) -> list[dict]: + """Run start -> process(chunks) -> fail(message) and return parsed event payloads.""" + raw: list[str] = [] + raw.extend(translator.start()) + for c in chunks: + raw.extend(translator.process(c)) + raw.extend(translator.fail(message)) + return _decode(raw) def _types(events: list[dict]) -> list[str]: @@ -251,6 +265,40 @@ def test_content_filter_finish_reason_is_incomplete(self): assert events[-1]["response"]["incomplete_details"] == {"reason": "content_filter"} +class TestFailedStream: + def test_fail_before_any_output_emits_bare_failed_event(self): + translator = ResponsesStreamTranslator(_req()) + events = _events_then_fail(translator, [], "boom") + assert _types(events) == ["response.created", "response.in_progress", "response.failed"] + failed = events[-1]["response"] + assert failed["status"] == "failed" + assert failed["error"] == {"message": "boom"} + assert failed["output"] == [] + + def test_fail_mid_message_closes_the_open_item_with_partial_text(self): + translator = ResponsesStreamTranslator(_req()) + events = _events_then_fail(translator, [_chunk(DeltaMessage(content="partial"))], "boom") + assert _types(events)[-4:] == [ + "response.output_text.done", + "response.content_part.done", + "response.output_item.done", + "response.failed", + ] + failed = events[-1]["response"] + assert failed["status"] == "failed" + assert failed["output"][0]["type"] == "message" + assert failed["output"][0]["content"][0]["text"] == "partial" + + def test_fail_mid_tool_call_closes_it_with_partial_arguments(self): + translator = ResponsesStreamTranslator(_req()) + tc = DeltaToolCall(index=0, id="call_1", function=DeltaFunctionCall(name="get_weather", arguments='{"lo')) + events = _events_then_fail(translator, [_chunk(DeltaMessage(tool_calls=[tc]))], "boom") + failed = events[-1]["response"] + assert failed["output"][0]["type"] == "function_call" + assert failed["output"][0]["arguments"] == '{"lo' + assert failed["output"][0]["status"] == "completed" + + class TestStreamWrapper: @pytest.mark.asyncio async def test_wraps_chat_sse_strings_into_events(self): diff --git a/tests/test_vllm_responses.py b/tests/test_vllm_responses.py new file mode 100644 index 0000000..2bee46d --- /dev/null +++ b/tests/test_vllm_responses.py @@ -0,0 +1,147 @@ +"""Unit tests for VllmInfer.create_response's own wiring: gating, request +translation error mapping, and pre-generation-vs-mid-stream error handling. + +Full DTO-shaping correctness (render_and_params -> build_choices/stream_chat_completion +against a real tokenizer) is covered by test_vllm_engine_ops.py's GPU-free +real-pipeline tests and by a manual end-to-end run against a real engine (the +parser-migration roadmap's standing convention for each phase) — this file +mocks `engine_ops` out entirely to isolate VllmInfer.create_response's own logic. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from vllm.exceptions import VLLMValidationError + +from modelship.infer.vllm.capabilities import VllmCapabilities +from modelship.infer.vllm.vllm_infer import VllmInfer +from modelship.openai.protocol import ( + ChatCompletionResponseStreamChoice, + ChatCompletionStreamResponse, + DeltaMessage, + ErrorResponse, + ResponsesRequest, +) + + +class _FakeRawRequest: + request_id = "req-1" + + async def is_disconnected(self) -> bool: + return False + + +def _make_infer() -> VllmInfer: + """Build a VllmInfer with __init__/start bypassed; only the fields + create_response reads are populated (mirrors test_vllm_vision.py).""" + infer = VllmInfer.__new__(VllmInfer) + infer._caps = VllmCapabilities(supports_image=False, supports_audio=False) + infer.model_config = MagicMock() + infer.model_config.name = "m" + infer.model_config.chat_template_kwargs = {} + infer.openai_serving_render = MagicMock() + infer.engine = MagicMock() + infer._enable_auto_tools = True + return infer + + +@pytest.mark.asyncio +async def test_no_render_pipeline_falls_back_to_not_supported(): + infer = VllmInfer.__new__(VllmInfer) # no openai_serving_render attribute at all + request = ResponsesRequest(model="m", input="hi") + + result = await infer.create_response(request, raw_request=_FakeRawRequest()) + + assert isinstance(result, ErrorResponse) + assert result._http_status == 404 + + +@pytest.mark.asyncio +async def test_previous_response_id_rejected_before_engine_ops_touched(): + infer = _make_infer() + request = ResponsesRequest(model="m", input="hi", previous_response_id="resp_1") + + with patch("modelship.infer.vllm.vllm_infer.engine_ops") as mock_ops: + result = await infer.create_response(request, raw_request=_FakeRawRequest()) + + assert isinstance(result, ErrorResponse) + assert result._http_status == 400 + mock_ops.render_and_params.assert_not_called() + + +@pytest.mark.asyncio +async def test_invalid_reasoning_effort_returns_400(): + infer = _make_infer() + request = ResponsesRequest(model="m", input="hi", reasoning={"effort": "turbo"}) + + result = await infer.create_response(request, raw_request=_FakeRawRequest()) + + assert isinstance(result, ErrorResponse) + assert result._http_status == 400 + + +@pytest.mark.asyncio +async def test_stream_prevalidation_error_returns_plain_error_not_generator(): + """A VLLMValidationError from render_and_params must short-circuit to a + plain ErrorResponse before any generator is created — the client gets a + 400 body, not a broken/empty event stream.""" + infer = _make_infer() + request = ResponsesRequest(model="m", input="hi", stream=True) + + with patch("modelship.infer.vllm.vllm_infer.engine_ops") as mock_ops: + mock_ops.build_vllm_request.return_value = MagicMock() + mock_ops.render_and_params = AsyncMock(side_effect=VLLMValidationError("too long", parameter="messages")) + result = await infer.create_response(request, raw_request=_FakeRawRequest()) + + assert isinstance(result, ErrorResponse) + assert result._http_status == 400 + + +@pytest.mark.asyncio +async def test_stream_success_produces_native_responses_events(): + infer = _make_infer() + infer.openai_serving_render.renderer.tokenizer = MagicMock() + request = ResponsesRequest(model="m", input="hi", stream=True) + + async def fake_stream(*_args, **_kwargs): + yield ChatCompletionStreamResponse( + model="m", choices=[ChatCompletionResponseStreamChoice(index=0, delta=DeltaMessage(content="hi!"))] + ) + yield ChatCompletionStreamResponse( + model="m", + choices=[ChatCompletionResponseStreamChoice(index=0, delta=DeltaMessage(), finish_reason="stop")], + ) + + with patch("modelship.infer.vllm.vllm_infer.engine_ops") as mock_ops: + mock_ops.build_vllm_request.return_value = MagicMock() + mock_ops.render_and_params = AsyncMock(return_value=(MagicMock(), MagicMock())) + mock_ops.stream_chat_completion = MagicMock(side_effect=fake_stream) + result = await infer.create_response(request, raw_request=_FakeRawRequest()) + body = "".join([chunk async for chunk in result]) + + assert "event: response.created" in body + assert "event: response.output_text.delta" in body + assert "event: response.completed" in body + + +@pytest.mark.asyncio +async def test_stream_mid_stream_exception_emits_failed_event(): + infer = _make_infer() + infer.openai_serving_render.renderer.tokenizer = MagicMock() + request = ResponsesRequest(model="m", input="hi", stream=True) + + async def fake_stream(*_args, **_kwargs): + yield ChatCompletionStreamResponse( + model="m", choices=[ChatCompletionResponseStreamChoice(index=0, delta=DeltaMessage(content="partial"))] + ) + raise RuntimeError("engine blew up") + + with patch("modelship.infer.vllm.vllm_infer.engine_ops") as mock_ops: + mock_ops.build_vllm_request.return_value = MagicMock() + mock_ops.render_and_params = AsyncMock(return_value=(MagicMock(), MagicMock())) + mock_ops.stream_chat_completion = MagicMock(side_effect=fake_stream) + result = await infer.create_response(request, raw_request=_FakeRawRequest()) + body = "".join([chunk async for chunk in result]) + + assert "event: response.failed" in body + assert "event: response.completed" not in body From 6f5d05f4d77364d96c65fe0acec0334439df2f64 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Sat, 4 Jul 2026 08:15:56 +0000 Subject: [PATCH 32/35] feat: make vllm loader installable on the cpu extra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Routes vllm to wheels.vllm.ai's +cpu index under the cpu extra (PyPI doesn't host these wheels, so this needed explicit index wiring, same pattern as llama-cpp-python's cu130 wheel). Also fixes a real bug this surfaced: vLLM's CPU backend repurposes gpu_memory_utilization to mean host-RAM fraction rather than VRAM, so the existing 0.9 GPU default would reserve 90% of node RAM and crash every CPU deploy at worker init — num_gpus: 0 vllm deploys now default it to 0.4 unless set explicitly. This unblocks a non-GGUF (safetensors/AWQ/GPTQ) CPU path for models like gemma whose tool-calling doesn't work on llama.cpp's parser, ahead of the planned llama_cpp loader removal. --- AGENTS.md | 2 +- CLAUDE.md | 5 +- config/examples/vllm-cpu.yaml | 34 ++++ docs/model-configuration.md | 36 +++- modelship/infer/infer_config.py | 22 ++- pyproject.toml | 13 ++ tests/test_config.py | 24 +++ uv.lock | 334 ++++++++++++++++++++++++++------ 8 files changed, 400 insertions(+), 70 deletions(-) create mode 100644 config/examples/vllm-cpu.yaml diff --git a/AGENTS.md b/AGENTS.md index bdfe417..52877aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ uv run pytest tests/test_config.py::TestLlamaCppConfig::test_defaults -v CI (`.github/workflows/ci.yml`) runs `uv sync --extra dev --extra gpu` on Linux, then `ruff check`, `ruff format --check`, `pyright`, and `pytest tests/ -v`. Match that locally before pushing. -`make lint` requires `--extra gpu` to be installed. Pyright resolves imports against the active venv, and `vllm`, `gguf`, `diffusers`, and `psutil` only ship under the gpu extra, so lint on a cpu-only sync fails with `reportMissingImports`. Tests run fine on either extra (the gpu extra is a superset). +`make lint` requires `--extra gpu` to be installed. Pyright resolves imports against the active venv, and `gguf`, `diffusers`, and `psutil` only ship under the gpu extra, so lint on a cpu-only sync fails with `reportMissingImports`. (`vllm` is importable under both extras as of the Stage E0 CPU wheel wiring — it's no longer gpu-only, just not enough on its own to make lint pass cpu-only.) Tests run fine on either extra (the gpu extra is a superset). Agents: when running tests on your own initiative (sanity-checking a change, verifying a bump), skip the slow `integration`-marked suite by default — `uv run pytest tests/ -v -m "not integration"`. Only run the full `make test` (which includes integration) when explicitly requested. diff --git a/CLAUDE.md b/CLAUDE.md index b9425c4..6b822a5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,7 +28,7 @@ uv run pytest tests/test_config.py::TestLlamaCppConfig::test_defaults -v CI mirrors `make lint` + `pytest tests/ -v`. Match it locally before pushing. -`make lint` requires the `gpu` extra — pyright fails with `reportMissingImports` for `vllm`, `gguf`, `diffusers`, and `psutil` under the cpu sync. Tests pass on either extra. +`make lint` requires the `gpu` extra — pyright fails with `reportMissingImports` for `gguf`, `diffusers`, and `psutil` under the cpu sync. (`vllm` is now importable under both extras — Stage E0 wired a CPU wheel index — but that alone doesn't unblock a cpu-only lint.) Tests pass on either extra. When running tests on your own initiative, skip the slow integration suite: `uv run pytest tests/ -v -m "not integration"`. Only run full `make test` when explicitly requested. @@ -66,7 +66,8 @@ Under `tests/`, `pytest-asyncio` for async. Tests **mock out Ray Serve** — the ## Sharp edges - `vllm==0.24.0` is pinned. Don't bump casually — TP scheduling in `mship_deploy.py:build_deployment_options` defaults to the Ray V2 executor, and the loader binds to vLLM-internal `entrypoints.*` module paths that upstream restructures between minors (the `vllm_infer.py` imports moved in 0.22/0.23). -- **GGUF is unsupported on the `vllm` loader.** 0.24 moved GGUF out of tree, and the only external `vllm-gguf-plugin` (`0.0.2`) has a stale `override_quantization_method` signature that breaks *every* quantized model on 0.24 — so it's deliberately not installed. `resolve_all_model_sources` (in `deploy/config.py`) rejects a `.gguf` on the vllm loader at driver preflight with a pointer to `llama_cpp`. Use `loader: llama_cpp` for GGUF; the vllm loader takes safetensors or AWQ/GPTQ/FP8 quants. +- **GGUF is unsupported on the `vllm` loader.** 0.24 moved GGUF out of tree, and the only external `vllm-gguf-plugin` (`0.0.2`) has a stale `override_quantization_method` signature that breaks *every* quantized model on 0.24 — so it's deliberately not installed. `resolve_all_model_sources` (in `deploy/config.py`) rejects a `.gguf` on the vllm loader at driver preflight with a pointer to `llama_cpp`. Use `loader: llama_cpp` for GGUF; the vllm loader takes safetensors or AWQ/GPTQ/FP8 quants. **This is unconditional on GPU vs. CPU** — the CPU wheel (below) doesn't relax it either; a GGUF gemma still needs `llama_cpp`/`llama_server`, or a non-GGUF checkpoint to run on `vllm`. +- `vllm==0.24.0+cpu` is installable on the `cpu` extra via an explicit `vllm-cpu` index (`wheels.vllm.ai/0.24.0/cpu`, scoped through `tool.uv.sources` same as `llama-cpp-python`'s cu130 wheel) — the URL embeds the vLLM version, so a future bump must update it. On vLLM's CPU backend, `gpu_memory_utilization` is repurposed to mean *fraction of host RAM* reserved for KV cache, not VRAM; `normalize_num_gpus_and_tp` (`infer_config.py`) lowers its default to `0.4` for `num_gpus: 0` vllm deploys so a naive CPU config doesn't ask to reserve 90% of node RAM and crash at worker init — an explicit value always overrides it. See `config/examples/vllm-cpu.yaml`. - `llama_cpp` GPU support relies on the prebuilt CUDA wheel from the `llama-cpp-cu130` index (`abetlen.github.io/llama-cpp-python/whl/cu130`), scoped to the `gpu` extra via `tool.uv.sources`. The `cpu` extra has no source entry, so it resolves the plain PyPI sdist (built from source) as before. Both extras pin the same `llama-cpp-python` floor so they stay in lockstep. - The cu130 wheel's `libllama.so` statically links `libcuda.so.1` (the real NVIDIA driver library), unlike torch/vllm which resolve CUDA lazily — so merely `import llama_cpp` crashes on any driverless machine, including CI's `ubuntu-latest` runner. `ci.yml`'s `test` job installs NVIDIA's `cuda-driver-dev` stub `libcuda.so` (via a pinned `.deb`, extracted with `dpkg-deb`, no apt repo added) and symlinks it to `libcuda.so.1` on `LD_LIBRARY_PATH` — its `cuInit()` returns `CUDA_ERROR_STUB_LIBRARY` instead of crashing, which is enough for the import to succeed. If that pinned `.deb` URL ever 404s, grab a current `cuda-driver-dev-13-*` build from `developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/`. - Metrics live on port **8079** (not 8000). `MSHIP_METRICS=false` or `--no-metrics` disables. When `mship_deploy` starts its own head (no `--use-existing-ray-cluster`), `connect_ray` pins that port via `ray.init(_metrics_export_port=…)` — a **private** Ray kwarg (accepted through `**kwargs`). A `TestConnectRay` test guards it so a Ray bump that drops it fails loudly. diff --git a/config/examples/vllm-cpu.yaml b/config/examples/vllm-cpu.yaml new file mode 100644 index 0000000..b2716bc --- /dev/null +++ b/config/examples/vllm-cpu.yaml @@ -0,0 +1,34 @@ +# vLLM loader on CPU — no GPU required (Stage E0). +# +# The main use case today: gemma models can't be tool-called through the +# llama.cpp-family loaders (llama_cpp/llama_server), but work fine through +# vllm. This is NOT a way to run your existing GGUF gemma file on CPU — the +# vllm loader rejects .gguf unconditionally, GPU or CPU. You need a non-GGUF +# checkpoint: safetensors, or an AWQ/GPTQ/compressed-tensors quant (CPU +# backend supports AWQ/GPTQ on x86, plus INT8 W8A8). Look for a +# community AWQ/GPTQ repack of the gemma model you want, or use the full +# safetensors checkpoint if you have the RAM for it. +# +# gpu_memory_utilization means something different here than on GPU: vLLM's +# CPU backend repurposes it as the fraction of HOST RAM to reserve for the KV +# cache, not VRAM. modelship defaults it to 0.4 for num_gpus: 0 deploys (the +# GPU-oriented 0.9 default would ask to reserve 90% of node RAM and fail at +# worker init on a real machine) — set it explicitly here to whatever your +# box can spare. For finer control, vLLM also reads VLLM_CPU_KVCACHE_SPACE +# (a fixed GiB budget instead of a RAM fraction) and VLLM_CPU_OMP_THREADS_BIND +# from the process environment directly; see docs/model-configuration.md. +# +# See docs/model-configuration.md for the full vllm_engine_kwargs reference. + +models: + - name: "gemma-cpu" + model: "org/gemma-3-AWQ" # replace with a real non-GGUF gemma checkpoint + usecase: "generate" + loader: "vllm" + num_gpus: 0 + num_cpus: 4 + vllm_engine_kwargs: + max_model_len: 8192 + enable_auto_tool_choice: true + tool_call_parser: "functiongemma" + gpu_memory_utilization: 0.4 diff --git a/docs/model-configuration.md b/docs/model-configuration.md index b3375f2..d1aabd5 100644 --- a/docs/model-configuration.md +++ b/docs/model-configuration.md @@ -184,7 +184,7 @@ The `vllm` loader supports chat/generation, embeddings, transcription, and trans | `dtype` | string | `auto` | Model dtype (`auto`, `float16`, `bfloat16`) | | `tokenizer` | string | model default | Custom tokenizer path | | `trust_remote_code` | bool | `false` | Allow remote code execution | -| `gpu_memory_utilization` | float | `0.9` | VRAM fraction (overridden by `num_gpus` when `num_gpus < 1`) | +| `gpu_memory_utilization` | float | `0.9` (`0.4` on CPU deploys) | VRAM fraction on GPU; on CPU it means *host RAM* fraction reserved for the KV cache instead (see [CPU (no GPU required)](#cpu-no-gpu-required) below). Overridden by `num_gpus` when `num_gpus < 1`, including `num_gpus: 0`. | | `quantization` | string | — | Quantization method (e.g. `awq`, `gptq`) | | `enable_auto_tool_choice` | bool | — | Enable automatic tool/function calling | | `tool_call_parser` | string | — | Tool call parser (e.g. `llama3_json`, `hermes`) | @@ -194,6 +194,40 @@ The `vllm` loader supports chat/generation, embeddings, transcription, and trans > **GGUF is not supported on the `vllm` loader.** vLLM 0.24 dropped in-tree GGUF, so > pointing the vllm loader at a `.gguf` is rejected at startup. Use `loader: llama_cpp` > for GGUF models; the vllm loader takes safetensors checkpoints or AWQ/GPTQ/FP8 quants. +> This is unconditional regardless of GPU vs. CPU — see below. + +### CPU (no GPU required) + +The `vllm` loader also installs on the `cpu` extra (`num_gpus: 0`). The main use case is +gemma models, which the llama.cpp-family loaders (`llama_cpp`/`llama_server`) can't +tool-call — vLLM handles gemma tool-calling correctly, GPU or CPU. This is **not** a way +to run an existing GGUF gemma file on CPU: the GGUF rejection above applies here too, so +you need a non-GGUF checkpoint (safetensors, or an AWQ/GPTQ/compressed-tensors quant — +the CPU backend supports AWQ/GPTQ on x86 plus INT8 W8A8). + +`gpu_memory_utilization` means something different on CPU: vLLM repurposes it as the +fraction of **host RAM** to reserve for the KV cache, not VRAM. modelship lowers its +default to `0.4` for `num_gpus: 0` deploys — the GPU-oriented `0.9` default would try to +reserve 90% of node RAM and fail at worker init on a real machine — but set it explicitly +based on what your box can spare. For finer control than a RAM fraction, vLLM also reads +`VLLM_CPU_KVCACHE_SPACE` (a fixed GiB budget) and `VLLM_CPU_OMP_THREADS_BIND` directly +from the process environment; these are vLLM-native env vars, not modelship config. + +```yaml +models: + - name: gemma-cpu + model: org/gemma-3-AWQ # a real non-GGUF gemma checkpoint + usecase: generate + loader: vllm + num_gpus: 0 + num_cpus: 4 + vllm_engine_kwargs: + enable_auto_tool_choice: true + tool_call_parser: functiongemma + gpu_memory_utilization: 0.4 +``` + +See `config/examples/vllm-cpu.yaml` for a complete example. ### Chat / Text Generation diff --git a/modelship/infer/infer_config.py b/modelship/infer/infer_config.py index 3c37a1b..b0fd26c 100644 --- a/modelship/infer/infer_config.py +++ b/modelship/infer/infer_config.py @@ -27,6 +27,13 @@ # re-bound with the same app name. _FINGERPRINT_EXCLUDED_FIELDS = {"name", "num_replicas", "autoscaling_config"} +# vLLM's CPU backend repurposes gpu_memory_utilization to mean "fraction of +# HOST RAM to reserve for the KV cache" (not VRAM) — the GPU-oriented 0.9 +# default asks to reserve 90% of node RAM and reliably raises at worker init +# on a real machine. Used only for num_gpus == 0 vllm deploys (see +# normalize_num_gpus_and_tp); an explicitly set value always wins. +_VLLM_CPU_DEFAULT_GPU_MEMORY_UTILIZATION = 0.4 + ChatTemplateContentFormatOption = Literal["auto", "string", "openai"] @@ -57,7 +64,7 @@ class VllmEngineConfig(BaseModel): dtype: str = "auto" tokenizer: str | None = None trust_remote_code: bool = False - gpu_memory_utilization: float = 0.9 # overridden by num_gpus when num_gpus < 1 + gpu_memory_utilization: float = 0.9 # overridden by num_gpus when num_gpus < 1 (incl. 0, CPU deploys) task: str = "auto" model_impl: str | None = None enable_log_requests: bool | None = False @@ -390,9 +397,20 @@ def normalize_num_gpus_and_tp(self): also set num_gpus, log a warning and use tp x pp (each slot owns a whole GPU). - When tp = pp = 1 and num_gpus >= 2 is set, auto-derive tp = num_gpus. + - num_gpus == 0: a CPU deploy. Lower gpu_memory_utilization's default + (see _VLLM_CPU_DEFAULT_GPU_MEMORY_UTILIZATION) — same rationale and + "explicit value always wins" mechanism as the fractional-GPU case. """ ng = self.num_gpus - if ng <= 0 or self.loader != ModelLoader.vllm: + if self.loader != ModelLoader.vllm: + return self + + if ng == 0: + if "gpu_memory_utilization" not in self.vllm_engine_kwargs.model_fields_set: + self.vllm_engine_kwargs.gpu_memory_utilization = _VLLM_CPU_DEFAULT_GPU_MEMORY_UTILIZATION + self.vllm_engine_kwargs.model_fields_set.add("gpu_memory_utilization") + return self + if ng < 0: return self tp = self.vllm_engine_kwargs.tensor_parallel_size diff --git a/pyproject.toml b/pyproject.toml index 15601af..6b5716e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,8 @@ cpu = [ "llama-cpp-python>=0.3.32", "stable-diffusion-cpp-python>=0.4.7", "onnxruntime>=1.20.1", + "vllm==0.24.0", + "vllm[audio]==0.24.0", ] kokoroonnx = ["kokoroonnx"] bark = ["bark"] @@ -107,6 +109,14 @@ name = "llama-cpp-cu130" url = "https://abetlen.github.io/llama-cpp-python/whl/cu130" explicit = true +# NOTE: this URL embeds the vllm version — a future bump of vllm==0.24.0 above +# must update this to match, or the cpu extra silently falls back to whatever +# (if anything) that path resolves to. +[[tool.uv.index]] +name = "vllm-cpu" +url = "https://wheels.vllm.ai/0.24.0/cpu" +explicit = true + [tool.uv.sources] kokoroonnx = { workspace = true } bark = { workspace = true } @@ -123,6 +133,9 @@ torchvision = [ llama-cpp-python = [ { index = "llama-cpp-cu130", extra = "gpu" }, ] +vllm = [ + { index = "vllm-cpu", extra = "cpu" }, +] [tool.uv] conflicts = [ diff --git a/tests/test_config.py b/tests/test_config.py index 45a9d9c..a2e1a9f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -398,6 +398,30 @@ def test_whole_gpu_leaves_gpu_memory_utilization_default(self): ) assert config.vllm_engine_kwargs.gpu_memory_utilization == 0.9 + def test_cpu_num_gpus_lowers_gpu_memory_utilization_default(self): + # On vLLM's CPU backend, gpu_memory_utilization means "fraction of host + # RAM to reserve," not VRAM — the GPU-oriented 0.9 default reserves 90% + # of node RAM and reliably raises at worker init on a real machine. + config = ModelshipModelConfig( + name="test-llm", + model="some-model", + usecase=ModelUsecase.generate, + loader=ModelLoader.vllm, + num_gpus=0, + ) + assert config.vllm_engine_kwargs.gpu_memory_utilization == 0.4 + + def test_explicit_gpu_memory_utilization_wins_over_cpu_default(self): + config = ModelshipModelConfig( + name="test-llm", + model="some-model", + usecase=ModelUsecase.generate, + loader=ModelLoader.vllm, + num_gpus=0, + vllm_engine_kwargs={"gpu_memory_utilization": 0.6}, + ) + assert config.vllm_engine_kwargs.gpu_memory_utilization == 0.6 + def test_num_gpus_integer_required_above_one(self): with pytest.raises(ValidationError, match="must be integers"): ModelshipModelConfig( diff --git a/uv.lock b/uv.lock index 069a236..dcf514d 100644 --- a/uv.lock +++ b/uv.lock @@ -6,10 +6,21 @@ resolution-markers = [ "sys_platform == 'emscripten' and extra != 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu'", "sys_platform == 'darwin' and extra != 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu'", "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu'", - "sys_platform == 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", - "sys_platform == 'emscripten' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", - "sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", + "(platform_machine == 's390x' and sys_platform == 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and sys_platform == 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')", + "platform_machine == 'aarch64' and sys_platform == 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", + "(platform_machine == 'ppc64le' and sys_platform == 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (platform_machine == 'riscv64' and sys_platform == 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", + "(platform_machine == 's390x' and sys_platform == 'emscripten' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and sys_platform == 'emscripten' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')", + "platform_machine == 'aarch64' and sys_platform == 'emscripten' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", + "(platform_machine == 'ppc64le' and sys_platform == 'emscripten' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (platform_machine == 'riscv64' and sys_platform == 'emscripten' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'emscripten' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", + "(platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')", + "platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", + "(platform_machine == 'ppc64le' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (platform_machine == 'riscv64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", + "(platform_machine == 's390x' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')", + "platform_machine == 'aarch64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", + "platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", "sys_platform == 'win32' and extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", "sys_platform == 'emscripten' and extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", "sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", @@ -444,7 +455,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "loguru" }, { name = "pydantic" }, - { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, + { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, + { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "extra == 'extra-9-modelship-gpu'" }, { name = "transformers" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2c/9e/d7f18bd9a0354088abc11a0c1f2c7698f7c49e5a709faedf6a46e388f693/compressed_tensors-0.17.0.tar.gz", hash = "sha256:15c20d06bdbcf35b51fc99fd125e7b9be1e1855567c33b7a46dfac26ad6fb126", size = 257091, upload-time = "2026-06-03T16:49:17.208Z" } @@ -822,13 +835,13 @@ wheels = [ [package.optional-dependencies] standard = [ { name = "email-validator" }, - { name = "fastapi-cli", extra = ["standard"], marker = "extra == 'extra-9-modelship-gpu'" }, + { name = "fastapi-cli", extra = ["standard"], marker = "extra == 'extra-9-modelship-cpu' or extra == 'extra-9-modelship-gpu'" }, { name = "httpx" }, { name = "jinja2" }, { name = "pydantic-extra-types" }, { name = "pydantic-settings" }, { name = "python-multipart" }, - { name = "uvicorn", extra = ["standard"], marker = "extra == 'extra-9-modelship-gpu'" }, + { name = "uvicorn", extra = ["standard"], marker = "extra == 'extra-9-modelship-cpu' or extra == 'extra-9-modelship-gpu'" }, ] [[package]] @@ -838,7 +851,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "rich-toolkit" }, { name = "typer" }, - { name = "uvicorn", extra = ["standard"], marker = "extra == 'extra-9-modelship-gpu'" }, + { name = "uvicorn", extra = ["standard"], marker = "extra == 'extra-9-modelship-cpu' or extra == 'extra-9-modelship-gpu'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6e/58/74797ae9e4610cfa0c6b34c8309096d3b20bb29be3b8b5fbf1004d10fa5f/fastapi_cli-0.0.24.tar.gz", hash = "sha256:1afc9c9e21d7ebc8a3ca5e31790cd8d837742be7e4f8b9236e99cb3451f0de00", size = 19043, upload-time = "2026-02-24T10:45:10.476Z" } wheels = [ @@ -848,7 +861,7 @@ wheels = [ [package.optional-dependencies] standard = [ { name = "fastapi-cloud-cli" }, - { name = "uvicorn", extra = ["standard"], marker = "extra == 'extra-9-modelship-gpu'" }, + { name = "uvicorn", extra = ["standard"], marker = "extra == 'extra-9-modelship-cpu' or extra == 'extra-9-modelship-gpu'" }, ] [[package]] @@ -858,12 +871,12 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fastar" }, { name = "httpx" }, - { name = "pydantic", extra = ["email"], marker = "extra == 'extra-9-modelship-gpu'" }, + { name = "pydantic", extra = ["email"], marker = "extra == 'extra-9-modelship-cpu' or extra == 'extra-9-modelship-gpu'" }, { name = "rich-toolkit" }, { name = "rignore" }, { name = "sentry-sdk" }, { name = "typer" }, - { name = "uvicorn", extra = ["standard"], marker = "extra == 'extra-9-modelship-gpu'" }, + { name = "uvicorn", extra = ["standard"], marker = "extra == 'extra-9-modelship-cpu' or extra == 'extra-9-modelship-gpu'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7f/f2/fcd66ce245b7e3c3d84ca8717eda8896945fbc17c87a9b03f490ff06ace7/fastapi_cloud_cli-0.15.1.tar.gz", hash = "sha256:71a46f8a1d9fea295544113d6b79f620dc5768b24012887887306d151165745d", size = 43851, upload-time = "2026-03-26T10:23:12.932Z" } wheels = [ @@ -1254,6 +1267,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "intel-openmp" +version = "2024.2.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/a6/92e7356c981fcfefa4fbd132791d28656ec02ff62124d283b10219999d5d/intel_openmp-2024.2.1-py2.py3-none-manylinux1_i686.whl", hash = "sha256:ba6de86c394331719c795e743bcff3bbc10c3a16a0d3622d90819593de4c7e34", size = 3848711, upload-time = "2024-08-07T15:13:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/64570ae938a8ee2337ed8ba28ae1d85d3555ee6e5faadabea9e8b43a900d/intel_openmp-2024.2.1-py2.py3-none-manylinux1_x86_64.whl", hash = "sha256:21892ea07a9c6e164707d2b44e6f396f9f2ab6cc7e108755011fad59b596182a", size = 29565156, upload-time = "2024-08-07T15:15:07.808Z" }, + { url = "https://files.pythonhosted.org/packages/c6/02/e5c7e10a4bedeec4eb0f61fa8fe951e2febf6ba9ae08f806ada5f2f35ada/intel_openmp-2024.2.1-py2.py3-none-win32.whl", hash = "sha256:3dbed102d8a79f091fc3364ff4b6268e1b2904d80a58926987fc6006171c18cd", size = 928744, upload-time = "2024-08-07T15:12:33.03Z" }, + { url = "https://files.pythonhosted.org/packages/c8/6c/9334569937fd3c6ec93f9fe3da268db38f5113673d568854fa18a20f36ec/intel_openmp-2024.2.1-py2.py3-none-win_amd64.whl", hash = "sha256:b1fb47eefc6cbc2358216684dd87bd2315464e6d29ac84f9313819fe1528d969", size = 3602212, upload-time = "2024-08-07T15:12:45.626Z" }, +] + [[package]] name = "interegular" version = "0.3.3" @@ -1461,10 +1485,21 @@ name = "llama-cpp-python" version = "0.3.32" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "sys_platform == 'darwin'", + "(platform_machine == 's390x' and sys_platform == 'win32') or (platform_machine == 'x86_64' and sys_platform == 'win32')", + "platform_machine == 'aarch64' and sys_platform == 'win32'", + "(platform_machine == 'ppc64le' and sys_platform == 'win32') or (platform_machine == 'riscv64' and sys_platform == 'win32')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'win32'", + "(platform_machine == 's390x' and sys_platform == 'emscripten') or (platform_machine == 'x86_64' and sys_platform == 'emscripten')", + "platform_machine == 'aarch64' and sys_platform == 'emscripten'", + "(platform_machine == 'ppc64le' and sys_platform == 'emscripten') or (platform_machine == 'riscv64' and sys_platform == 'emscripten')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'emscripten'", + "(platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(platform_machine == 'ppc64le' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'riscv64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')", + "platform_machine == 'aarch64' and sys_platform == 'darwin'", + "platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin'", ] dependencies = [ { name = "diskcache" }, @@ -1572,7 +1607,7 @@ dependencies = [ { name = "jsonschema" }, { name = "pydantic" }, { name = "pydantic-settings" }, - { name = "pyjwt", extra = ["crypto"], marker = "extra == 'extra-9-modelship-gpu'" }, + { name = "pyjwt", extra = ["crypto"], marker = "extra == 'extra-9-modelship-cpu' or extra == 'extra-9-modelship-gpu'" }, { name = "python-multipart" }, { name = "pywin32", marker = "sys_platform == 'win32'" }, { name = "sse-starlette" }, @@ -1604,7 +1639,7 @@ dependencies = [ { name = "numpy" }, { name = "pillow" }, { name = "pydantic" }, - { name = "pydantic-extra-types", extra = ["pycountry"], marker = "extra == 'extra-9-modelship-gpu'" }, + { name = "pydantic-extra-types", extra = ["pycountry"], marker = "extra == 'extra-9-modelship-cpu' or extra == 'extra-9-modelship-gpu'" }, { name = "requests" }, { name = "tiktoken" }, { name = "typing-extensions" }, @@ -1648,7 +1683,8 @@ dependencies = [ { name = "httpx" }, { name = "jmespath" }, { name = "pydantic" }, - { name = "setuptools" }, + { name = "setuptools", version = "77.0.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-9-modelship-cpu'" }, + { name = "setuptools", version = "80.9.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-9-modelship-gpu'" }, { name = "starlette" }, { name = "supervisor" }, ] @@ -1690,11 +1726,12 @@ cpu = [ { name = "onnxruntime" }, { name = "sentence-transformers" }, { name = "stable-diffusion-cpp-python" }, - { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "torchvision", version = "0.26.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "torchvision", version = "0.26.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torchvision", version = "0.26.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, { name = "transformers" }, + { name = "vllm", version = "0.24.0+cpu", source = { registry = "https://wheels.vllm.ai/0.24.0/cpu" }, extra = ["audio"], marker = "extra == 'extra-9-modelship-cpu'" }, ] dev = [ { name = "fakeredis" }, @@ -1717,7 +1754,7 @@ gpu = [ { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, { name = "torchvision", version = "0.26.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, { name = "transformers" }, - { name = "vllm", extra = ["audio"], marker = "extra == 'extra-9-modelship-gpu'" }, + { name = "vllm", version = "0.24.0", source = { registry = "https://pypi.org/simple" }, extra = ["audio"], marker = "extra == 'extra-9-modelship-gpu'" }, ] kokoroonnx = [ { name = "kokoroonnx" }, @@ -1782,7 +1819,9 @@ requires-dist = [ { name = "torchvision", marker = "extra == 'gpu'", specifier = ">=0.25.0", index = "https://download.pytorch.org/whl/cu130", conflict = { package = "modelship", extra = "gpu" } }, { name = "transformers", marker = "extra == 'cpu'", specifier = ">=5.5.3" }, { name = "transformers", marker = "extra == 'gpu'", specifier = ">=5.5.3" }, + { name = "vllm", marker = "extra == 'cpu'", specifier = "==0.24.0", index = "https://wheels.vllm.ai/0.24.0/cpu", conflict = { package = "modelship", extra = "cpu" } }, { name = "vllm", marker = "extra == 'gpu'", specifier = "==0.24.0" }, + { name = "vllm", extras = ["audio"], marker = "extra == 'cpu'", specifier = "==0.24.0", index = "https://wheels.vllm.ai/0.24.0/cpu", conflict = { package = "modelship", extra = "cpu" } }, { name = "vllm", extras = ["audio"], marker = "extra == 'gpu'", specifier = "==0.24.0" }, { name = "whispercpp", marker = "extra == 'whispercpp'", editable = "plugins/whispercpp" }, ] @@ -3511,8 +3550,8 @@ dependencies = [ { name = "numpy" }, { name = "scikit-learn" }, { name = "scipy" }, - { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "extra == 'extra-9-modelship-gpu'" }, { name = "tqdm" }, { name = "transformers" }, @@ -3570,10 +3609,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/5b/a9fe517912cd6e28cf43a212b80cb679ff179a91b623138a99796d7d18a0/setproctitle-1.3.7-cp312-cp312-win_amd64.whl", hash = "sha256:9888ceb4faea3116cf02a920ff00bfbc8cc899743e4b4ac914b03625bdc3c300", size = 13247, upload-time = "2025-09-05T12:49:49.16Z" }, ] +[[package]] +name = "setuptools" +version = "77.0.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(platform_machine == 's390x' and sys_platform == 'win32') or (platform_machine == 'x86_64' and sys_platform == 'win32')", + "platform_machine == 'aarch64' and sys_platform == 'win32'", + "(platform_machine == 'ppc64le' and sys_platform == 'win32') or (platform_machine == 'riscv64' and sys_platform == 'win32')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'win32'", + "(platform_machine == 's390x' and sys_platform == 'emscripten') or (platform_machine == 'x86_64' and sys_platform == 'emscripten')", + "platform_machine == 'aarch64' and sys_platform == 'emscripten'", + "(platform_machine == 'ppc64le' and sys_platform == 'emscripten') or (platform_machine == 'riscv64' and sys_platform == 'emscripten')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'emscripten'", + "(platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(platform_machine == 'ppc64le' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'riscv64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')", + "platform_machine == 'aarch64' and sys_platform == 'darwin'", + "platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin'", +] +sdist = { url = "https://files.pythonhosted.org/packages/81/ed/7101d53811fd359333583330ff976e5177c5e871ca8b909d1d6c30553aa3/setuptools-77.0.3.tar.gz", hash = "sha256:583b361c8da8de57403743e756609670de6fb2345920e36dc5c2d914c319c945", size = 1367236, upload-time = "2025-03-20T14:38:08.777Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/07/99f2cefae815c66eb23148f15d79ec055429c38fa8986edcc712ab5f3223/setuptools-77.0.3-py3-none-any.whl", hash = "sha256:67122e78221da5cf550ddd04cf8742c8fe12094483749a792d56cd669d6cf58c", size = 1255678, upload-time = "2025-03-20T14:38:06.621Z" }, +] + [[package]] name = "setuptools" version = "80.9.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'win32' and extra != 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu'", + "sys_platform == 'emscripten' and extra != 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu'", + "sys_platform == 'darwin' and extra != 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu'", + "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu'", + "sys_platform == 'win32' and extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", + "sys_platform == 'emscripten' and extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", + "sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu'", +] sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, @@ -3617,9 +3691,9 @@ dependencies = [ { name = "einops" }, { name = "huggingface-hub" }, { name = "numpy" }, - { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')" }, - { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "extra == 'extra-9-modelship-gpu'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/38/5b64fb15c1cf02233252975c43c4b85ccacd9f77c55f7fee72b16b3bd2f6/snac-1.2.1.tar.gz", hash = "sha256:697f27fc5b98308eee8946739e5fd9c1b4ec629ef51b4f01c08dace1290685ee", size = 8737, upload-time = "2024-09-11T15:00:46.175Z" } @@ -3793,7 +3867,7 @@ dependencies = [ { name = "ml-dtypes" }, { name = "numpy" }, { name = "psutil" }, - { name = "setuptools", marker = "sys_platform == 'darwin'" }, + { name = "setuptools", version = "80.9.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, { name = "torch-c-dlpack-ext" }, { name = "tqdm" }, @@ -3861,16 +3935,16 @@ name = "torch" version = "2.11.0" source = { registry = "https://download.pytorch.org/whl/cpu" } resolution-markers = [ - "sys_platform == 'darwin'", + "platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin'", ] dependencies = [ - { name = "filelock", marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "fsspec", marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "jinja2", marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "networkx", marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "setuptools", marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "sympy", marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "typing-extensions", marker = "(sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "filelock", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "fsspec", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "jinja2", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "networkx", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "setuptools", version = "77.0.3", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "sympy", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:43b35116802c85fb88d99f4a396b8bd4472bfca1dd82e69499e5a4f9b8b4e252", upload-time = "2026-03-23T15:16:58Z" }, @@ -3896,7 +3970,7 @@ dependencies = [ { name = "nvidia-cusparselt-cu13", marker = "(sys_platform == 'linux' and extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, { name = "nvidia-nccl-cu13", marker = "(sys_platform == 'linux' and extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, { name = "nvidia-nvshmem-cu13", marker = "(sys_platform == 'linux' and extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "setuptools", marker = "(extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')" }, + { name = "setuptools", version = "80.9.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')" }, { name = "sympy", marker = "(extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')" }, { name = "triton", marker = "(sys_platform == 'linux' and extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, { name = "typing-extensions", marker = "(extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (extra != 'extra-9-modelship-cpu' and extra != 'extra-9-modelship-gpu')" }, @@ -3913,18 +3987,29 @@ name = "torch" version = "2.11.0+cpu" source = { registry = "https://download.pytorch.org/whl/cpu" } resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -dependencies = [ - { name = "filelock", marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "fsspec", marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "jinja2", marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "networkx", marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "setuptools", marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "sympy", marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "typing-extensions", marker = "(sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + "(platform_machine == 's390x' and sys_platform == 'win32') or (platform_machine == 'x86_64' and sys_platform == 'win32')", + "platform_machine == 'aarch64' and sys_platform == 'win32'", + "(platform_machine == 'ppc64le' and sys_platform == 'win32') or (platform_machine == 'riscv64' and sys_platform == 'win32')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'win32'", + "(platform_machine == 's390x' and sys_platform == 'emscripten') or (platform_machine == 'x86_64' and sys_platform == 'emscripten')", + "platform_machine == 'aarch64' and sys_platform == 'emscripten'", + "(platform_machine == 'ppc64le' and sys_platform == 'emscripten') or (platform_machine == 'riscv64' and sys_platform == 'emscripten')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'emscripten'", + "(platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(platform_machine == 'ppc64le' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'riscv64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')", + "platform_machine == 'aarch64' and sys_platform == 'darwin'", +] +dependencies = [ + { name = "filelock", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, + { name = "fsspec", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, + { name = "jinja2", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, + { name = "networkx", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, + { name = "setuptools", version = "77.0.3", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, + { name = "sympy", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, + { name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:2db3ae5404e32cb42b5fcbd94f13607761eaec0cf1687fde95095289d1e26cfb", upload-time = "2026-04-28T00:06:06Z" }, @@ -3954,7 +4039,7 @@ dependencies = [ { name = "nvidia-cusparselt-cu13", marker = "(sys_platform == 'linux' and extra == 'extra-9-modelship-gpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, { name = "nvidia-nccl-cu13", marker = "(sys_platform == 'linux' and extra == 'extra-9-modelship-gpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, { name = "nvidia-nvshmem-cu13", marker = "(sys_platform == 'linux' and extra == 'extra-9-modelship-gpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "setuptools", marker = "extra == 'extra-9-modelship-gpu'" }, + { name = "setuptools", version = "80.9.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-9-modelship-gpu'" }, { name = "sympy", marker = "extra == 'extra-9-modelship-gpu'" }, { name = "triton", marker = "(sys_platform == 'linux' and extra == 'extra-9-modelship-gpu') or (extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, { name = "typing-extensions", marker = "extra == 'extra-9-modelship-gpu'" }, @@ -3996,12 +4081,12 @@ name = "torchvision" version = "0.26.0" source = { registry = "https://download.pytorch.org/whl/cpu" } resolution-markers = [ - "sys_platform == 'darwin'", + "platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", marker = "sys_platform == 'darwin'" }, - { name = "pillow", marker = "sys_platform == 'darwin'" }, - { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" }, + { name = "numpy", marker = "platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin'" }, + { name = "pillow", marker = "platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin'" }, + { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin'" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c409e1c3fdebec7a3834465086dbda8bf7680eff79abf7fd2f10c6b59520a7a4", upload-time = "2026-03-23T15:36:09Z" }, @@ -4012,14 +4097,25 @@ name = "torchvision" version = "0.26.0+cpu" source = { registry = "https://download.pytorch.org/whl/cpu" } resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -dependencies = [ - { name = "numpy", marker = "sys_platform != 'darwin'" }, - { name = "pillow", marker = "sys_platform != 'darwin'" }, - { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" }, + "(platform_machine == 's390x' and sys_platform == 'win32') or (platform_machine == 'x86_64' and sys_platform == 'win32')", + "platform_machine == 'aarch64' and sys_platform == 'win32'", + "(platform_machine == 'ppc64le' and sys_platform == 'win32') or (platform_machine == 'riscv64' and sys_platform == 'win32')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'win32'", + "(platform_machine == 's390x' and sys_platform == 'emscripten') or (platform_machine == 'x86_64' and sys_platform == 'emscripten')", + "platform_machine == 'aarch64' and sys_platform == 'emscripten'", + "(platform_machine == 'ppc64le' and sys_platform == 'emscripten') or (platform_machine == 'riscv64' and sys_platform == 'emscripten')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'emscripten'", + "(platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(platform_machine == 'ppc64le' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'riscv64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')", + "platform_machine == 'aarch64' and sys_platform == 'darwin'", +] +dependencies = [ + { name = "numpy", marker = "platform_machine == 'aarch64' or platform_machine == 's390x' or platform_machine == 'x86_64' or sys_platform != 'darwin'" }, + { name = "pillow", marker = "platform_machine == 'aarch64' or platform_machine == 's390x' or platform_machine == 'x86_64' or sys_platform != 'darwin'" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "platform_machine == 'aarch64' or platform_machine == 's390x' or platform_machine == 'x86_64' or sys_platform != 'darwin'" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:17f0b542331fc94230b4214c6d123f038af7330fd81019608c0d2402f3bc3079", upload-time = "2026-03-23T15:36:09Z" }, @@ -4208,6 +4304,12 @@ wheels = [ name = "vllm" version = "0.24.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'win32'", + "sys_platform == 'emscripten'", + "sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] dependencies = [ { name = "aiohttp" }, { name = "anthropic" }, @@ -4266,7 +4368,7 @@ dependencies = [ { name = "safetensors" }, { name = "sentencepiece" }, { name = "setproctitle" }, - { name = "setuptools" }, + { name = "setuptools", version = "80.9.0", source = { registry = "https://pypi.org/simple" } }, { name = "six" }, { name = "starlette" }, { name = "tiktoken" }, @@ -4297,6 +4399,108 @@ audio = [ { name = "soxr" }, ] +[[package]] +name = "vllm" +version = "0.24.0+cpu" +source = { registry = "https://wheels.vllm.ai/0.24.0/cpu" } +resolution-markers = [ + "(platform_machine == 's390x' and sys_platform == 'win32') or (platform_machine == 'x86_64' and sys_platform == 'win32')", + "platform_machine == 'aarch64' and sys_platform == 'win32'", + "(platform_machine == 'ppc64le' and sys_platform == 'win32') or (platform_machine == 'riscv64' and sys_platform == 'win32')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'win32'", + "(platform_machine == 's390x' and sys_platform == 'emscripten') or (platform_machine == 'x86_64' and sys_platform == 'emscripten')", + "platform_machine == 'aarch64' and sys_platform == 'emscripten'", + "(platform_machine == 'ppc64le' and sys_platform == 'emscripten') or (platform_machine == 'riscv64' and sys_platform == 'emscripten')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'emscripten'", + "(platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(platform_machine == 'ppc64le' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'riscv64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", + "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')", + "platform_machine == 'aarch64' and sys_platform == 'darwin'", + "platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin'", +] +dependencies = [ + { name = "aiohttp" }, + { name = "anthropic" }, + { name = "blake3" }, + { name = "cachetools" }, + { name = "cbor2" }, + { name = "cloudpickle" }, + { name = "compressed-tensors" }, + { name = "depyf" }, + { name = "diskcache" }, + { name = "einops" }, + { name = "fastapi", extra = ["standard"], marker = "extra == 'extra-9-modelship-cpu'" }, + { name = "filelock" }, + { name = "ijson" }, + { name = "intel-openmp", marker = "platform_machine == 'x86_64'" }, + { name = "jsonschema" }, + { name = "lark" }, + { name = "llguidance", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 'x86_64'" }, + { name = "lm-format-enforcer" }, + { name = "mcp" }, + { name = "mistral-common", extra = ["image"], marker = "extra == 'extra-9-modelship-cpu'" }, + { name = "model-hosting-container-standards" }, + { name = "msgspec" }, + { name = "ninja" }, + { name = "numba", marker = "platform_machine != 's390x'" }, + { name = "numpy" }, + { name = "openai" }, + { name = "openai-harmony" }, + { name = "opencv-python-headless" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-sdk" }, + { name = "opentelemetry-semantic-conventions-ai" }, + { name = "outlines-core" }, + { name = "partial-json-parser" }, + { name = "pillow" }, + { name = "prometheus-client" }, + { name = "prometheus-fastapi-instrumentator" }, + { name = "protobuf" }, + { name = "psutil" }, + { name = "py-cpuinfo" }, + { name = "pybase64" }, + { name = "pydantic" }, + { name = "python-json-logger" }, + { name = "pyyaml" }, + { name = "pyzmq" }, + { name = "regex" }, + { name = "requests" }, + { name = "safetensors" }, + { name = "sentencepiece" }, + { name = "setproctitle" }, + { name = "setuptools", version = "77.0.3", source = { registry = "https://pypi.org/simple" } }, + { name = "six" }, + { name = "starlette" }, + { name = "tiktoken" }, + { name = "tokenizers" }, + { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'ppc64le' and sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'riscv64' and sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torchaudio", marker = "platform_machine != 'riscv64' and platform_machine != 's390x'" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'riscv64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torchvision", version = "0.26.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'riscv64' and platform_machine != 's390x' and sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'riscv64' and sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "tqdm" }, + { name = "transformers" }, + { name = "typing-extensions" }, + { name = "watchfiles" }, + { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, +] +wheels = [ + { url = "https://wheels.vllm.ai/ee0da84ab9e04ac7610e28580af62c365e898389/vllm-0.24.0%2Bcpu-cp38-abi3-manylinux_2_34_aarch64.whl" }, + { url = "https://wheels.vllm.ai/ee0da84ab9e04ac7610e28580af62c365e898389/vllm-0.24.0%2Bcpu-cp38-abi3-manylinux_2_34_x86_64.whl" }, +] + +[package.optional-dependencies] +audio = [ + { name = "av" }, + { name = "mistral-common", extra = ["audio"], marker = "extra == 'extra-9-modelship-cpu'" }, + { name = "scipy" }, + { name = "soundfile" }, + { name = "soxr" }, +] + [[package]] name = "watchfiles" version = "1.1.1" @@ -4387,7 +4591,9 @@ dependencies = [ { name = "apache-tvm-ffi" }, { name = "numpy" }, { name = "pydantic" }, - { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, + { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, + { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "extra == 'extra-9-modelship-gpu'" }, { name = "transformers" }, { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "typing-extensions" }, From 3afc9a7135de8839be74c6c24a0a36e2b1299649 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:49:43 +0000 Subject: [PATCH 33/35] feat!: remove llama_cpp loader, cut GGUF entirely over to llama_server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llama_server has had full feature parity (embeddings, vision, logprobs, concurrency) since Stage B4, plus real request concurrency via --parallel slots and native /v1/responses shaping from Stage D, leaving no reason to keep both GGUF loaders. llama_cpp's one real gap, gemma tool calling, isn't fixed by llama_server either — the supported path is loader: vllm on the CPU extra (config/examples/vllm-cpu.yaml), landed in Stage E0. Removes the llama_cpp loader module, config schema, preflight registration, llama-cpp-python dependency and its CUDA wheel/CI plumbing, and migrates catalog/profile/example configs and tests onto llama_server. Docs updated throughout; docs/architecture.md's /v1/responses section also brought up to date with the Stage D native-shaping design it had drifted from. --- .github/workflows/ci.yml | 15 - AGENTS.md | 9 +- CLAUDE.md | 14 +- README.md | 16 +- config/examples/README.md | 6 +- config/examples/llama-cpp.yaml | 41 -- config/examples/llama-server.yaml | 11 +- config/examples/mini-pc.yaml | 6 +- config/examples/vllm-cpu.yaml | 12 +- docs/architecture.md | 18 +- docs/model-configuration.md | 106 +-- docs/plugins.md | 2 +- docs/troubleshooting.md | 4 - modelship/deploy/config.py | 27 +- modelship/deploy/profiles/catalog.py | 10 +- modelship/deploy/profiles/generator.py | 2 +- modelship/deploy/profiles/selector.py | 2 +- modelship/infer/infer_config.py | 108 +-- modelship/infer/llama_cpp/capabilities.py | 23 - modelship/infer/llama_cpp/llama_cpp_infer.py | 242 ------- .../infer/llama_cpp/openai/serving_chat.py | 420 ----------- .../llama_cpp/openai/serving_embedding.py | 63 -- modelship/infer/llama_cpp/structured.py | 64 -- modelship/infer/llama_cpp/tool_grammar.py | 365 ---------- modelship/infer/llama_cpp/utils.py | 87 --- .../infer/llama_server/llama_server_infer.py | 11 +- modelship/infer/model_deployment.py | 4 - modelship/infer/model_resolver.py | 2 +- .../stable_diffusion_cpp_infer.py | 4 +- modelship/logging.py | 1 - modelship/openai/parsers/streaming.py | 12 +- .../openai/parsers/tool_calling/input.py | 2 +- modelship/openai/protocol/chat.py | 6 +- .../openai/protocol/responses/adapter.py | 2 +- .../openai/protocol/responses/streaming.py | 2 +- modelship/preflight/base.py | 7 - modelship/preflight/llama_cpp.py | 54 +- modelship/preflight/stable_diffusion_cpp.py | 2 +- pyproject.toml | 11 - tests/test_config.py | 158 +---- tests/test_effective_config.py | 2 +- tests/test_integration.py | 663 ++---------------- tests/test_integration_profiles.py | 8 +- tests/test_llama_cpp_chat_trace.py | 105 --- tests/test_llama_cpp_gpu_offload.py | 58 -- tests/test_llama_cpp_logprobs.py | 53 -- tests/test_llama_cpp_renderer.py | 77 -- tests/test_llama_cpp_structured.py | 171 ----- tests/test_llama_cpp_tool_choice_grammar.py | 107 --- tests/test_llama_cpp_tool_grammar.py | 455 ------------ tests/test_logging.py | 2 +- tests/test_mistral_specials_smoketest.py | 63 -- tests/test_model_resolver.py | 2 +- tests/test_mship_deploy.py | 8 +- tests/test_parser_resolution.py | 10 - tests/test_preflight_llama_cpp.py | 281 -------- tests/test_profiles_generator.py | 6 +- tests/test_profiles_selector.py | 2 +- tests/test_vllm_gguf_guard.py | 4 +- uv.lock | 54 -- 60 files changed, 181 insertions(+), 3901 deletions(-) delete mode 100644 config/examples/llama-cpp.yaml delete mode 100644 modelship/infer/llama_cpp/capabilities.py delete mode 100644 modelship/infer/llama_cpp/llama_cpp_infer.py delete mode 100644 modelship/infer/llama_cpp/openai/serving_chat.py delete mode 100644 modelship/infer/llama_cpp/openai/serving_embedding.py delete mode 100644 modelship/infer/llama_cpp/structured.py delete mode 100644 modelship/infer/llama_cpp/tool_grammar.py delete mode 100644 modelship/infer/llama_cpp/utils.py delete mode 100644 tests/test_llama_cpp_chat_trace.py delete mode 100644 tests/test_llama_cpp_gpu_offload.py delete mode 100644 tests/test_llama_cpp_logprobs.py delete mode 100644 tests/test_llama_cpp_renderer.py delete mode 100644 tests/test_llama_cpp_structured.py delete mode 100644 tests/test_llama_cpp_tool_choice_grammar.py delete mode 100644 tests/test_llama_cpp_tool_grammar.py delete mode 100644 tests/test_preflight_llama_cpp.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c52d41b..ba009ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,21 +55,6 @@ jobs: - name: Set up Python run: uv python install 3.12 - # The gpu extra's llama-cpp-python wheel (cu130) statically links libcuda.so.1 - # and fails to import at all without it, even for CPU-only test configs. This - # runner has no GPU/driver, so provide NVIDIA's official stub libcuda.so (from - # cuda-driver-dev, meant for driverless build/CI environments): its cuInit() - # returns CUDA_ERROR_STUB_LIBRARY instead of crashing, which is enough to let - # the module load. See CLAUDE.md sharp edges for details. - - name: Install CUDA driver stub for llama-cpp-python import - run: | - curl -sSL -o /tmp/cuda-driver-dev.deb \ - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-driver-dev-13-0_13.0.96-1_amd64.deb - dpkg-deb -x /tmp/cuda-driver-dev.deb /tmp/cuda-stub - stub_dir=/tmp/cuda-stub/usr/local/cuda-13.0/targets/x86_64-linux/lib/stubs - ln -s libcuda.so "$stub_dir/libcuda.so.1" - echo "LD_LIBRARY_PATH=$stub_dir:${LD_LIBRARY_PATH}" >> "$GITHUB_ENV" - - name: Install dependencies run: uv sync --extra dev --extra gpu diff --git a/AGENTS.md b/AGENTS.md index 52877aa..d8179b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ make lint-fix # ruff check --fix + ruff format make test # uv run pytest tests/ -v # Run a single test -uv run pytest tests/test_config.py::TestLlamaCppConfig::test_defaults -v +uv run pytest tests/test_config.py::TestLlamaServerConfig::test_defaults -v ``` CI (`.github/workflows/ci.yml`) runs `uv sync --extra dev --extra gpu` on Linux, then `ruff check`, `ruff format --check`, `pyright`, and `pytest tests/ -v`. Match that locally before pushing. @@ -69,7 +69,7 @@ The Docker image's `CMD` is `uv run --no-sync mship_deploy.py` (against the venv - `modelship/openai/api.py` — FastAPI gateway. Uses `RequestWatcher` + a single shared `DisconnectRegistry` Ray actor (keyed by request id) to propagate client disconnects across process boundaries. - `modelship/infer/model_deployment.py` — the single `@serve.deployment` actor class; lazily imports the right backend based on `config.loader`. - `modelship/infer/infer_config.py` — pydantic config schemas **and** `RawRequestProxy` / `DisconnectRegistry`. `RawRequestProxy` exists because FastAPI `Request` cannot cross Ray process boundaries; any new attribute vLLM reads from `raw_request` must be added there. -- `modelship/infer/{vllm,transformers,diffusers,llama_cpp,custom}/` — one subdir per loader. Each has an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. `modelship/infer/llama_server/llama_server_infer.py` is a flat file with no `openai/` subpackage — it proxies a `llama-server` subprocess's own OpenAI-compatible HTTP API rather than parsing output in-process. +- `modelship/infer/{vllm,transformers,diffusers,custom}/` — one subdir per loader. Each has an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. `modelship/infer/llama_server/llama_server_infer.py` is a flat file with no `openai/` subpackage — it proxies a `llama-server` subprocess's own OpenAI-compatible HTTP API rather than parsing output in-process. - `modelship/plugins/base_plugin.py` — `BasePlugin` ABC that plugin packages subclass as `ModelPlugin`. - `plugins/*` — workspace packages, each opt-in via a root extra. The plugin module name and the extra name must match (`ensure_plugin()` calls `importlib.import_module(config.plugin)` and the error message says `uv sync --extra `). @@ -102,9 +102,8 @@ Commit messages matter: use Conventional Commits prefixes so the changelog gener - `config/models.yaml` is gitignored; `mship_deploy.py` errors out with a pointer to `config/examples/` if missing. - vLLM version is pinned (`vllm==0.24.0`). Do not bump casually — the TP scheduling logic in `mship_deploy.py:build_deployment_options` defaults to the Ray V2 executor, and the loader imports vLLM-internal `entrypoints.*` module paths that upstream restructures between minors. -- **GGUF is not supported on the `vllm` loader.** 0.24 moved GGUF out of tree; the only external `vllm-gguf-plugin` (`0.0.2`) has a stale `override_quantization_method` signature incompatible with 0.24's quantization API (it breaks *all* quantized models, not just GGUF), so it is deliberately not installed. `resolve_all_model_sources` rejects a `.gguf` on the vllm loader at driver preflight and points to `llama_cpp`. For GGUF use `loader: llama_cpp`; feed the vllm loader safetensors or an AWQ/GPTQ/FP8 quant. -- `llama_cpp` loader supports CPU or whole-GPU offload. The gpu extra installs a prebuilt CUDA (cu130) wheel from `abetlen.github.io/llama-cpp-python/whl/cu130`; the cpu extra keeps the plain PyPI CPU-only wheel. `num_gpus` must be `0` or a whole integer for llama_cpp — fractional is rejected at config time (llama.cpp has no VRAM-fraction knob). `num_gpus >= 1` honors `n_gpu_layers`; `stable_diffusion_cpp` remains forced to `num_gpus: 0` in `mship_deploy.py:build_deployment_options`. -- `llama_server` loader (GGUF, same whole-GPU-only `num_gpus` rule as `llama_cpp`) launches a `llama-server` subprocess — found via `MSHIP_LLAMA_SERVER_BIN`, pinned in the Docker images at `/opt/llama.cpp/llama-server.sh` — and proxies its native OpenAI API instead of parsing output in-process. `--parallel` slots give real request concurrency, unlike `llama_cpp`'s single `asyncio.Lock`. Tool-call/reasoning parsing is llama-server's own, auto-detected per chat template: named-function `tool_choice` forcing is unsupported globally (silently falls back to `auto`), and `tool_choice: required` is grammar-enforced for harmony-style templates but a silent no-op for hermes-style ones (e.g. Qwen3); bare `response_format: {"type": "json_object"}` (no `schema` key) is also unenforced despite llama-server's own docs claiming support — `type: json_schema` requests (what modelship sends whenever a schema is given) are unaffected. No persistent on-disk prompt cache. See `docs/model-configuration.md`'s llama_server section for the full field table and examples. +- **GGUF is not supported on the `vllm` loader.** 0.24 moved GGUF out of tree; the only external `vllm-gguf-plugin` (`0.0.2`) has a stale `override_quantization_method` signature incompatible with 0.24's quantization API (it breaks *all* quantized models, not just GGUF), so it is deliberately not installed. `resolve_all_model_sources` rejects a `.gguf` on the vllm loader at driver preflight and points to `llama_server`. For GGUF use `loader: llama_server`; feed the vllm loader safetensors or an AWQ/GPTQ/FP8 quant. +- `llama_server` loader (GGUF) launches a `llama-server` subprocess — found via `MSHIP_LLAMA_SERVER_BIN`, pinned in the Docker images at `/opt/llama.cpp/llama-server.sh` — and proxies its native OpenAI API instead of parsing output in-process. `num_gpus` must be `0` or a whole integer — fractional is rejected at config time (llama.cpp has no VRAM-fraction knob). `num_gpus >= 1` honors `n_gpu_layers`; `stable_diffusion_cpp` remains forced to `num_gpus: 0` in `mship_deploy.py:build_deployment_options`. `--parallel` slots give real request concurrency instead of serializing behind a single lock. Tool-call/reasoning parsing is llama-server's own, auto-detected per chat template: named-function `tool_choice` forcing is unsupported globally (silently falls back to `auto`), and `tool_choice: required` is grammar-enforced for harmony-style templates but a silent no-op for hermes-style ones (e.g. Qwen3); bare `response_format: {"type": "json_object"}` (no `schema` key) is also unenforced despite llama-server's own docs claiming support — `type: json_schema` requests (what modelship sends whenever a schema is given) are unaffected. No persistent on-disk prompt cache. See `docs/model-configuration.md`'s llama_server section for the full field table and examples. - Metrics are on by default on port **8079** (not 8000). Disable with `--no-metrics` or `MSHIP_METRICS=false`. - Log level `TRACE` (below `DEBUG`) is a custom level and logs full request/response payloads. - Docker images are multi-arch (amd64 + arm64). The CPU image uses the unified `Dockerfile` with `--build-arg MSHIP_VARIANT=cpu` and has a different tag suffix (`:latest-cpu`). diff --git a/CLAUDE.md b/CLAUDE.md index 6b822a5..c161c1c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,7 +23,7 @@ make lint-fix # auto-fix ruff issues make test # uv run pytest tests/ -v # Single test -uv run pytest tests/test_config.py::TestLlamaCppConfig::test_defaults -v +uv run pytest tests/test_config.py::TestLlamaServerConfig::test_defaults -v ``` CI mirrors `make lint` + `pytest tests/ -v`. Match it locally before pushing. @@ -45,11 +45,11 @@ Docker's `CMD` is `uv run --no-sync mship_deploy.py` (auto-detecting CPUs/GPUs u ## Architecture map -- `mship_deploy.py` — Ray init + deploy loop. `build_deployment_options` (in `modelship/deploy/actor_options.py`) handles GPU allocation: multi-slot vLLM deploys (`tp*pp > 1`) always build a Ray Serve placement group (one whole-GPU bundle per slot, STRICT_PACK) that vLLM's ray executor inherits via `get_current_placement_group()`. Single-slot deploys use a scalar `num_gpus` on the outer actor (fractional sharing supported). Fractional `num_gpus` with `tp*pp > 1` is rejected at config time — Ray packs fractional PG bundles onto the same physical GPU. `llama_cpp` loader supports whole-GPU offload (`num_gpus` must be `0` or an integer — fractional is rejected, llama.cpp has no VRAM-fraction knob); `stable_diffusion_cpp` remains CPU-only, forced to `num_gpus: 0`. +- `mship_deploy.py` — Ray init + deploy loop. `build_deployment_options` (in `modelship/deploy/actor_options.py`) handles GPU allocation: multi-slot vLLM deploys (`tp*pp > 1`) always build a Ray Serve placement group (one whole-GPU bundle per slot, STRICT_PACK) that vLLM's ray executor inherits via `get_current_placement_group()`. Single-slot deploys use a scalar `num_gpus` on the outer actor (fractional sharing supported). Fractional `num_gpus` with `tp*pp > 1` is rejected at config time — Ray packs fractional PG bundles onto the same physical GPU. `llama_server` loader supports whole-GPU offload (`num_gpus` must be `0` or an integer — fractional is rejected, llama.cpp has no VRAM-fraction knob); `stable_diffusion_cpp` remains CPU-only, forced to `num_gpus: 0`. - `modelship/openai/api.py` — FastAPI gateway. Uses `RequestWatcher` + a single shared `DisconnectRegistry` Ray actor (keyed by request id) to propagate client disconnects across process boundaries and cancel in-flight inference. - `modelship/infer/model_deployment.py` — the single `@serve.deployment` actor class; lazily imports the right backend from `config.loader`. - `modelship/infer/infer_config.py` — pydantic config schemas plus `RawRequestProxy` / `DisconnectRegistry`. `RawRequestProxy` exists because FastAPI `Request` can't cross Ray process boundaries. **Any new attribute vLLM reads from `raw_request` must be added there.** -- `modelship/infer/{vllm,transformers,diffusers,llama_cpp,custom}/` — one subdir per loader, each with an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. `modelship/infer/llama_server/llama_server_infer.py` is the exception: a single flat file (no `openai/` subpackage) — it proxies a `llama-server` subprocess's own OpenAI-compatible HTTP API rather than running modelship's parsers in-process. +- `modelship/infer/{vllm,transformers,diffusers,custom}/` — one subdir per loader, each with an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. `modelship/infer/llama_server/llama_server_infer.py` is the exception: a single flat file (no `openai/` subpackage) — it proxies a `llama-server` subprocess's own OpenAI-compatible HTTP API rather than running modelship's parsers in-process. - `modelship/plugins/base_plugin.py` — `BasePlugin` ABC that each plugin package subclasses as `ModelPlugin`. - `plugins/*` — workspace packages, each opt-in via a matching root extra. The plugin module name and extra name **must match** (`ensure_plugin()` does `importlib.import_module(config.plugin)`). @@ -66,14 +66,12 @@ Under `tests/`, `pytest-asyncio` for async. Tests **mock out Ray Serve** — the ## Sharp edges - `vllm==0.24.0` is pinned. Don't bump casually — TP scheduling in `mship_deploy.py:build_deployment_options` defaults to the Ray V2 executor, and the loader binds to vLLM-internal `entrypoints.*` module paths that upstream restructures between minors (the `vllm_infer.py` imports moved in 0.22/0.23). -- **GGUF is unsupported on the `vllm` loader.** 0.24 moved GGUF out of tree, and the only external `vllm-gguf-plugin` (`0.0.2`) has a stale `override_quantization_method` signature that breaks *every* quantized model on 0.24 — so it's deliberately not installed. `resolve_all_model_sources` (in `deploy/config.py`) rejects a `.gguf` on the vllm loader at driver preflight with a pointer to `llama_cpp`. Use `loader: llama_cpp` for GGUF; the vllm loader takes safetensors or AWQ/GPTQ/FP8 quants. **This is unconditional on GPU vs. CPU** — the CPU wheel (below) doesn't relax it either; a GGUF gemma still needs `llama_cpp`/`llama_server`, or a non-GGUF checkpoint to run on `vllm`. -- `vllm==0.24.0+cpu` is installable on the `cpu` extra via an explicit `vllm-cpu` index (`wheels.vllm.ai/0.24.0/cpu`, scoped through `tool.uv.sources` same as `llama-cpp-python`'s cu130 wheel) — the URL embeds the vLLM version, so a future bump must update it. On vLLM's CPU backend, `gpu_memory_utilization` is repurposed to mean *fraction of host RAM* reserved for KV cache, not VRAM; `normalize_num_gpus_and_tp` (`infer_config.py`) lowers its default to `0.4` for `num_gpus: 0` vllm deploys so a naive CPU config doesn't ask to reserve 90% of node RAM and crash at worker init — an explicit value always overrides it. See `config/examples/vllm-cpu.yaml`. -- `llama_cpp` GPU support relies on the prebuilt CUDA wheel from the `llama-cpp-cu130` index (`abetlen.github.io/llama-cpp-python/whl/cu130`), scoped to the `gpu` extra via `tool.uv.sources`. The `cpu` extra has no source entry, so it resolves the plain PyPI sdist (built from source) as before. Both extras pin the same `llama-cpp-python` floor so they stay in lockstep. -- The cu130 wheel's `libllama.so` statically links `libcuda.so.1` (the real NVIDIA driver library), unlike torch/vllm which resolve CUDA lazily — so merely `import llama_cpp` crashes on any driverless machine, including CI's `ubuntu-latest` runner. `ci.yml`'s `test` job installs NVIDIA's `cuda-driver-dev` stub `libcuda.so` (via a pinned `.deb`, extracted with `dpkg-deb`, no apt repo added) and symlinks it to `libcuda.so.1` on `LD_LIBRARY_PATH` — its `cuInit()` returns `CUDA_ERROR_STUB_LIBRARY` instead of crashing, which is enough for the import to succeed. If that pinned `.deb` URL ever 404s, grab a current `cuda-driver-dev-13-*` build from `developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/`. +- **GGUF is unsupported on the `vllm` loader.** 0.24 moved GGUF out of tree, and the only external `vllm-gguf-plugin` (`0.0.2`) has a stale `override_quantization_method` signature that breaks *every* quantized model on 0.24 — so it's deliberately not installed. `resolve_all_model_sources` (in `deploy/config.py`) rejects a `.gguf` on the vllm loader at driver preflight with a pointer to `llama_server`. Use `loader: llama_server` for GGUF; the vllm loader takes safetensors or AWQ/GPTQ/FP8 quants. **This is unconditional on GPU vs. CPU** — the CPU wheel (below) doesn't relax it either; a GGUF gemma still needs `llama_server`, or a non-GGUF checkpoint to run on `vllm`. +- `vllm==0.24.0+cpu` is installable on the `cpu` extra via an explicit `vllm-cpu` index (`wheels.vllm.ai/0.24.0/cpu`, scoped through `tool.uv.sources`) — the URL embeds the vLLM version, so a future bump must update it. On vLLM's CPU backend, `gpu_memory_utilization` is repurposed to mean *fraction of host RAM* reserved for KV cache, not VRAM; `normalize_num_gpus_and_tp` (`infer_config.py`) lowers its default to `0.4` for `num_gpus: 0` vllm deploys so a naive CPU config doesn't ask to reserve 90% of node RAM and crash at worker init — an explicit value always overrides it. See `config/examples/vllm-cpu.yaml`. - Metrics live on port **8079** (not 8000). `MSHIP_METRICS=false` or `--no-metrics` disables. When `mship_deploy` starts its own head (no `--use-existing-ray-cluster`), `connect_ray` pins that port via `ray.init(_metrics_export_port=…)` — a **private** Ray kwarg (accepted through `**kwargs`). A `TestConnectRay` test guards it so a Ray bump that drops it fails loudly. - `TRACE` is a custom log level below `DEBUG`; it logs full request/response payloads. - Docker CPU image uses the unified `Dockerfile` with `--build-arg MSHIP_VARIANT=cpu` and has a `:latest-cpu` tag suffix. -- `llama_server` loader launches a `llama-server` subprocess (found via `MSHIP_LLAMA_SERVER_BIN`, pinned in the Docker images at `/opt/llama.cpp`) and proxies its native OpenAI API — concurrency comes from `--parallel` slots instead of `llama_cpp`'s single `asyncio.Lock`. Same whole-GPU-only `num_gpus` rule as `llama_cpp`. Tool-call/reasoning parsing is llama-server's own, auto-detected per chat template: named-function `tool_choice` forcing is globally unsupported (silently falls back to `auto`), and `tool_choice: required` is grammar-enforced for harmony-style templates but a silent no-op for hermes-style ones (e.g. Qwen3); bare `response_format: {"type": "json_object"}` (no `schema` key) is also unenforced despite llama-server's own docs claiming support (verified against the b9859 binary directly) — `type: json_schema` requests, which is what modelship actually sends whenever a schema is given, are unaffected and correctly constrained. See `docs/model-configuration.md`'s llama_server section. No persistent on-disk prompt cache (llama-server has no equivalent to `llama_cpp`'s `cache: {type: disk}`). +- `llama_server` loader launches a `llama-server` subprocess (found via `MSHIP_LLAMA_SERVER_BIN`, pinned in the Docker images at `/opt/llama.cpp`) and proxies its native OpenAI API — concurrency comes from `--parallel` slots instead of a single `asyncio.Lock`. `num_gpus` must be `0` or a whole integer (fractional is rejected, llama.cpp has no VRAM-fraction knob). Tool-call/reasoning parsing is llama-server's own, auto-detected per chat template: named-function `tool_choice` forcing is globally unsupported (silently falls back to `auto`), and `tool_choice: required` is grammar-enforced for harmony-style templates but a silent no-op for hermes-style ones (e.g. Qwen3); bare `response_format: {"type": "json_object"}` (no `schema` key) is also unenforced despite llama-server's own docs claiming support (verified against the b9859 binary directly) — `type: json_schema` requests, which is what modelship actually sends whenever a schema is given, are unaffected and correctly constrained. See `docs/model-configuration.md`'s llama_server section. No persistent on-disk prompt cache. ## Further reading diff --git a/README.md b/README.md index 1cf1e5a..e814e88 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Python 3.12+](https://img.shields.io/badge/python-3.12+-blue.svg)](https://www.python.org/downloads/) -Self-hosted, OpenAI-compatible inference for the agentic era. Modelship runs your whole AI stack — reasoning LLMs with universal tool calling and the Responses API, plus embeddings, speech-to-text, text-to-speech, and image generation — as many models sharing your GPUs (or CPU) behind a single gateway. Built on [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) with pluggable inference backends: [vLLM](https://github.com/vllm-project/vllm) for high-throughput GPU inference, [HuggingFace Transformers](https://github.com/huggingface/transformers) for CPU and lightweight GPU workloads, [llama.cpp](https://github.com/abetlen/llama-cpp-python) for high-efficiency GGUF models on CPU, [Diffusers](https://github.com/huggingface/diffusers) for image generation, and a plugin system for custom backends. +Self-hosted, OpenAI-compatible inference for the agentic era. Modelship runs your whole AI stack — reasoning LLMs with universal tool calling and the Responses API, plus embeddings, speech-to-text, text-to-speech, and image generation — as many models sharing your GPUs (or CPU) behind a single gateway. Built on [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) with pluggable inference backends: [vLLM](https://github.com/vllm-project/vllm) for high-throughput GPU or CPU inference, [HuggingFace Transformers](https://github.com/huggingface/transformers) for CPU and lightweight GPU workloads, [llama.cpp](https://github.com/ggml-org/llama.cpp) for high-efficiency GGUF models on CPU or GPU, [Diffusers](https://github.com/huggingface/diffusers) for image generation, and a plugin system for custom backends. ## Why Modelship? @@ -13,7 +13,7 @@ Most self-hosted inference tools focus on running a single model. Modelship is f - **One server, many models** — run a full AI stack (chat + TTS + STT + embeddings + image gen) on a single machine instead of juggling separate services - **GPU memory control** — allocate exact GPU fractions per model (e.g. 70% for the LLM, 5% for TTS) so everything fits on your hardware - **Mix and match backends** — use vLLM for high-throughput GPU inference, Transformers or llama.cpp for CPU-only workloads, Diffusers for images, and plugins for custom backends — in the same deployment -- **Agentic-ready** — reasoning (`` → `reasoning_content`), universal tool/function calling, and the `/v1/responses` API work across the vLLM, llama.cpp, and Transformers loaders — the OpenAI-spec surface agents already target +- **Agentic-ready** — reasoning (`` → `reasoning_content`), universal tool/function calling, and the `/v1/responses` API work across the vLLM and llama.cpp (`llama_server`) loaders — the OpenAI-spec surface agents already target - **Drop-in OpenAI replacement** — any OpenAI SDK client works out of the box, making it easy to integrate with existing apps and tools like [Home Assistant](docs/home-assistant.md) ## Architecture @@ -41,7 +41,7 @@ graph TD EMB["Embedding Deployment
e.g. Nomic Embed
50% GPU"] end - subgraph CPU["CPU — Transformers / llama.cpp"] + subgraph CPU["CPU — Transformers / llama-server"] LLM_CPU["LLM Deployment
e.g. Qwen3-0.6B
CPU-only"] STT_CPU["STT Deployment
e.g. Whisper Small
CPU-only"] end @@ -55,8 +55,8 @@ Each model runs as an isolated [Ray Serve](https://docs.ray.io/en/latest/serve/i | Backend | Best for | GPU required | |---|---|---| -| **vLLM** | High-throughput chat, embeddings, transcription | Yes | -| **llama.cpp** | High-efficiency quantized GGUF models (chat, embeddings) | No | +| **vLLM** | High-throughput chat, embeddings, transcription | No — installs on GPU or CPU | +| **llama.cpp** (`llama_server`) | High-efficiency quantized GGUF models (chat, embeddings, vision) | No | | **Transformers** | Chat, embeddings, transcription, TTS on CPU or lightweight GPU | No | | **Diffusers** | Image generation | Yes | | **Custom (plugins)** | TTS backends (Kokoro ONNX, Bark, Orpheus), STT backends (whisper.cpp) | No | @@ -77,7 +77,7 @@ Models can be deployed across multiple GPUs, run on CPU-only, or both — multip - **CPU-only support** — run models without a GPU using the Transformers backend (chat, embeddings, transcription, TTS). Useful for development, testing, or small models that don't need GPU acceleration - **Multiple inference backends** — vLLM for high-throughput GPU inference, HuggingFace Transformers for CPU and lightweight GPU workloads, Diffusers for image generation, and a plugin system for custom backends - **Zero-downtime hot-reloads** — modify your `models.yaml` and run a cluster reconcile; changes are applied incrementally without interrupting the API gateway or unchanged models -- **Advanced agentic capabilities** — native support for DeepSeek-style reasoning (`` blocks parsed into `reasoning_content`) and universal tool/function calling across vLLM, GGUF (llama.cpp), and Transformers backends +- **Advanced agentic capabilities** — native support for DeepSeek-style reasoning (`` blocks parsed into `reasoning_content`) and universal tool/function calling across the vLLM, GGUF (`llama_server`), and Transformers backends - **Per-model isolated deployments** — each model runs in its own Ray Serve deployment with independent lifecycle, health checks, failure isolation, and configurable replica count - **OpenAI-compatible API** — drop-in replacement for any OpenAI SDK client - **Streaming** — SSE streaming for chat completions and TTS audio @@ -109,9 +109,9 @@ models: - name: reasoning-qwen model: "lmstudio-community/Qwen3-0.6B-GGUF:*Q4_K_M.gguf" usecase: generate - loader: llama_cpp + loader: llama_server num_cpus: 3 - llama_cpp_config: + llama_server_config: n_ctx: 4096 # Give reasoning space to think EOF diff --git a/config/examples/README.md b/config/examples/README.md index 6634a2c..364b2f1 100644 --- a/config/examples/README.md +++ b/config/examples/README.md @@ -4,20 +4,20 @@ Ready-to-run `models.yaml` configs for common scenarios. Mount one into the cont | File | What it runs | Hardware | |---|---|---| -| [llama-cpp.yaml](llama-cpp.yaml) | Quantized GGUF chat + embeddings | CPU (any arch) | | [llama-server.yaml](llama-server.yaml) | Quantized GGUF chat (concurrent), vision, embeddings via a llama-server subprocess | CPU or NVIDIA GPU | +| [vllm-cpu.yaml](vllm-cpu.yaml) | Quantized (AWQ/GPTQ) chat via vLLM's CPU backend — the path for gemma tool calling, which llama-server's parsers can't handle | CPU | | [transformers-cpu.yaml](transformers-cpu.yaml) | Llama 3.2 1B + Nomic embed + Whisper + MMS-TTS | CPU | | [vllm.yaml](vllm.yaml) | High-throughput chat with tool calling, embeddings, Whisper | NVIDIA GPU | | [diffusers.yaml](diffusers.yaml) | SDXL Turbo image generation | NVIDIA GPU | | [kokoro-tts.yaml](kokoro-tts.yaml) | Kokoro ONNX TTS with GPU + CPU fallback replicas | Mixed | | [full-stack.yaml](full-stack.yaml) | LLM + TTS + STT + embeddings on one GPU | NVIDIA GPU | -| [mini-pc.yaml](mini-pc.yaml) | Low-resource stack: llama.cpp chat + Kokoro ONNX TTS + whisper.cpp STT | CPU (e.g. Intel N100) | +| [mini-pc.yaml](mini-pc.yaml) | Low-resource stack: llama-server chat + Kokoro ONNX TTS + whisper.cpp STT | CPU (e.g. Intel N100) | Example: ```bash docker run --rm --shm-size=8g \ - -v ./config/examples/llama-cpp.yaml:/modelship/config/models.yaml \ + -v ./config/examples/llama-server.yaml:/modelship/config/models.yaml \ -v ./models-cache:/.cache \ -p 8000:8000 \ ghcr.io/alez007/modelship:latest-cpu diff --git a/config/examples/llama-cpp.yaml b/config/examples/llama-cpp.yaml deleted file mode 100644 index b5df6bb..0000000 --- a/config/examples/llama-cpp.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# llama.cpp loader examples — CPU-only GGUF inference. -# Ideal for quantized models on laptops, mini-PCs, and ARM hardware. - -models: - # Chat: GGUF pulled from a HuggingFace repo (glob after `:` picks the right quant) - - name: "qwen" - model: "lmstudio-community/Qwen2.5-7B-Instruct-GGUF:*Q4_K_M.gguf" - usecase: "generate" - loader: "llama_cpp" - num_cpus: 3 - - # Chat: GGUF from a local path (bind-mount the directory into the container) - - name: "llama3-local" - model: "/.cache/huggingface/llama-3-8b-instruct.Q4_K_M.gguf" - usecase: "generate" - loader: "llama_cpp" - num_cpus: 4 - llama_cpp_config: - n_ctx: 4096 - n_batch: 512 - - # Tool calling: FunctionGemma emits a custom `call:FUNC{...}` syntax. The - # tool-call parser is auto-detected from the model. Tool behavior is driven per - # request by the OpenAI `tool_choice` parameter — there is no deploy-level flag: - # - sending `tools` auto-constrains decoding with a GBNF grammar (caps the call - # count, pins the tool name, constrains enum args), stopping the runaway - # repetition small (270M) checkpoints fall into. Free-text stays reachable. - # - `tool_choice: "required"` (or a named function) forces a tool call by - # dropping the free-text branch — helps tiny checkpoints that decline to act. - - name: "function-gemma" - model: "/.cache/huggingface/functiongemma-270m-it.Q8_0.gguf" - usecase: "generate" - loader: "llama_cpp" - num_cpus: 2 - - # Embeddings - - name: "nomic-embed" - model: "nomic-ai/nomic-embed-text-v1.5-GGUF:nomic-embed-text-v1.5.Q4_K_M.gguf" - usecase: "embed" - loader: "llama_cpp" - num_cpus: 2 diff --git a/config/examples/llama-server.yaml b/config/examples/llama-server.yaml index e66ec36..6b69d1c 100644 --- a/config/examples/llama-server.yaml +++ b/config/examples/llama-server.yaml @@ -1,9 +1,8 @@ # llama_server loader examples — GGUF inference via a llama-server subprocess -# proxied over its native OpenAI-compatible HTTP API. Prefer this over -# llama_cpp for concurrent traffic: --parallel slots overlap requests instead -# of serializing behind a single lock. Requires MSHIP_LLAMA_SERVER_BIN to -# point at a llama-server executable (see docs/development.md); the Docker -# images ship a pinned build already. +# proxied over its native OpenAI-compatible HTTP API. --parallel slots let +# concurrent requests overlap instead of serializing behind a single lock. +# Requires MSHIP_LLAMA_SERVER_BIN to point at a llama-server executable (see +# docs/development.md); the Docker images ship a pinned build already. models: # Chat: GGUF pulled from a HuggingFace repo, 4 concurrent request slots. @@ -18,7 +17,7 @@ models: # Chat: GGUF from a local path (bind-mount the directory into the container). # Tool calling and reasoning parsing are llama-server's own, auto-detected # from the chat template — no modelship-level parser config. See - # docs/model-configuration.md for the tool_choice gaps vs. llama_cpp. + # docs/model-configuration.md for the tool_choice gaps vs. the OpenAI spec. - name: "llama3-local" model: "/.cache/huggingface/llama-3-8b-instruct.Q4_K_M.gguf" usecase: "generate" diff --git a/config/examples/mini-pc.yaml b/config/examples/mini-pc.yaml index ffb9ea3..b21d1b7 100644 --- a/config/examples/mini-pc.yaml +++ b/config/examples/mini-pc.yaml @@ -1,5 +1,5 @@ # Low-resource home-assistant stack for a mini-PC (e.g. Intel N100, 16 GB RAM). -# CPU-only: quantized chat via llama.cpp, Kokoro ONNX TTS, whisper.cpp STT. +# CPU-only: quantized chat via llama-server, Kokoro ONNX TTS, whisper.cpp STT. # Note: kokoroonnx needs espeak-ng on the host (apt-get install -y espeak-ng). models: @@ -7,9 +7,9 @@ models: - name: "qwen" model: "lmstudio-community/Qwen2.5-3B-Instruct-GGUF:*Q4_K_M.gguf" usecase: "generate" - loader: "llama_cpp" + loader: "llama_server" num_cpus: 2 - llama_cpp_config: + llama_server_config: n_ctx: 4096 # TTS — Kokoro on CPU via ONNX. diff --git a/config/examples/vllm-cpu.yaml b/config/examples/vllm-cpu.yaml index b2716bc..c3d8dfb 100644 --- a/config/examples/vllm-cpu.yaml +++ b/config/examples/vllm-cpu.yaml @@ -1,11 +1,11 @@ -# vLLM loader on CPU — no GPU required (Stage E0). +# vLLM loader on CPU — no GPU required. # # The main use case today: gemma models can't be tool-called through the -# llama.cpp-family loaders (llama_cpp/llama_server), but work fine through -# vllm. This is NOT a way to run your existing GGUF gemma file on CPU — the -# vllm loader rejects .gguf unconditionally, GPU or CPU. You need a non-GGUF -# checkpoint: safetensors, or an AWQ/GPTQ/compressed-tensors quant (CPU -# backend supports AWQ/GPTQ on x86, plus INT8 W8A8). Look for a +# llama_server loader (llama.cpp's own parsers don't support it), but work +# fine through vllm. This is NOT a way to run your existing GGUF gemma file +# on CPU — the vllm loader rejects .gguf unconditionally, GPU or CPU. You +# need a non-GGUF checkpoint: safetensors, or an AWQ/GPTQ/compressed-tensors +# quant (CPU backend supports AWQ/GPTQ on x86, plus INT8 W8A8). Look for a # community AWQ/GPTQ repack of the gemma model you want, or use the full # safetensors checkpoint if you have the RAM for it. # diff --git a/docs/architecture.md b/docs/architecture.md index a3db920..3405b6e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,27 +36,29 @@ Each deployment uses one of the following loaders: | Loader | Backend | Use cases | GPU required | |--------|---------|-----------|--------------| -| `vllm` | vLLM engine | Chat/generation, embeddings, transcription, translation | Yes | -| `llama_cpp` | llama-cpp-python | Chat/generation, embeddings (GGUF models) | No — runs on CPU or GPU (GGUF offload) | +| `vllm` | vLLM engine | Chat/generation, embeddings, transcription, translation | No — installs on GPU or CPU | +| `llama_server` | llama-server subprocess | Chat/generation, embeddings, vision (GGUF models) | No — runs on CPU or GPU (GGUF offload) | | `transformers` | PyTorch + HuggingFace | Chat/generation, embeddings, transcription, translation, TTS | No — runs on CPU or GPU | | `diffusers` | HuggingFace Diffusers | Image generation (any `AutoPipelineForText2Image` model) | Yes | | `stable_diffusion_cpp` | stable-diffusion.cpp | Image generation (GGUF models: SD1.5/SDXL/SD-Turbo, all-in-one Flux) | No — currently CPU-only | | `custom` | Plugin system | TTS backends (Kokoro ONNX, Bark, Orpheus), STT backends (whisper.cpp) | No | -The `transformers` loader is ideal for CPU-only deployments, smaller models, or development/testing without a GPU. It uses HuggingFace `pipeline()` under the hood and handles audio resampling automatically for speech-to-text models. The `llama_cpp` loader provides high-efficiency inference for quantized GGUF models on CPU or GPU (`n_gpu_layers` offload, whole GPUs only — fractional `num_gpus` is rejected). The `vllm` loader provides higher throughput on GPU with continuous batching and PagedAttention. +The `transformers` loader is ideal for CPU-only deployments, smaller models, or development/testing without a GPU. It uses HuggingFace `pipeline()` under the hood and handles audio resampling automatically for speech-to-text models. The `llama_server` loader provides high-efficiency inference for quantized GGUF models on CPU or GPU (`n_gpu_layers` offload, whole GPUs only — fractional `num_gpus` is rejected) by proxying a `llama-server` subprocess's own OpenAI-compatible API. The `vllm` loader provides higher throughput with continuous batching and PagedAttention, on GPU or CPU. ## Responses API (`/v1/responses`) -`/v1/responses` is implemented as a **stateless adapter at the gateway edge**, not a new inference path. The route translates a `ResponsesRequest` into a `ChatCompletionRequest`, runs it through the unchanged `handle.generate` deployment method, and translates the chat result back into the Responses shape. Because it reuses the chat pipeline, it works identically across every chat-capable loader (vLLM, transformers, llama_cpp) with zero loader-side code. +`/v1/responses` is shaped natively per loader, not via a chat-completions round trip. `BaseInfer.create_response(request, raw_request)` is a hookable method (defaulting to "not supported," like every other unimplemented capability); `VllmInfer` and `LlamaServerInfer` are the loaders that implement it, building the Responses envelope directly from their own parsed `(reasoning, content, tool_calls)` output — the same `ParsedChatOutput` seam `/v1/chat/completions` uses — rather than baking a `ChatCompletionResponse` and translating that back. `ModelDeployment.respond()` mirrors the existing `generate()` dispatch; the gateway route does a fail-fast validation pass and then calls straight through to it. -- **Non-streaming** — `chat_response_to_responses` maps `choices[].message` into `output[]` items (`reasoning` → reasoning item, content → message item, tool calls → `function_call` items) and remaps usage. -- **Streaming** (`stream: true`) — `ResponsesStreamTranslator` consumes the chat SSE chunk stream (the one wire shape all loaders share) and re-emits the Responses event protocol (`response.created` → `output_item.added` → `output_text.delta` / `reasoning_summary_text.delta` / `function_call_arguments.delta` → `output_item.done` → `response.completed`), tracking `output_index` / `sequence_number`. Output items are opened lazily on their first delta and closed at stream end, so the translation is independent of how a model interleaves reasoning, text, and tool calls. +- **Non-streaming** — `chat_utils.build_responses_items_from_parsed` maps the parsed tuple into `output[]` items (`reasoning` → reasoning item, content → message item, tool calls → `function_call` items), then `protocol/responses/adapter.build_response_object` builds the envelope and remaps usage. +- **Streaming** (`stream: true`) — `ResponsesStreamTranslator` is fed loader-native typed chunks directly (vLLM's `engine_ops.stream_chat_completion`, llama-server's own delta fields) and emits the Responses event protocol (`response.created` → `output_item.added` → `output_text.delta` / `reasoning_summary_text.delta` / `function_call_arguments.delta` → `output_item.done` → `response.completed`), tracking `output_index` / `sequence_number`. Output items are opened lazily on their first delta and closed at stream end, so the translation is independent of how a model interleaves reasoning, text, and tool calls. -**Supported:** text, reasoning (as a first-class `reasoning` output item), and client-driven tool calling (`function_call` / `function_call_output` round-trip), streaming and non-streaming. +**Supported:** text, reasoning (as a first-class `reasoning` output item), and client-driven tool calling (`function_call` / `function_call_output` round-trip), streaming and non-streaming — on the `vllm` and `llama_server` loaders. + +**404s on other loaders** (`transformers`, `diffusers`, `custom`) — there is no generic fallback; a loader must implement `create_response` itself. **Rejected with a clear 400** (rather than silently dropped): `previous_response_id`, `background`, and hosted built-in tools (e.g. `web_search`). These require server-side conversation state, which `/v1/responses` does not yet keep — `store` is accepted but never persisted (the response echoes `store: false`). Encrypted reasoning (`reasoning.encrypted_content`) is also not implemented. -Any chat-capable model works — no special `models.yaml` entry is needed: +Any `vllm` or `llama_server` model works — no special `models.yaml` entry is needed: ```python from openai import OpenAI diff --git a/docs/model-configuration.md b/docs/model-configuration.md index d1aabd5..4780e4d 100644 --- a/docs/model-configuration.md +++ b/docs/model-configuration.md @@ -112,7 +112,7 @@ Selection is **all-or-nothing**: the highest tier whose *complete* stack fits is | `name` | string | Model identifier used in API requests | | `model` | string | HuggingFace repo ID, local path, or `repo:filename` (see [Model source](#model-source)). Required for built-in loaders; optional for `loader: custom` | | `usecase` | string | `generate`, `embed`, `transcription`, `translation`, `tts`, or `image` | -| `loader` | string | `vllm`, `transformers`, `diffusers`, `llama_cpp`, `llama_server`, `stable_diffusion_cpp`, or `custom` | +| `loader` | string | `vllm`, `transformers`, `diffusers`, `llama_server`, `stable_diffusion_cpp`, or `custom` | | `plugin` | string | Plugin module name (required when `loader: custom`); automatically loaded from wheels when referenced | | `num_gpus` | float \| int | GPU allocation. Fractional `< 1` shares one GPU (also sets vLLM `gpu_memory_utilization`); integer `≥ 1` requests that many whole GPUs (for `vllm`, this auto-sets `tensor_parallel_size = num_gpus` unless tp/pp is already specified). | | `num_cpus` | float | CPU units to allocate (default `0.1`) | @@ -122,11 +122,10 @@ Selection is **all-or-nothing**: the highest tier whose *complete* stack fits is | `vllm_engine_kwargs` | object | Passed directly to the vLLM engine (see below) | | `transformers_config` | object | Transformers loader options (see below) | | `diffusers_config` | object | Diffusers pipeline options (see below) | -| `llama_cpp_config` | object | llama.cpp loader options (see below) | | `llama_server_config` | object | llama-server loader options (see below) | | `stable_diffusion_cpp_config` | object | stable-diffusion.cpp loader options (see below) | | `plugin_config` | object | Plugin-specific options passed through to the plugin | -| `chat_template_kwargs` | object | Extra variables forwarded into the chat-template render on text loaders (`vllm`, `transformers`, `llama_cpp`) — e.g. `enable_thinking: false` for Qwen3. Only has an effect if the model's template branches on the key; ignored on paths that bypass the template (llama.cpp's native chat-handler fallback when `chat_format` is set or no template resolves). A per-request `chat_template_kwargs` overrides the model default on `vllm`. | +| `chat_template_kwargs` | object | Extra variables forwarded into the chat-template render on text loaders (`vllm`, `transformers`) — e.g. `enable_thinking: false` for Qwen3. Only has an effect if the model's template branches on the key. A per-request `chat_template_kwargs` overrides the model default on `vllm`. | ## Model source @@ -192,15 +191,15 @@ The `vllm` loader supports chat/generation, embeddings, transcription, and trans | `kv_cache_dtype` | string | — | KV cache dtype (e.g. `fp8`) | > **GGUF is not supported on the `vllm` loader.** vLLM 0.24 dropped in-tree GGUF, so -> pointing the vllm loader at a `.gguf` is rejected at startup. Use `loader: llama_cpp` +> pointing the vllm loader at a `.gguf` is rejected at startup. Use `loader: llama_server` > for GGUF models; the vllm loader takes safetensors checkpoints or AWQ/GPTQ/FP8 quants. > This is unconditional regardless of GPU vs. CPU — see below. ### CPU (no GPU required) The `vllm` loader also installs on the `cpu` extra (`num_gpus: 0`). The main use case is -gemma models, which the llama.cpp-family loaders (`llama_cpp`/`llama_server`) can't -tool-call — vLLM handles gemma tool-calling correctly, GPU or CPU. This is **not** a way +gemma models, which the `llama_server` loader can't tool-call (llama.cpp's own parsers +don't support it) — vLLM handles gemma tool-calling correctly, GPU or CPU. This is **not** a way to run an existing GGUF gemma file on CPU: the GGUF rejection above applies here too, so you need a non-GGUF checkpoint (safetensors, or an AWQ/GPTQ/compressed-tensors quant — the CPU backend supports AWQ/GPTQ on x86 plus INT8 W8A8). @@ -420,94 +419,11 @@ models: guidance_scale: 0.0 ``` -## llama.cpp Loader - -The `llama_cpp` loader uses [llama-cpp-python](https://github.com/abetlen/llama-cpp-python) to run GGUF models. It supports **CPU or GPU inference** (GGUF offload via `n_gpu_layers`). `num_gpus` must be `0` (CPU-only, `n_gpu_layers` forced to `0`) or a whole integer number of GPUs — fractional `num_gpus` is rejected at config time, since llama.cpp has no VRAM-fraction knob. With `num_gpus >= 1` on the GPU build, `n_gpu_layers` is honored (default `-1` = offload all layers); multi-GPU deploys use llama.cpp's default even layer split across the visible GPUs (`tensor_split`, `split_mode`, and `main_gpu` are available via `model_kwargs` for finer control). This loader is ideal for running quantized models efficiently, including on hardware without dedicated GPUs. - -| Field | Type | Default | Description | -|---|---|---|---| -| `n_ctx` | int | `2048` | Maximum sequence length | -| `n_batch` | int | `512` | Batch size for prompt processing | -| `n_gpu_layers` | int | `-1` | Layers to offload to GPU (`-1` = all). Honored when `num_gpus >= 1` on a GPU-capable build; forced to `0` otherwise | -| `chat_format` | string | — | Chat template format (e.g. `llama-3`) | -| `model_kwargs` | object | `{}` | Extra keyword arguments passed to the `Llama` constructor (e.g. `tensor_split`, `split_mode`, `main_gpu`) | -| `cache` | object | — | llama.cpp's native prompt-state cache (see below). Omit to disable. | - -> **Note:** Setting `MSHIP_LOG_LEVEL` to `TRACE` will enable `verbose` mode in the underlying llama.cpp engine. - -#### Prompt cache (`cache`) - -When set, llama.cpp's native prompt-state cache is attached to the model (via `Llama.set_cache`). It stores the model's evaluated KV state keyed by prompt prefix, so a later request that shares a prefix skips re-evaluating it — useful for repeated system prompts or long shared contexts. - -| Field | Type | Default | Description | -|---|---|---|---| -| `type` | string | `ram` | `ram` keeps states in process memory; `disk` persists them (survives replica restarts, at the cost of disk I/O) | -| `capacity` | string/int | `2GiB` | Eviction ceiling for cached states. Accepts a human-readable size — `2GiB`, `512MB`, `1.5gb` — or a bare byte count. Decimal units (`KB`/`MB`/`GB`/`TB`) are powers of 1000; binary units (`KiB`/`MiB`/`GiB`/`TiB`) powers of 1024 | - -The `disk` cache is stored under `$MSHIP_CACHE_DIR/llama_cache/` (default `/.cache`), keyed by the deployment name (model name + config fingerprint + gateway). This isolates the store per model configuration version and per gateway, so a different model, a changed config, or another gateway never cross-loads — or concurrently corrupts — an incompatible cache. The fingerprint is stable across redeploys of the same config, so persistence holds. - -> **Note:** `type: disk` requires a single replica. llama.cpp's on-disk cache has no file locking, so replicas sharing the store would corrupt it — combining `disk` with `num_replicas > 1` (or an `autoscaling_config` whose `max_replicas > 1`) is a config error that stops the deploy. Use `type: ram` (per-process, always safe) for multi-replica models. - -```yaml -models: - - name: "qwen-gguf-hf" - model: "lmstudio-community/Qwen2.5-7B-Instruct-GGUF:*Q4_K_M.gguf" - usecase: "generate" - loader: "llama_cpp" - llama_cpp_config: - cache: - type: disk - capacity: 4GiB -``` - -#### Tool calling (`tool_choice`) - -Tool-call behavior is driven entirely by the per-request OpenAI `tool_choice` parameter — there is no deploy-level flag. Sending a `tools` array is the opt-in. - -When a request carries `tools` and the model has a usable `tool_call_parser`, decoding is automatically constrained by a [GBNF](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md) grammar compiled from those tool schemas (it caps the call count, pins `name` to a real tool, and constrains `arguments` to each tool's JSON schema — preventing malformed envelopes, invented fields, and runaway repetition). `tool_choice` selects the grammar's root: - -- `auto` (default) — the model may answer in free text **or** call tools. The free-text branch cannot contain a literal `<`, so it is skipped on a reasoning-enabled deployment (which would emit ``); tool calls are still parsed from the raw output. -- `required` / `{"type":"function","function":{"name":"X"}}` — the free-text branch is dropped, so the model **must** call a tool. Because that root cannot emit ``, it is enforced even on a reasoning-capable model. -- `none` — tools are suppressed entirely. - -Per-loader enforcement: **vLLM** honors `tool_choice` natively. **llama_cpp** enforces `required`/named via the grammar above (when a grammar-emitting parser is resolved — currently the JSON family `hermes` and the Gemma family; others are left unconstrained, logged once). **transformers** has no constrained decoding, so `required`/named are best-effort (tools are passed and the model is trusted). **llama_server** delegates entirely to llama-server's own grammar, which is per-model-family rather than a fixed modelship rule — see the [llama_server Loader](#llama_server-loader) section below for the specifics and gaps. - -Caveats: the grammar enforces *structure*, not *choice* (it can't pick the right value); JSON-schema numeric bounds aren't expressible as GBNF ranges (digit shape only); and the tool-call grammar takes precedence over a `response_format` grammar on the same request — the two cannot be combined. - -#### Reasoning detection - -A model's reasoning parser is auto-detected from its chat template. After load, each loader renders a generation prompt with the deployment's effective `chat_template_kwargs` and confirms whether reasoning is actually emitted; if you suppress it (e.g. `chat_template_kwargs: {enable_thinking: false}` on Qwen3), the parser is disabled — speeding up inference and letting `tool_choice=auto` fully constrain. There is no modelship-level reasoning flag. - -GGUF variants in a HuggingFace repo are picked via the `:filename` syntax on the -`model:` field (see [Model source](#model-source)). The selector is a glob and -must match exactly one file. - -### Chat / Text Generation (GGUF) - -```yaml -models: - - name: "qwen-gguf-hf" - model: "lmstudio-community/Qwen2.5-7B-Instruct-GGUF:*Q4_K_M.gguf" - usecase: "generate" - loader: "llama_cpp" - num_cpus: 3 -``` - -### Embeddings (GGUF) - -```yaml -models: - - name: nomic-embed - model: "nomic-ai/nomic-embed-text-v1.5-GGUF:nomic-embed-text-v1.5.Q4_K_M.gguf" - usecase: embed - loader: llama_cpp -``` - ## llama_server Loader -The `llama_server` loader runs GGUF models by launching a [`llama-server`](https://github.com/ggml-org/llama.cpp) subprocess and proxying its native OpenAI-compatible HTTP API, instead of binding llama.cpp in-process like `llama_cpp` does. Chat templating, tool-call parsing, and reasoning parsing are all llama-server's own (`--jinja --reasoning-format auto`), not modelship's. This trades modelship's own parser coverage for llama-server's `--parallel` request slots, so concurrent requests actually overlap instead of serializing behind a single lock (as `llama_cpp` does) — prefer this loader over `llama_cpp` for any deployment expecting concurrent traffic. It requires the `llama-server` binary to be discoverable via `MSHIP_LLAMA_SERVER_BIN` (see [development.md](development.md#llama-server-binary-llama_server-loader)); the Docker images ship a pinned build at `/opt/llama.cpp`. +The `llama_server` loader runs GGUF models by launching a [`llama-server`](https://github.com/ggml-org/llama.cpp) subprocess and proxying its native OpenAI-compatible HTTP API. Chat templating, tool-call parsing, and reasoning parsing are all llama-server's own (`--jinja --reasoning-format auto`), not modelship's. `--parallel` request slots let concurrent requests actually overlap instead of serializing behind a single lock. It requires the `llama-server` binary to be discoverable via `MSHIP_LLAMA_SERVER_BIN` (see [development.md](development.md#llama-server-binary-llama_server-loader)); the Docker images ship a pinned build at `/opt/llama.cpp`. -Like `llama_cpp`, `num_gpus` must be `0` (CPU-only) or a whole integer number of GPUs — fractional is rejected at config time. +`num_gpus` must be `0` (CPU-only) or a whole integer number of GPUs — fractional is rejected at config time, since llama.cpp has no VRAM-fraction knob. | Field | Type | Default | Description | |---|---|---|---| @@ -519,7 +435,7 @@ Like `llama_cpp`, `num_gpus` must be `0` (CPU-only) or a whole integer number of | `mmproj` | string | — | Multimodal projector file/repo ref (e.g. a CLIP model) for vision models — see [Vision](#vision-gguf) below | | `extra_args` | list[string] | `[]` | Escape hatch: extra flags appended verbatim to the `llama-server` launch command | -> **Note:** the `cache` block (`llama_cpp`'s persistent on-disk prompt cache) is **not supported**. llama-server manages request-level prefix caching internally and automatically within its slots (comparable to `llama_cpp`'s `type: ram` cache), but has no equivalent to modelship's restart-persistent `type: disk` cache. +> **Note:** there is no persistent on-disk prompt cache. llama-server manages request-level prefix caching internally and automatically within its slots, but has no equivalent to modelship's restart-persistent disk cache. ```yaml models: @@ -531,7 +447,7 @@ models: parallel: 4 ``` -### Tool calling and reasoning gaps vs. `llama_cpp` +### Tool calling and reasoning gaps vs. the OpenAI spec llama-server auto-detects both the tool-call and reasoning parser from the model's chat template — there is no modelship-level override. Two gaps are real and per-model-family, not per-loader, so test against the specific model in use before relying on either: @@ -539,7 +455,7 @@ llama-server auto-detects both the tool-call and reasoning parser from the model - **`tool_choice: required` enforcement depends on the model's chat template family.** It's grammar-enforced for harmony-style templates (e.g. gpt-oss) but a silent no-op for hermes-style templates (e.g. Qwen3) — the model may still answer in free text with no error. - **Bare `response_format: {"type": "json_object"}` (no `schema` key) is not enforced**, despite llama-server's own docs describing it as supported "plain JSON output" — verified directly against the b9859 binary. The model can answer in free text with no error. This doesn't affect `type: json_schema` requests (which modelship sends whenever a schema is given, e.g. structured outputs) — those carry a `schema` and llama-server does constrain them correctly. -Two wins over `llama_cpp`, also verified per-model-family: `response_format`/`json_schema` can be combined with reasoning in the same request (`llama_cpp` rejects that combination outright), and `logprobs`/`top_logprobs` are forwarded and returned (`llama_cpp` doesn't support either). +`response_format`/`json_schema` can be combined with reasoning in the same request, and `logprobs`/`top_logprobs` are forwarded and returned. ### Vision (GGUF) @@ -567,7 +483,7 @@ models: ## stable-diffusion.cpp Loader -The `stable_diffusion_cpp` loader uses [stable-diffusion.cpp](https://github.com/leejet/stable-diffusion.cpp) (via [stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python)) for **CPU-only image generation** — the image counterpart to the `llama_cpp` text loader. It runs GGUF-quantized single-file diffusion checkpoints (SD1.5, SDXL, SD-Turbo, all-in-one Flux) in a few GB of RAM, with no GPU. Any `num_gpus` is ignored (a warning is logged and the actor is allocated `num_gpus: 0`). `usecase` is always `image` (defaulted if omitted) and it serves `/v1/images/generations`, `/v1/images/edits`, and `/v1/images/variations`. +The `stable_diffusion_cpp` loader uses [stable-diffusion.cpp](https://github.com/leejet/stable-diffusion.cpp) (via [stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python)) for **CPU-only image generation**. It runs GGUF-quantized single-file diffusion checkpoints (SD1.5, SDXL, SD-Turbo, all-in-one Flux) in a few GB of RAM, with no GPU. Any `num_gpus` is ignored (a warning is logged and the actor is allocated `num_gpus: 0`). `usecase` is always `image` (defaulted if omitted) and it serves `/v1/images/generations`, `/v1/images/edits`, and `/v1/images/variations`. | Field | Type | Default | Description | |---|---|---|---| diff --git a/docs/plugins.md b/docs/plugins.md index 0baa827..d40ae93 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -61,7 +61,7 @@ module-root = "" #### Dependency contract -Plugins assume the host environment provides `modelship` itself plus the full core/gpu/cpu stack: `torch`, `torchvision`, `transformers`, `sentence-transformers`, `numpy`, `scipy`, `librosa`, `soundfile`, `llama-cpp-python`, `onnxruntime[-gpu]`, `diffusers`, `accelerate`, `vllm`. **Do not redeclare any of these in your plugin's `dependencies`.** +Plugins assume the host environment provides `modelship` itself plus the full core/gpu/cpu stack: `torch`, `torchvision`, `transformers`, `sentence-transformers`, `numpy`, `scipy`, `librosa`, `soundfile`, `onnxruntime[-gpu]`, `diffusers`, `accelerate`, `vllm`. **Do not redeclare any of these in your plugin's `dependencies`.** Why: plugin wheels are shipped to Ray workers via `runtime_env`, which installs them into a per-job venv layered over a base image that already has the core stack baked in. Redeclaring `modelship` or `torch` causes pip to either pull a second copy from PyPI (version drift, two packages on `sys.path`) or error on the layered install. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 7b6f936..9920fc8 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -26,10 +26,6 @@ If you previously used the old `/root/.cache/huggingface` mount path, switch to Ray's object store needs shared memory. Always pass `--shm-size=8g` (or larger for big models). Without it, you'll see Ray worker crashes or silent hangs during deployment. -## `n_gpu_layers` ignored with llama_cpp loader - -`n_gpu_layers` is honored on the GPU build (`MSHIP_VARIANT=gpu`) when the model's `num_gpus` is set to `1` or more (a whole integer — fractional `num_gpus` is rejected at config time, since llama.cpp has no VRAM-fraction knob). It's forced to `0` and a warning is logged when either `num_gpus` is `0`, or the installed `llama-cpp-python` was built without GPU support (e.g. the CPU image's plain PyPI wheel). - ## arm64 vs amd64 image selection Both `ghcr.io/alez007/modelship:latest` and `:latest-cpu` are multi-arch since v0.1.25. Docker picks the right one automatically for your host. If you need to force an arch (e.g. cross-building), use `--platform linux/arm64` or `linux/amd64`. diff --git a/modelship/deploy/config.py b/modelship/deploy/config.py index d127a67..51300c8 100644 --- a/modelship/deploy/config.py +++ b/modelship/deploy/config.py @@ -25,21 +25,11 @@ def _is_explicit_tool_opt_out(cfg) -> bool: - vllm: ``enable_auto_tool_choice: false`` — user disabled tool calling. - transformers: ``tool_calls_enabled: false`` — user disabled tool calling. - - llama_cpp: ``tool_calls_enabled: false`` (disabled) OR ``chat_format`` set - (user wants llama-cpp-python's own function-calling handler — we must not - also wire up our parser). """ if cfg.loader == ModelLoader.vllm: return cfg.vllm_engine_kwargs is not None and cfg.vllm_engine_kwargs.enable_auto_tool_choice is False if cfg.loader == ModelLoader.transformers: return cfg.transformers_config is not None and cfg.transformers_config.tool_calls_enabled is False - if cfg.loader == ModelLoader.llama_cpp: - if cfg.llama_cpp_config is None: - return False - if cfg.llama_cpp_config.tool_calls_enabled is False: - return True - if cfg.llama_cpp_config.chat_format is not None: - return True return False @@ -166,13 +156,15 @@ def resolve_all_model_sources(yml_conf: ModelshipConfig) -> None: # GGUF is not supported on the vllm loader: vLLM 0.24 moved GGUF out of # tree, and the only external plugin is incompatible with 0.24's - # quantization API. Reject early with a pointer to llama_cpp instead of - # letting vLLM misparse the .gguf as a config.json deep in engine init. + # quantization API. Reject early with a pointer to llama_server instead + # of letting vLLM misparse the .gguf as a config.json deep in engine init. if cfg.loader == ModelLoader.vllm and cfg._resolved_path.lower().endswith(".gguf"): raise ValueError( f"Model '{cfg.name}' resolves to a GGUF file, which the vllm loader does not support " - f"(vLLM 0.24 dropped in-tree GGUF). Use `loader: llama_cpp` for GGUF models, or point " - f"the vllm loader at a non-GGUF checkpoint (safetensors, or an AWQ/GPTQ/FP8 quant)." + f"(vLLM 0.24 dropped in-tree GGUF). Use `loader: llama_server` for GGUF models, or point " + f"the vllm loader at a non-GGUF checkpoint (safetensors, or an AWQ/GPTQ/FP8 quant — this is " + f"also the supported path for gemma models, whose tool calling llama.cpp's parsers can't " + f"handle; see config/examples/vllm-cpu.yaml)." ) @@ -184,7 +176,8 @@ def resolve_all_tool_parsers(yml_conf: ModelshipConfig) -> None: onto `_resolved_tool_call_parser` so loader code has a single source of truth and never re-implements the precedence. - Auto-detection runs for vllm, transformers, and llama_cpp. diffusers has + Auto-detection runs for vllm and transformers. llama_server does its own + tool-call detection internally, unrelated to this registry; diffusers has no chat path; custom is plugin-managed. Behavior per model: @@ -198,7 +191,7 @@ def resolve_all_tool_parsers(yml_conf: ModelshipConfig) -> None: """ registered = set(available_parsers()) for cfg in yml_conf.models: - if cfg.loader not in (ModelLoader.vllm, ModelLoader.transformers, ModelLoader.llama_cpp): + if cfg.loader not in (ModelLoader.vllm, ModelLoader.transformers): continue if cfg.usecase != ModelUsecase.generate: continue @@ -298,7 +291,7 @@ def resolve_all_reasoning_parsers(yml_conf: ModelshipConfig) -> None: - Not detected: leaves None (reasoning disabled). """ for cfg in yml_conf.models: - if cfg.loader not in (ModelLoader.vllm, ModelLoader.transformers, ModelLoader.llama_cpp): + if cfg.loader not in (ModelLoader.vllm, ModelLoader.transformers): continue if cfg.usecase != ModelUsecase.generate: continue diff --git a/modelship/deploy/profiles/catalog.py b/modelship/deploy/profiles/catalog.py index ddf5a3a..ca439f5 100644 --- a/modelship/deploy/profiles/catalog.py +++ b/modelship/deploy/profiles/catalog.py @@ -96,7 +96,7 @@ def draws_from_vram(self) -> bool: _GENERATE_CPU = ( ModelSpec( model="bartowski/Qwen2.5-1.5B-Instruct-GGUF:*Q4_K_M.gguf", - loader=ModelLoader.llama_cpp, + loader=ModelLoader.llama_server, usecase=ModelUsecase.generate, footprint_bytes=int(1.5 * _GiB), req_min=_req(1, 1.8, 18), @@ -104,7 +104,7 @@ def draws_from_vram(self) -> bool: ), ModelSpec( model="bartowski/Llama-3.2-3B-Instruct-GGUF:*Q4_K_M.gguf", - loader=ModelLoader.llama_cpp, + loader=ModelLoader.llama_server, usecase=ModelUsecase.generate, footprint_bytes=int(3.0 * _GiB), req_min=_req(2, 3.0, 30), @@ -112,7 +112,7 @@ def draws_from_vram(self) -> bool: ), ModelSpec( model="bartowski/Qwen2.5-7B-Instruct-GGUF:*Q4_K_M.gguf", - loader=ModelLoader.llama_cpp, + loader=ModelLoader.llama_server, usecase=ModelUsecase.generate, footprint_bytes=int(6.0 * _GiB), req_min=_req(4, 5.0, 45), @@ -120,7 +120,7 @@ def draws_from_vram(self) -> bool: ), ModelSpec( model="bartowski/Qwen2.5-14B-Instruct-GGUF:*Q4_K_M.gguf", - loader=ModelLoader.llama_cpp, + loader=ModelLoader.llama_server, usecase=ModelUsecase.generate, footprint_bytes=int(11.0 * _GiB), req_min=_req(6, 9.5, 70), @@ -226,7 +226,7 @@ def draws_from_vram(self) -> bool: _EMBED = ( ModelSpec( model="nomic-ai/nomic-embed-text-v1.5-GGUF:*f16.gguf", - loader=ModelLoader.llama_cpp, + loader=ModelLoader.llama_server, usecase=ModelUsecase.embed, footprint_bytes=int(0.6 * _GiB), req_min=_req(0.5, 0.6, 7), diff --git a/modelship/deploy/profiles/generator.py b/modelship/deploy/profiles/generator.py index c5466a4..3704a42 100644 --- a/modelship/deploy/profiles/generator.py +++ b/modelship/deploy/profiles/generator.py @@ -37,7 +37,7 @@ # Maps a built-in loader to the models.yaml field its inner config lives under. _LOADER_CONFIG_FIELD = { - ModelLoader.llama_cpp: "llama_cpp_config", + ModelLoader.llama_server: "llama_server_config", ModelLoader.stable_diffusion_cpp: "stable_diffusion_cpp_config", ModelLoader.diffusers: "diffusers_config", ModelLoader.vllm: "vllm_engine_kwargs", diff --git a/modelship/deploy/profiles/selector.py b/modelship/deploy/profiles/selector.py index 1cbd3c9..ddf4e15 100644 --- a/modelship/deploy/profiles/selector.py +++ b/modelship/deploy/profiles/selector.py @@ -35,7 +35,7 @@ logger = get_logger("deploy.profiles.selector") # Fraction of free RAM a stack may occupy — headroom for the OS, page cache, and -# KV-cache growth. Matches the llama_cpp preflight's RAM util. +# KV-cache growth. Matches the llama_server preflight's RAM util. _UTILIZATION = 0.8 # Absolute floor: below this we refuse outright (the user's "don't even try"). diff --git a/modelship/infer/infer_config.py b/modelship/infer/infer_config.py index b0fd26c..44bff3b 100644 --- a/modelship/infer/infer_config.py +++ b/modelship/infer/infer_config.py @@ -1,6 +1,5 @@ import asyncio import hashlib -import re import time from collections.abc import Callable from enum import StrEnum @@ -8,7 +7,7 @@ import ray from fastapi import Request -from pydantic import BaseModel, Field, PrivateAttr, field_validator, model_validator +from pydantic import BaseModel, Field, PrivateAttr, model_validator from ray.exceptions import RayActorError from starlette.datastructures import Headers, State @@ -50,7 +49,6 @@ class ModelLoader(StrEnum): vllm = "vllm" transformers = "transformers" diffusers = "diffusers" - llama_cpp = "llama_cpp" llama_server = "llama_server" stable_diffusion_cpp = "stable_diffusion_cpp" custom = "custom" @@ -104,77 +102,9 @@ class DiffusersConfig(BaseModel): guidance_scale: float = 7.5 -_SIZE_UNITS = { - "b": 1, - "kb": 1000, - "mb": 1000**2, - "gb": 1000**3, - "tb": 1000**4, - "kib": 1 << 10, - "mib": 1 << 20, - "gib": 1 << 30, - "tib": 1 << 40, -} -_SIZE_RE = re.compile(r"^\s*([0-9]*\.?[0-9]+)\s*([a-z]*)\s*$", re.IGNORECASE) - - -def parse_size(value: str | int) -> int: - """Parse a human-readable byte size ('2GiB', '512MB', '1.5gb') into bytes. - A bare number (or int) is taken as bytes. Decimal units (KB/MB/GB/TB) are - powers of 1000; binary units (KiB/MiB/GiB/TiB) powers of 1024.""" - if isinstance(value, int): - size = value - else: - match = _SIZE_RE.match(value) - if not match: - raise ValueError(f"invalid size '{value}'; use e.g. '2GiB', '512MB', or a byte count") - number, unit = match.groups() - multiplier = _SIZE_UNITS.get((unit or "b").lower()) - if multiplier is None: - raise ValueError(f"invalid size unit in '{value}'; expected one of B/KB/MB/GB/TB or KiB/MiB/GiB/TiB") - size = int(float(number) * multiplier) - if size <= 0: - raise ValueError(f"size must be positive, got {size} bytes") - return size - - -class LlamaCppCacheConfig(BaseModel): - """llama.cpp's native prompt-state cache (via `Llama.set_cache`). Stores the - evaluated KV state keyed by prompt prefix, so a later request sharing a prefix - skips re-evaluating it. `ram` keeps states in process memory; `disk` persists - them under MSHIP_CACHE_DIR (survives replica restarts at the cost of disk I/O).""" - - type: Literal["ram", "disk"] = "ram" - # Eviction ceiling for cached states, as a human-readable size ('2GiB', - # '512MB') or a bare byte count. Default 2 GiB matches llama-cpp-python. - capacity: str | int = "2GiB" - - @field_validator("capacity") - @classmethod - def _check_capacity(cls, value: str | int) -> str | int: - parse_size(value) # raises on invalid size/unit or non-positive - return value - - @property - def capacity_bytes(self) -> int: - return parse_size(self.capacity) - - -class LlamaCppConfig(BaseModel): - n_gpu_layers: int = -1 - n_ctx: int = 2048 - n_batch: int = 512 - chat_format: str | None = None - model_kwargs: dict[str, Any] = Field(default_factory=dict) - tool_calls_enabled: bool | None = None - # llama.cpp's native prompt-state cache; None disables it. - cache: LlamaCppCacheConfig | None = None - - class LlamaServerConfig(BaseModel): """Tunables for the ``llama_server`` loader, which drives a `llama-server` - subprocess over its native OpenAI-compatible HTTP API rather than binding - llama.cpp in-process (that's `llama_cpp`/`LlamaCppConfig`).""" + subprocess over its native OpenAI-compatible HTTP API.""" n_ctx: int = 2048 n_batch: int = 512 @@ -287,14 +217,12 @@ class ModelshipModelConfig(BaseModel): vllm_engine_kwargs: VllmEngineConfig = Field(default_factory=VllmEngineConfig) transformers_config: TransformersConfig | None = None diffusers_config: DiffusersConfig | None = None - llama_cpp_config: LlamaCppConfig | None = None llama_server_config: LlamaServerConfig | None = None stable_diffusion_cpp_config: StableDiffusionCppConfig | None = None plugin_config: dict[str, Any] | None = None # plugin devs parse this themselves # Extra variables forwarded verbatim into the chat-template Jinja render on # every text loader (e.g. `enable_thinking: false` for Qwen3). Only does - # something if the model's template branches on the key; ignored on paths - # that bypass the template (llama_cpp's native chat-handler fallback). + # something if the model's template branches on the key. chat_template_kwargs: dict[str, Any] = Field(default_factory=dict) _resolved_path: str | None = PrivateAttr(default=None) @@ -338,34 +266,10 @@ def check_autoscaling_excludes_num_replicas(self): return self @model_validator(mode="after") - def check_disk_cache_single_replica(self): - # llama.cpp's LlamaDiskCache reads/writes/prunes its SQLite store with no - # file locking. Replicas of the same model share one cache dir, so >1 - # replica races into corruption. RAM cache is per-process and always safe. - if self.loader != ModelLoader.llama_cpp: - return self - if self.llama_cpp_config is None or self.llama_cpp_config.cache is None: - return self - if self.llama_cpp_config.cache.type != "disk": - return self - multi_replica = self.num_replicas > 1 or ( - self.autoscaling_config is not None and self.autoscaling_config.max_replicas > 1 - ) - if multi_replica: - raise ValueError( - f"model '{self.name}': llama_cpp_config.cache.type='disk' cannot be combined with " - f"multiple replicas (num_replicas > 1 or autoscaling max_replicas > 1) — the disk " - f"cache is not process-safe and replicas would corrupt a shared store. Use " - f"cache.type='ram' (per-process, safe) or keep a single replica." - ) - return self - - @model_validator(mode="after") - def validate_llama_cpp_num_gpus(self): + def validate_llama_server_num_gpus(self): # llama.cpp has no VRAM-fraction knob, so a fractional GPU share can't be - # honored — require whole GPUs (or 0 for CPU). Applies equally to - # llama_server, which drives the same llama.cpp engine out of process. - if self.loader in (ModelLoader.llama_cpp, ModelLoader.llama_server) and self.num_gpus != int(self.num_gpus): + # honored — require whole GPUs (or 0 for CPU). + if self.loader == ModelLoader.llama_server and self.num_gpus != int(self.num_gpus): raise ValueError( f"num_gpus={self.num_gpus!r} is not allowed for the {self.loader.value} loader: " f"use an integer number of whole GPUs, or 0 for CPU. Fractional GPU " diff --git a/modelship/infer/llama_cpp/capabilities.py b/modelship/infer/llama_cpp/capabilities.py deleted file mode 100644 index 3d5d2fd..0000000 --- a/modelship/infer/llama_cpp/capabilities.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Modality capability detection for a loaded llama.cpp ``Llama`` instance.""" - -from dataclasses import dataclass - -from llama_cpp import Llama - - -@dataclass(frozen=True) -class LlamaCppCapabilities: - """Modalities the underlying ``Llama`` instance can ingest.""" - - supports_image: bool - supports_audio: bool = False # llama-cpp-python ships no native audio chat handlers today - - @classmethod - def detect(cls, llama: Llama) -> "LlamaCppCapabilities": - # The only reliable signal that a Llama instance can ingest images is whether - # the user wired up a multimodal chat_handler at load time - # (e.g. Llava15ChatHandler, Llava16ChatHandler, MoondreamChatHandler). - # Those handlers expose `load_image` to decode image_url parts. - handler = getattr(llama, "chat_handler", None) - supports_image = handler is not None and hasattr(handler, "load_image") - return cls(supports_image=supports_image) diff --git a/modelship/infer/llama_cpp/llama_cpp_infer.py b/modelship/infer/llama_cpp/llama_cpp_infer.py deleted file mode 100644 index ab43c6b..0000000 --- a/modelship/infer/llama_cpp/llama_cpp_infer.py +++ /dev/null @@ -1,242 +0,0 @@ -import asyncio -import os -from collections.abc import AsyncGenerator - -from llama_cpp import Llama, LlamaDiskCache, LlamaRAMCache, llama_supports_gpu_offload - -from modelship.infer.base_infer import BaseInfer -from modelship.infer.infer_config import LlamaCppConfig, ModelshipModelConfig, ModelUsecase, RawRequestProxy -from modelship.infer.llama_cpp.capabilities import LlamaCppCapabilities -from modelship.infer.llama_cpp.openai.serving_chat import OpenAIServingChat -from modelship.infer.llama_cpp.openai.serving_embedding import OpenAIServingEmbedding -from modelship.infer.llama_cpp.utils import build_tool_call_renderer -from modelship.logging import get_logger -from modelship.openai.parsers.reasoning import resolve_active_reasoning_parser -from modelship.openai.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, - EmbeddingRequest, - EmbeddingResponse, - ErrorResponse, -) -from modelship.preflight import discover_hardware, merge_with_user_overrides, run_preflight -from modelship.utils import cache_dir, drop_reserved_kwargs - -logger = get_logger("infer.llama_cpp") - - -class LlamaCppInfer(BaseInfer): - def __init__(self, model_config: ModelshipModelConfig): - super().__init__(model_config) - user_config = model_config.llama_cpp_config or LlamaCppConfig() - user_overrides = user_config.model_dump(exclude_unset=True) - - # Preflight: hardware-aware safe defaults the user can override. - # User-supplied values always win; divergences are logged so - # misconfigured deploys are visible without spelunking llama.cpp logs. - recommendation = run_preflight(model_config, discover_hardware()) - if recommendation: - logger.info("preflight recommendation for '%s': %s", model_config.name, recommendation) - else: - logger.info("preflight recommendation for '%s': none", model_config.name) - merged = merge_with_user_overrides(recommendation, user_overrides, model_name=model_config.name) - self.config = user_config.model_copy(update=merged) - - # Automatically enable verbose mode if MSHIP_LOG_LEVEL is TRACE. - # Other log levels are handled via the 'llama_cpp' Python logger (configured in logging.py). - mship_log_level = os.environ.get("MSHIP_LOG_LEVEL", "INFO").upper() - self._verbose = mship_log_level == "TRACE" - - # Honor GPU offload only when the installed llama-cpp-python has a GPU backend - # AND Ray assigned this actor a GPU. Otherwise force CPU so the CPU wheel and - # num_gpus:0 deploys still work. - gpu_capable = llama_supports_gpu_offload() - gpu_assigned = self.model_config.num_gpus > 0 - if gpu_capable and gpu_assigned: - self._n_gpu_layers = self.config.n_gpu_layers - else: - if gpu_assigned and not gpu_capable and self.config.n_gpu_layers != 0: - logger.warning( - "n_gpu_layers=%s ignored for model '%s': llama-cpp-python was built " - "without GPU support; running CPU-only.", - self.config.n_gpu_layers, - self.model_config.name, - ) - self._n_gpu_layers = 0 - - self.llamacpp: Llama | None = None - self.serving_chat: OpenAIServingChat | None = None - self.serving_embedding: OpenAIServingEmbedding | None = None - logger.info( - "initialising llama.cpp engine (verbose=%s) with config: %s", - self._verbose, - self.config.model_dump(), - ) - - def shutdown(self) -> None: - if self.llamacpp: - logger.info("Shutting down llama.cpp engine for %s", self.model_config.name) - # llama-cpp-python relies on __del__ for resource cleanup. - del self.llamacpp - self.llamacpp = None - self.serving_chat = None - self.serving_embedding = None - - def __del__(self): - self.shutdown() - - async def start(self) -> None: - logger.info("Start llama.cpp infer for model: %s", self.model_config) - loop = asyncio.get_event_loop() - - model_path = self.model_config._resolved_path - if not model_path: - raise ValueError( - f"LlamaCpp deployment '{self.model_config.name}' is missing a resolved model path. " - f"Check driver logs for resolution errors." - ) - - self.llamacpp = await loop.run_in_executor( - None, - lambda: Llama( - model_path=model_path, - n_gpu_layers=self._n_gpu_layers, - n_ctx=self.config.n_ctx, - n_batch=self.config.n_batch, - chat_format=self.config.chat_format, - verbose=self._verbose, - embedding=self.model_config.usecase == ModelUsecase.embed, - **self.config.model_kwargs, - ), - ) - - self._set_max_context_length(self.config.n_ctx) - - assert self.llamacpp is not None - self._enable_cache() - capabilities = LlamaCppCapabilities.detect(self.llamacpp) - if capabilities.supports_image: - logger.info("Multimodal (vision) capability detected for model: %s", self.model_config.name) - - if self.model_config.usecase == ModelUsecase.generate: - parser_name = self.model_config._resolved_tool_call_parser - reasoning_name = self.model_config._resolved_reasoning_parser - template = self.model_config._resolved_chat_template - # Default to driving the model through raw `create_completion` - # so reasoning + tool-call markers in the model's output are - # visible to ChatOutputStreamer regardless of whether parsers - # are configured for this request. Fall back to llama-cpp's - # native chat handler only when: - # - the user explicitly set `chat_format` (presumed opt-in - # to llama-cpp's templating; our parsers are bypassed), or - # - no chat template was resolvable (we'd have nothing to - # render with). - if self.config.chat_format is None and template is not None: - # Strip keys render_jinja_template already receives positionally — - # a collision is a duplicate-keyword TypeError at render time. - template_kwargs = drop_reserved_kwargs( - self.model_config.chat_template_kwargs, - # `messages` is the conversation in the template context; the rest - # are render_jinja_template's own positional args. - { - "messages", - "conversations", - "tools", - "chat_template", - "add_generation_prompt", - "bos_token", - "eos_token", - }, - logger=logger, - context=f"model '{self.model_config.name}'", - ) - renderer = build_tool_call_renderer(self.llamacpp, template, template_kwargs) - # Confirm the capability-detected reasoning parser against the real - # render: a deployment that suppressed reasoning via chat_template_kwargs - # downgrades to None, so `tool_choice=auto` can constrain. - reasoning_name = resolve_active_reasoning_parser( - reasoning_name, lambda: renderer.render([{"role": "user", "content": "hi"}], None) - ) - else: - renderer = None - if self.model_config.chat_template_kwargs: - logger.warning( - "model '%s' sets chat_template_kwargs but `chat_format` is set or no chat " - "template is available; falling back to llama-cpp's native chat handler and " - "the kwargs will not be honored.", - self.model_config.name, - ) - if parser_name is not None or reasoning_name is not None: - logger.warning( - "model '%s' has parsers resolved (tool=%s, reasoning=%s) but `chat_format` is set " - "or no chat template is available; falling back to llama-cpp's native chat handler " - "and our parsers will not run.", - self.model_config.name, - parser_name, - reasoning_name, - ) - parser_name = None - reasoning_name = None - self.serving_chat = OpenAIServingChat( - self.llamacpp, - self.model_config.name, - capabilities, - tool_call_parser=parser_name, - reasoning_parser=reasoning_name, - renderer=renderer, - ) - elif self.model_config.usecase == ModelUsecase.embed: - self.serving_embedding = OpenAIServingEmbedding(self.llamacpp, self.model_config.name) - - def _enable_cache(self) -> None: - """Attach llama.cpp's native prompt-state cache if configured.""" - cache_config = self.config.cache - if cache_config is None: - return - assert self.llamacpp is not None - if cache_config.type == "disk": - # Root the cache under MSHIP_CACHE_DIR (not the Ray actor's ephemeral - # working dir) so it survives restarts/reschedules, and key it by the - # deployment name (name + config fingerprint + gateway). llama.cpp keys - # entries on prompt tokens alone, so this prevents a different model, - # changed config, or another gateway from cross-loading or concurrently - # corrupting an incompatible store. The fingerprint is stable across - # redeploys of the same config, so persistence holds. - gateway_name = os.environ.get("MSHIP_GATEWAY_NAME", "") - deployment_name = self.model_config.deployment_name(gateway_name) - cache_path = os.path.join(cache_dir(), "llama_cache", deployment_name) - cache = LlamaDiskCache(cache_dir=cache_path, capacity_bytes=cache_config.capacity_bytes) - logger.info( - "enabled llama.cpp disk prompt cache at %s (capacity=%d bytes) for model '%s'", - cache_path, - cache_config.capacity_bytes, - self.model_config.name, - ) - else: - cache = LlamaRAMCache(capacity_bytes=cache_config.capacity_bytes) - logger.info( - "enabled llama.cpp ram prompt cache (capacity=%d bytes) for model '%s'", - cache_config.capacity_bytes, - self.model_config.name, - ) - self.llamacpp.set_cache(cache) - - async def warmup(self) -> None: - if self.serving_chat is not None: - await self.serving_chat.warmup() - elif self.serving_embedding is not None: - await self.serving_embedding.warmup() - - async def create_chat_completion( - self, request: ChatCompletionRequest, raw_request: RawRequestProxy - ) -> ErrorResponse | ChatCompletionResponse | AsyncGenerator[str, None]: - if self.serving_chat is None: - return await super().create_chat_completion(request, raw_request) - return await self.serving_chat.create_chat_completion(request, raw_request) - - async def create_embedding( - self, request: EmbeddingRequest, raw_request: RawRequestProxy - ) -> ErrorResponse | EmbeddingResponse: - if self.serving_embedding is None: - return await super().create_embedding(request, raw_request) - return await self.serving_embedding.create_embedding(request, raw_request) diff --git a/modelship/infer/llama_cpp/openai/serving_chat.py b/modelship/infer/llama_cpp/openai/serving_chat.py deleted file mode 100644 index 9b56e4c..0000000 --- a/modelship/infer/llama_cpp/openai/serving_chat.py +++ /dev/null @@ -1,420 +0,0 @@ -import asyncio -import inspect -import json -import time -from collections.abc import AsyncGenerator, AsyncIterator, Iterator -from typing import Any, cast - -from llama_cpp import Llama - -from modelship.infer.base_serving import OpenAIServing -from modelship.infer.infer_config import RawRequestProxy -from modelship.infer.llama_cpp.capabilities import LlamaCppCapabilities -from modelship.infer.llama_cpp.structured import build_llama_grammar -from modelship.infer.llama_cpp.tool_grammar import build_tool_call_grammar -from modelship.infer.llama_cpp.utils import LlamaCppToolCallRenderer -from modelship.logging import TRACE, get_logger -from modelship.openai.chat_utils import ( - ParsedChatOutput, - UnsupportedContentError, - build_from_parsed, - normalize_chat_messages, -) -from modelship.openai.parsers.reasoning import get_parser as get_reasoning_parser -from modelship.openai.parsers.streaming import finish_reason_for, parse_chat_completion_text, stream_chat_completion -from modelship.openai.parsers.tool_calling import get_parser, request_forces_tool_call, resolve_tools_for_request -from modelship.openai.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, - ErrorResponse, - UsageInfo, - create_error_response, -) -from modelship.utils import base_request_id - -logger = get_logger("infer.llama_cpp.chat") - - -class OpenAIServingChat(OpenAIServing): - request_id_prefix = "chat" - - def __init__( - self, - llama: Llama, - model_name: str, - capabilities: LlamaCppCapabilities, - tool_call_parser: str | None = None, - reasoning_parser: str | None = None, - renderer: LlamaCppToolCallRenderer | None = None, - ): - self._llama = llama - self.model_name = model_name - self._caps = capabilities - self._lock = asyncio.Lock() - self._accepted_params = set(inspect.signature(llama.create_chat_completion).parameters) - self._completion_accepted_params = set(inspect.signature(llama.create_completion).parameters) - self.tool_call_parser = tool_call_parser - self.reasoning_parser = reasoning_parser - self._renderer = renderer - self._logged_reasoning_unconstrained = False - # The renderer's presence is the sole switch between paths: - # - renderer set → drive `create_completion` raw, route every - # response through `ChatOutputStreamer`. Reasoning + tool-call - # extraction happens here. Default for any model that exposes - # a chat template via GGUF metadata. - # - renderer None → fall back to llama-cpp's - # `create_chat_completion`. Used when the user opted into - # llama-cpp's own templating with `chat_format` on - # LlamaCppConfig — they're presumed to know our parsers won't - # fire for that config. - if tool_call_parser is not None: - assert renderer is not None, "renderer is required when tool_call_parser is set" - get_parser(tool_call_parser) - if reasoning_parser is not None: - assert renderer is not None, "renderer is required when reasoning_parser is set" - get_reasoning_parser(reasoning_parser) - - async def warmup(self) -> None: - logger.info("Warming up llama.cpp chat model: %s", self.model_name) - request = ChatCompletionRequest( - model=self.model_name, - messages=[{"role": "user", "content": "warmup"}], - max_tokens=1, - ) - await self.create_chat_completion(request, RawRequestProxy(None, {})) - logger.info("Warmup chat done for %s", self.model_name) - - async def create_chat_completion( - self, request: ChatCompletionRequest, raw_request: RawRequestProxy - ) -> ErrorResponse | ChatCompletionResponse | AsyncGenerator[str, None]: - request_id = f"{self.request_id_prefix}-{base_request_id(raw_request)}" - logger.info("chat completion request %s: stream=%s", request_id, request.stream) - logger.log( - TRACE, - "chat request %s: messages=%s tools=%s tool_choice=%s", - request_id, - request.messages, - request.tools, - request.tool_choice, - ) - - # llama.cpp can produce token logprobs, but the modelship wrapper does - # not yet thread them into the OpenAI `choices[].logprobs` response. - # Reject explicitly rather than silently dropping the request fields, - # which used to leave clients waiting for logprobs that never arrived. - if request.logprobs or request.top_logprobs: - msg = ( - "logprobs / top_logprobs are not yet supported on the llama.cpp loader. " - "Resend the request without them." - ) - logger.warning("chat request %s rejected: %s", request_id, msg) - return create_error_response(msg) - - try: - messages = normalize_chat_messages( - request.messages, - supports_image=self._caps.supports_image, - supports_audio=self._caps.supports_audio, - ) - except UnsupportedContentError as e: - logger.warning("chat request %s rejected: %s", request_id, e) - return create_error_response(e) - - tools = resolve_tools_for_request(request.tools, request.tool_choice) - if tools and self.tool_call_parser is None: - logger.warning( - "chat request %s asks for %d tool(s) but model %r has no usable tool-call parser; ignoring tools", - request_id, - len(tools), - self.model_name, - ) - tools = None - - # Reasoning-enabled deployments emit `...` before content. - # A JSON grammar from response_format would exclude the `<` token and - # break reasoning emission. Reject upfront so the user picks a - # non-reasoning deployment for schema-constrained traffic. - if self.reasoning_parser and request.response_format: - fmt_type = request.response_format.get("type") - if fmt_type not in (None, "text"): - msg = ( - f"response_format with type={fmt_type!r} cannot be combined with a " - f"reasoning-enabled deployment ({self.model_name!r}). The schema grammar " - "prevents the reasoning parser's required tokens from being emitted. " - "Use a non-reasoning deployment, or drop response_format." - ) - logger.warning("chat request %s rejected: %s", request_id, msg) - return create_error_response(msg) - - tool_parser_name = self.tool_call_parser if tools else None - # Renderer presence decides the path (see __init__ comment): when - # the user set `chat_format` on LlamaCppConfig we leave it to - # llama-cpp; otherwise we render the prompt ourselves so the - # ChatOutputStreamer sees the model's raw bytes regardless of - # whether reasoning or tools are active for this request. - if self._renderer is not None: - return await self._handle_with_parsers( - request, request_id, messages, tools, tool_parser_name=tool_parser_name - ) - return await self._handle_native(request, request_id, messages) - - async def _handle_native( - self, - request: ChatCompletionRequest, - request_id: str, - messages: list[dict], - ) -> ErrorResponse | ChatCompletionResponse | AsyncGenerator[str, None]: - """Pass-through to ``llama.create_chat_completion`` — no parser involved. - - Used when neither tool calling nor reasoning is active for this - deployment, or when the user configured ``chat_format`` (in - which case our parsers are intentionally bypassed and - llama-cpp-python's own chat handler is responsible). - """ - kwargs = self._build_kwargs(request, messages) - loop = asyncio.get_event_loop() - llama = self._llama - - if request.stream: - - async def stream_generator() -> AsyncGenerator[str, None]: - async with self._lock: - raw = await loop.run_in_executor( - None, - lambda: llama.create_chat_completion(**kwargs, stream=True), # type: ignore[arg-type] - ) - iterator = cast(Iterator[dict], raw) - while True: - chunk = await loop.run_in_executor(None, lambda: next(iterator, None)) - if chunk is None: - break - yield f"data: {json.dumps(chunk)}\n\n" - await asyncio.sleep(0) - yield "data: [DONE]\n\n" - - return stream_generator() - - async with self._lock: - try: - result = await loop.run_in_executor( - None, - lambda: llama.create_chat_completion(**kwargs, stream=False), # type: ignore[arg-type] - ) - return ChatCompletionResponse.model_validate(result) - except Exception as e: - logger.warning("llama_cpp chat inference failed: %s", e) - return create_error_response(e) - - async def _handle_with_parsers( - self, - request: ChatCompletionRequest, - request_id: str, - messages: list[dict], - tools: list[dict[str, Any]] | None, - *, - tool_parser_name: str | None, - ) -> ErrorResponse | ChatCompletionResponse | AsyncGenerator[str, None]: - """Render prompt ourselves, run raw completion, parse via shared streamer. - - Bypasses ``llama.create_chat_completion`` because that re-runs - llama-cpp's own chat templating (which also strips tokens our - parsers need to see). The unified streamer handles reasoning - and tool calls in a single pass over the model output. - """ - assert self._renderer is not None # guarded at __init__ - renderer = self._renderer - reasoning_parser_name = self.reasoning_parser - - try: - prompt = renderer.render(messages, tools) - except Exception as e: - logger.warning("llama_cpp prompt rendering failed for %s: %s", request_id, e) - return create_error_response(e) - - max_tokens = request.max_tokens - if max_tokens is None and request.max_completion_tokens is not None: - max_tokens = request.max_completion_tokens - - completion_kwargs = self._build_completion_kwargs(request, prompt) - # `tool_choice` drives the grammar. `force` (required/named) builds a - # tool-only root that can't emit ``, so it ignores the reasoning - # parser. The `auto` grammar's `content ::= [^<]+` excludes `<`, which also - # opens ``, so it yields to a reasoning-enabled deployment. - force = request_forces_tool_call(request.tool_choice) - if tool_parser_name and tools and (force or not self.reasoning_parser): - grammar = build_tool_call_grammar(get_parser(tool_parser_name), tools, require_tool_call=force) - if grammar is not None: - if completion_kwargs.get("grammar") is not None: - logger.warning( - "chat request %s: response_format grammar overridden by the tool-call grammar; " - "the two cannot be combined", - request_id, - ) - completion_kwargs["grammar"] = grammar - elif tool_parser_name and tools and self.reasoning_parser and not self._logged_reasoning_unconstrained: - self._logged_reasoning_unconstrained = True - logger.info( - "tool-call grammar left off for reasoning-enabled deployment %r on tool_choice=auto; " - "the grammar would block the reasoning block", - self.model_name, - ) - prompt_tokens = renderer.count_tokens(prompt) - loop = asyncio.get_event_loop() - llama = self._llama - - if request.stream: - include_usage = bool(request.stream_options and request.stream_options.include_usage) - return self._locked_stream_with_parsers( - request_id=request_id, - completion_kwargs=completion_kwargs, - tool_parser_name=tool_parser_name, - reasoning_parser_name=reasoning_parser_name, - prompt_tokens=prompt_tokens, - max_tokens=max_tokens, - include_usage=include_usage, - ) - - async with self._lock: - try: - raw = await loop.run_in_executor( - None, - lambda: llama.create_completion(**completion_kwargs, stream=False), # type: ignore[arg-type] - ) - result = cast(dict, raw) - except Exception as e: - logger.warning("llama_cpp inference failed for %s: %s", request_id, e) - return create_error_response(e) - - completion_text = result["choices"][0]["text"] if result.get("choices") else "" - logger.log(TRACE, "chat response %s: %r", request_id, completion_text) - usage = result.get("usage") or {} - completion_tokens = int(usage.get("completion_tokens") or renderer.count_tokens(completion_text)) - - parsed = parse_chat_completion_text( - completion_text, - parser_name=tool_parser_name, - reasoning_parser_name=reasoning_parser_name, - ) - finish_reason = finish_reason_for(parsed, completion_tokens, max_tokens) - prompt_tokens_used = int(usage.get("prompt_tokens") or prompt_tokens) - return build_from_parsed( - request_id=request_id, - model_name=self.model_name, - choices=[ - ParsedChatOutput(content=parsed.content, reasoning=parsed.reasoning, tool_calls=parsed.tool_calls) - ], - usage=UsageInfo( - prompt_tokens=prompt_tokens_used, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens_used + completion_tokens, - ), - finish_reasons=[finish_reason], - created=int(time.time()), - ) - - async def _locked_stream_with_parsers( - self, - *, - request_id: str, - completion_kwargs: dict, - tool_parser_name: str | None, - reasoning_parser_name: str | None, - prompt_tokens: int, - max_tokens: int | None, - include_usage: bool, - ) -> AsyncGenerator[str, None]: - assert self._renderer is not None - renderer = self._renderer - # Buffer the raw model text so we can emit it once at end-of-stream - # (in a finally, so a client disconnect still logs what was produced). - buffered: list[str] = [] - async with self._lock: - try: - async for chunk in stream_chat_completion( - request_id=request_id, - model_name=self.model_name, - text_chunks=self._raw_text_chunks(completion_kwargs, buffered), - parser_name=tool_parser_name, - reasoning_parser_name=reasoning_parser_name, - count_tokens=renderer.count_tokens, - prompt_tokens=prompt_tokens, - max_tokens=max_tokens, - include_usage=include_usage, - created=int(time.time()), - ): - yield chunk - finally: - logger.log(TRACE, "chat response %s (stream): %r", request_id, "".join(buffered)) - - async def _raw_text_chunks(self, completion_kwargs: dict, buffer: list[str] | None = None) -> AsyncIterator[str]: - """Drive ``llama.create_completion(stream=True)`` and yield text pieces.""" - loop = asyncio.get_event_loop() - llama = self._llama - raw = await loop.run_in_executor( - None, - lambda: llama.create_completion(**completion_kwargs, stream=True), # type: ignore[arg-type] - ) - iterator = cast(Iterator[dict], raw) - while True: - chunk = await loop.run_in_executor(None, lambda: next(iterator, None)) - if chunk is None: - return - text = (chunk.get("choices") or [{}])[0].get("text") or "" - if text: - if buffer is not None: - buffer.append(text) - yield text - - def _build_kwargs(self, request: ChatCompletionRequest, messages: list[dict]) -> dict: - params = request.model_dump(exclude_none=True) - params["messages"] = messages - if "max_tokens" not in params and "max_completion_tokens" in params: - params["max_tokens"] = params["max_completion_tokens"] - params.pop("max_completion_tokens", None) - - # Convert OpenAI-shaped response_format → LlamaGrammar. llama-cpp-python's - # own handler only recognizes {"type": "json_object", "schema": ...} and - # silently drops {"type": "json_schema", ...}, so we own the conversion. - grammar = build_llama_grammar(params.pop("response_format", None)) - - kwargs: dict = {} - dropped: list[str] = [] - for k, v in params.items(): - if k == "stream": - continue - if k in self._accepted_params: - kwargs[k] = v - else: - dropped.append(k) - if dropped: - logger.warning( - "llama_cpp: dropping request params not supported by create_chat_completion: %s", - dropped, - ) - if grammar is not None: - kwargs["grammar"] = grammar - return kwargs - - def _build_completion_kwargs(self, request: ChatCompletionRequest, prompt: str) -> dict: - params = request.model_dump(exclude_none=True) - params["prompt"] = prompt - if "max_tokens" not in params and "max_completion_tokens" in params: - params["max_tokens"] = params["max_completion_tokens"] - params.pop("max_completion_tokens", None) - # These are consumed by our renderer/parser, not by `create_completion`. - for k in ("messages", "tools", "tool_choice", "stream", "stream_options"): - params.pop(k, None) - # logprobs/top_logprobs are rejected up front in create_chat_completion when - # truthy, but `model_dump(exclude_none=True)` still keeps their falsy defaults - # (False / 0). `create_completion` accepts `logprobs` (Optional[int]), so a - # leftover `logprobs=False` is `not None` and triggers needless logprob work - # in llama-cpp-python. Drop both unconditionally. - for k in ("logprobs", "top_logprobs"): - params.pop(k, None) - - grammar = build_llama_grammar(params.pop("response_format", None)) - - kwargs = {k: v for k, v in params.items() if k in self._completion_accepted_params} - if grammar is not None: - kwargs["grammar"] = grammar - return kwargs diff --git a/modelship/infer/llama_cpp/openai/serving_embedding.py b/modelship/infer/llama_cpp/openai/serving_embedding.py deleted file mode 100644 index 5e5e342..0000000 --- a/modelship/infer/llama_cpp/openai/serving_embedding.py +++ /dev/null @@ -1,63 +0,0 @@ -import asyncio -import inspect - -from llama_cpp import Llama - -from modelship.infer.base_serving import OpenAIServing -from modelship.infer.infer_config import RawRequestProxy -from modelship.logging import get_logger -from modelship.openai.protocol import ( - EmbeddingRequest, - EmbeddingResponse, - ErrorResponse, - create_error_response, -) -from modelship.utils import base_request_id - -logger = get_logger("infer.llama_cpp.embedding") - - -class OpenAIServingEmbedding(OpenAIServing): - request_id_prefix = "embd" - - def __init__(self, llama: Llama, model_name: str): - self._llama = llama - self.model_name = model_name - self._lock = asyncio.Lock() - self._accepted_params = set(inspect.signature(llama.create_embedding).parameters) - - async def warmup(self) -> None: - logger.info("Warming up llama.cpp embedding model: %s", self.model_name) - request = EmbeddingRequest(model=self.model_name, input="warmup") - await self.create_embedding(request, RawRequestProxy(None, {})) - logger.info("Warmup embedding done for %s", self.model_name) - - async def create_embedding( - self, request: EmbeddingRequest, raw_request: RawRequestProxy - ) -> ErrorResponse | EmbeddingResponse: - request_id = f"{self.request_id_prefix}-{base_request_id(raw_request)}" - logger.info("embedding request %s", request_id) - - params = request.model_dump(exclude_none=True) - kwargs: dict = {} - dropped: list[str] = [] - for k, v in params.items(): - if k in self._accepted_params: - kwargs[k] = v - else: - dropped.append(k) - if dropped: - logger.warning( - "llama_cpp: dropping request params not supported by create_embedding: %s", - dropped, - ) - - loop = asyncio.get_event_loop() - llama = self._llama - async with self._lock: - try: - result = await loop.run_in_executor(None, lambda: llama.create_embedding(**kwargs)) - return EmbeddingResponse.model_validate(result) - except Exception as e: - logger.warning("llama_cpp embedding failed: %s", e) - return create_error_response(e) diff --git a/modelship/infer/llama_cpp/structured.py b/modelship/infer/llama_cpp/structured.py deleted file mode 100644 index 82f8099..0000000 --- a/modelship/infer/llama_cpp/structured.py +++ /dev/null @@ -1,64 +0,0 @@ -"""OpenAI ``response_format`` → llama.cpp ``LlamaGrammar`` conversion. - -llama-cpp-python's ``create_chat_completion`` accepts a ``response_format`` -kwarg, but its built-in handler only recognizes the narrower -``{"type": "json_object", "schema": ...}`` shape and silently returns no -grammar for OpenAI's ``{"type": "json_schema", "json_schema": {...}}``. -We convert the OpenAI shape into a ``LlamaGrammar`` here and pass it via -the ``grammar`` kwarg, which both ``create_chat_completion`` and -``create_completion`` accept. -""" - -from __future__ import annotations - -import json -from typing import Any - -from llama_cpp import LlamaGrammar - -from modelship.logging import get_logger - -logger = get_logger("infer.llama_cpp.structured") - -# Permissive JSON-object grammar: any valid JSON object. Used when the -# caller requested ``{"type": "json_object"}`` without a schema. -# Built fresh per request rather than cached: ``LlamaGrammar`` wraps a -# stateful C-side parser pointer, so sharing one instance across -# concurrent requests (or across Llama instances in the same process) -# would corrupt sampling state. Compilation cost for this trivial -# schema is negligible. -_JSON_OBJECT_SCHEMA = {"type": "object"} - - -def build_llama_grammar(response_format: dict[str, Any] | None) -> LlamaGrammar | None: - """Convert an OpenAI-shaped ``response_format`` to a ``LlamaGrammar``. - - Returns ``None`` when no constraint should apply (missing, ``text``, - or a malformed payload — logged as a warning). - """ - if not response_format: - return None - - fmt_type = response_format.get("type") - if fmt_type in (None, "text"): - return None - - if fmt_type == "json_object": - return LlamaGrammar.from_json_schema(json.dumps(_JSON_OBJECT_SCHEMA), verbose=False) - - if fmt_type == "json_schema": - spec = response_format.get("json_schema") or {} - schema = spec.get("schema") - if not isinstance(schema, dict): - logger.warning( - "response_format.json_schema is missing a 'schema' object; skipping grammar constraint", - ) - return None - try: - return LlamaGrammar.from_json_schema(json.dumps(schema), verbose=False) - except Exception as exc: - logger.warning("failed to compile json_schema into LlamaGrammar: %s; skipping", exc) - return None - - logger.warning("unsupported response_format type %r; skipping grammar constraint", fmt_type) - return None diff --git a/modelship/infer/llama_cpp/tool_grammar.py b/modelship/infer/llama_cpp/tool_grammar.py deleted file mode 100644 index dd26698..0000000 --- a/modelship/infer/llama_cpp/tool_grammar.py +++ /dev/null @@ -1,365 +0,0 @@ -"""Build a GBNF ``LlamaGrammar`` that constrains tool-call output. - -Compiled from a request's ``tools`` and passed to -``llama.create_completion(grammar=...)`` — the same ``grammar`` kwarg -:mod:`structured` uses for ``response_format``. Unlike a ``response_format`` -grammar (which forces the whole response to be one JSON value), this grammar's -root permits a free-text answer *or* a bounded sequence of enveloped tool -calls wrapped in optional conversational text, so a non-tool answer and a -chatty preamble before a tool call are both reachable. - -JSON-family parsers (Hermes) wrap each call's ``{"name", "arguments"}`` JSON in -literal marker tags, so the body is built by reusing llama.cpp's own -``json_schema_to_gbnf`` and only the envelope is hand-written. - -Gemma-family parsers (FunctionGemma / Gemma 4) use a non-JSON DSL — -``call:FUNC{key:val,n:42}`` — so ``json_schema_to_gbnf`` can't -express it; that body is hand-built by :func:`_build_gemma_tool_call_gbnf`, -which maps each tool's parameter schema to value rules (enum alternations, -delimited strings, bare numbers/bools, arrays, nested objects). The envelope -caps the number of calls (``_MAX_TOOL_CALLS``) to stop the runaway repetition -small FunctionGemma checkpoints fall into. Other families return ``None``. - -The Gemma grammar is loose by design: it pins the call count, tool name, and -enum values. Key uniqueness, ordering, and ``required`` arguments are enforced -at the top level, and only nested-object shape remains loose. -""" - -from __future__ import annotations - -import json -import re -from typing import TYPE_CHECKING, Any - -from llama_cpp import LlamaGrammar -from llama_cpp.llama_grammar import json_schema_to_gbnf - -from modelship.logging import get_logger - -if TYPE_CHECKING: - from modelship.openai.parsers.tool_calling import ToolCallParser - -logger = get_logger("infer.llama_cpp.tool_grammar") - -# Parsers whose body is a JSON ``{"name", "arguments"}`` object wrapped in literal -# ``start_marker`` / ``end_marker`` tags — the only shape ``json_schema_to_gbnf`` -# plus the envelope below can express. Qwen3-Coder shares Hermes' markers but has -# an XML body, so it's excluded; Mistral / llama3_json are JSON but use a -# different envelope shape, not wired here. -_JSON_FAMILY_PARSERS = frozenset({"hermes"}) - -# Parsers whose call body is a custom ``call:FUNC{...}`` DSL — handled by the -# hand-built emitter, not ``json_schema_to_gbnf``. -_GEMMA_FAMILY_PARSERS = frozenset({"function_gemma", "gemma4"}) - -# Max enveloped calls the grammar allows per response. Held at 1 to isolate -# single-command correctness and hardest-stop the 270M runaway; raise to relax. -_MAX_TOOL_CALLS = 1 - -# Recursion guard for nested object/array schemas — beyond this, value rules -# collapse to the generic recursive value rule (``gv``), which accepts any -# string / number / bool / null / array / object the DSL can express. -_MAX_SCHEMA_DEPTH = 5 - -# Parser names already logged as unconstrained, to avoid repeating per request. -_logged_unconstrained: set[str] = set() - - -def _gbnf_literal(s: str) -> str: - """Render ``s`` as a GBNF double-quoted string literal. - - GBNF accepts the same ``\\"`` / ``\\\\`` escapes JSON uses, so ``json.dumps`` - produces a valid literal; ``ensure_ascii=False`` keeps UTF-8 enum values - intact rather than emitting ``\\uXXXX`` escapes. - """ - return json.dumps(s, ensure_ascii=False) - - -def build_tool_call_gbnf( - parser: ToolCallParser, tools: list[dict[str, Any]], *, require_tool_call: bool = False -) -> str | None: - """Assemble the GBNF text for the tool-call grammar, or ``None`` if unsupported. - - Split from compilation so the emitted grammar can be inspected/tested as - text. Returns ``None`` (logged once per parser) for parsers without an - emitter and on conversion failure. When ``require_tool_call`` is set the - root drops its free-text escape, so the model must emit a tool call. - """ - if not tools: - return None - - if parser.name in _GEMMA_FAMILY_PARSERS: - return _build_gemma_tool_call_gbnf(parser, tools, require_tool_call=require_tool_call) - - if parser.name not in _JSON_FAMILY_PARSERS: - if parser.name not in _logged_unconstrained: - _logged_unconstrained.add(parser.name) - logger.info( - "tool-call grammar: no emitter for tool_call_parser %r; left unconstrained", - parser.name, - ) - return None - - # One anyOf branch per tool: ``name`` pinned to the tool's name via ``const``, - # ``arguments`` constrained to its parameter schema. ``additionalProperties: - # false`` forbids keys outside the envelope. Entries lacking a function name - # are skipped — a ``None`` const yields an unsatisfiable branch. - tool_schemas = [] - for t in tools: - func = t.get("function") - if not func or not func.get("name"): - continue - tool_schemas.append( - { - "type": "object", - "properties": { - "name": {"const": func["name"]}, - "arguments": func.get("parameters") or {"type": "object"}, - }, - "required": ["name", "arguments"], - "additionalProperties": False, - } - ) - - if not tool_schemas: - return None - - meta_schema = {"anyOf": tool_schemas} - - try: - inner = json_schema_to_gbnf(json.dumps(meta_schema)) - except Exception as exc: - logger.warning("tool-call grammar: failed to convert tool schemas to GBNF: %s; skipping", exc) - return None - - # The converter's entry rule is ``root``; rename only its LHS so our own - # ``root`` can wrap it. Nothing references it, so a line-anchored sub suffices. - inner = re.sub(r"^\s*root\s*::=", "tc-json ::=", inner, flags=re.M) - - start = json.dumps(parser.start_marker) - end = json.dumps(parser.end_marker) - - # Exclude the start marker's first char from `content` so free text yields to a - # tool call when the marker begins. Escape it if it's special inside a GBNF class. - start_char = parser.start_marker[0] if parser.start_marker else "<" - escaped_char = f"\\{start_char}" if start_char in "]-^\\" else start_char - - # Allow optional leading/trailing conversational text and optional whitespace - # around the JSON envelope, so a chatty prefix can't strand the model in the - # content branch. `tool-calls` caps at two calls so decoding can't loop forever. - # `require_tool_call` drops the free-text branch so a tool call is mandatory. - if require_tool_call: - root_rule = "root ::= tool-calls\n" - content_rule = "" - else: - root_rule = "root ::= ( content )? tool-calls ( content )? | content\n" - content_rule = f"content ::= [^{escaped_char}]+\n" - prefix = ( - root_rule + "tool-calls ::= tool-call ( tc-ws tool-call )?\n" - f"tool-call ::= {start} tc-ws tc-json tc-ws {end}\n" + content_rule + "tc-ws ::= [ \\t\\n\\r]*\n" - ) - return prefix + inner - - -def _build_gemma_tool_call_gbnf( - parser: ToolCallParser, tools: list[dict[str, Any]], *, require_tool_call: bool = False -) -> str | None: - """Hand-build the GBNF for a Gemma-family ``call:FUNC{...}`` DSL body. - - ``json_schema_to_gbnf`` only speaks JSON, so the per-tool body is emitted - directly: a name literal, then a brace-wrapped alternation of ``key:value`` - pairs whose value rules come from :func:`_emit_value`. Returns ``None`` when - no tool carries a usable function name. - """ - delim = parser.string_delim or "" - d_lit = _gbnf_literal(delim) - # String values exclude the marker/delim lead char ('<' for both families) - # so a value can't swallow the closing delim or end marker mid-generation. - excl = (delim or parser.start_marker or "<")[0] - excl_cls = f"\\{excl}" if excl in "]-^\\" else excl - str_val = f"( {d_lit} [^{excl_cls}]* {d_lit} )" - - # Generic recursive value rules, emitted only when referenced (free-form - # objects, unconstrained arrays, schemaless props, or past the depth cap). - # ``gv`` accepts any value the DSL can express, so the grammar never forces - # a delimited string where the model legitimately needs ``{...}`` / ``[...]``. - uses_generic = False - - def generic(ref: str) -> str: - nonlocal uses_generic - uses_generic = True - return ref - - def emit_enum_value(v: object) -> str: - if isinstance(v, bool): - return '"true"' if v else '"false"' - if isinstance(v, int | float): - return _gbnf_literal(str(v)) # bare, unquoted in the wire format - if v is None: - return '"null"' - return f"{d_lit} {_gbnf_literal(str(v))} {d_lit}" - - def emit_value(schema: object, depth: int = 0) -> str: - """Return a parenthesized GBNF expression matching one value of ``schema``. - - Always parenthesized so it can sit on either side of a ``|`` without - leaking precedence into the enclosing alternation. - """ - if not isinstance(schema, dict) or depth > _MAX_SCHEMA_DEPTH: - return generic("gv") - - enum = schema.get("enum") - if enum: - return "( " + " | ".join(emit_enum_value(v) for v in enum) + " )" - - for combinator in ("anyOf", "oneOf"): - subs = schema.get(combinator) - if isinstance(subs, list) and subs: - return "( " + " | ".join(emit_value(s, depth + 1) for s in subs) + " )" - - t = schema.get("type") - if isinstance(t, list) and t: - return "( " + " | ".join(emit_value({**schema, "type": tt}, depth + 1) for tt in t) + " )" - if t == "string": - return str_val - if t == "integer": - return '( "-"? [0-9]+ )' - if t == "number": - return '( "-"? [0-9]+ ( "." [0-9]+ )? )' - if t == "boolean": - return '( "true" | "false" )' - if t == "null": - return '( "null" )' - if t == "array": - items = schema.get("items") - item = emit_value(items, depth + 1) if isinstance(items, dict) else generic("gv") - return f'( "[" ( {item} ( "," {item} )* )? "]" )' - if t == "object": - props = schema.get("properties") - if isinstance(props, dict) and props: - pair = ( - "( " - + " | ".join(f"{_gbnf_literal(f'{k}:')} {emit_value(v, depth + 1)}" for k, v in props.items()) - + " )" - ) - return f'( "{{" ( {pair} ( "," {pair} )* )? "}}" )' - # Free-form object (no/empty properties): accept any DSL object. - return generic("gv-obj") - return generic("gv") - - tool_rules: list[str] = [] - n_tools = 0 - for t in tools: - func = t.get("function") - if not func or not func.get("name"): - continue - idx = n_tools - n_tools += 1 - name_lit = _gbnf_literal(func["name"]) - params = func.get("parameters") or {} - props = params.get("properties") if isinstance(params, dict) else None - if isinstance(props, dict) and props: - prop_items = list(props.items()) - for j, (k, v) in enumerate(prop_items): - tool_rules.append(f"tool-{idx}-p{j} ::= {_gbnf_literal(f'{k}:')} {emit_value(v)}") - m = len(prop_items) - # Ordered-optional args: each property appears at most once, in schema order, - # with correct commas (first present property is bare, every later one carries a - # leading comma). Enforces key *uniqueness* — a free ``pair ( "," pair )*`` - # alternation let the model repeat a key (e.g. ``name`` twice) -> corrupt call. - # Enforces top-level ``required`` fields by restricting branches to only valid - # ordered subsets that include all top-level required properties. - req_val = params.get("required") - if isinstance(req_val, str): - req_keys = {req_val} - elif isinstance(req_val, list): - req_keys = {k for k in req_val if isinstance(k, str)} - else: - req_keys = set() - - req_set = {j for j, (k, _) in enumerate(prop_items) if k in req_keys} - first_req = min(req_set) if req_set else None - - branch_js = range(first_req + 1) if first_req is not None else range(m) - - def later(n: int) -> str: - return f'"," tool-{idx}-p{n}' if n in req_set else f'( "," tool-{idx}-p{n} )?' # noqa: B023 - - branches = " | ".join( - " ".join([f"tool-{idx}-p{j}"] + [later(n) for n in range(j + 1, m)]) for j in branch_js - ) - args_rhs = f"( {branches} )?" if first_req is None else f"( {branches} )" - tool_rules.append(f"tool-{idx}-args ::= {args_rhs}") - tool_rules.append(f'tool-{idx} ::= {name_lit} "{{" tool-{idx}-args "}}"') - else: - # All-optional / no-property intents collapse to FUNC{}. - tool_rules.append(f'tool-{idx} ::= {name_lit} "{{" "}}"') - - if n_tools == 0: - return None - - call_choice = " | ".join(f"tool-{i}" for i in range(n_tools)) - # Calls are concatenated back-to-back (no separator) up to the cap. - tool_calls_rhs = "tool-call" + " ( tool-call )?" * (_MAX_TOOL_CALLS - 1) - - start = _gbnf_literal(parser.start_marker) - end = _gbnf_literal(parser.end_marker) - start_char = parser.start_marker[0] if parser.start_marker else "<" - content_excl = f"\\{start_char}" if start_char in "]-^\\" else start_char - - if require_tool_call: - root_rule = "root ::= ws tool-calls ws" - content_rules: list[str] = [] - else: - # A turn is *either* tool calls or free text — never text wrapped around a call. - # The old ``( content )? tool-calls ( content )?`` let a small model spill malformed - # DSL as leading/trailing content (e.g. ``capturePlayercall:HassMediaNext{}``), which - # then got captured into chat history and poisoned later turns. - root_rule = "root ::= ws tool-calls ws | content" - content_rules = [f"content ::= [^{content_excl}]+"] - envelope = [ - root_rule, - f"tool-calls ::= {tool_calls_rhs}", - f'tool-call ::= {start} "call:" call-choice {end}', - f"call-choice ::= {call_choice}", - "ws ::= [ \\t\\n\\r]*", - *content_rules, - ] - - generic_rules: list[str] = [] - if uses_generic: - generic_rules = [ - "gv ::= gv-str | gv-num | gv-bool | gv-null | gv-arr | gv-obj", - f"gv-str ::= {d_lit} [^{excl_cls}]* {d_lit}", - 'gv-num ::= "-"? [0-9]+ ( "." [0-9]+ )?', - 'gv-bool ::= "true" | "false"', - 'gv-null ::= "null"', - 'gv-arr ::= "[" ( gv ( "," gv )* )? "]"', - 'gv-obj ::= "{" ( gv-pair ( "," gv-pair )* )? "}"', - 'gv-pair ::= gv-key ":" gv', - # Bare key: anything up to the structural ``:`` / separators / delim lead. - f"gv-key ::= [^:,{{}}\\[\\]{excl_cls}]+", - ] - - return "\n".join(envelope + tool_rules + generic_rules) + "\n" - - -def build_tool_call_grammar( - parser: ToolCallParser, tools: list[dict[str, Any]], *, require_tool_call: bool = False -) -> LlamaGrammar | None: - """Compile a tool-call-constraining grammar, or ``None`` if unsupported. - - ``parser`` supplies the envelope markers; ``tools`` is the OpenAI-shaped list - already resolved for the request. ``require_tool_call`` forces a tool call by - dropping the free-text root branch. Returns ``None`` for parsers without an - emitter and on any conversion/compile failure, so the caller falls back to - unconstrained decoding rather than erroring. - """ - text = build_tool_call_gbnf(parser, tools, require_tool_call=require_tool_call) - if text is None: - return None - try: - return LlamaGrammar.from_string(text, verbose=False) - except Exception as exc: - logger.warning("tool-call grammar: failed to compile: %s; skipping", exc) - return None diff --git a/modelship/infer/llama_cpp/utils.py b/modelship/infer/llama_cpp/utils.py deleted file mode 100644 index 6db7484..0000000 --- a/modelship/infer/llama_cpp/utils.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Utilities for the llama_cpp loader.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any - -from modelship.logging import get_logger - -if TYPE_CHECKING: - from llama_cpp import Llama - -logger = get_logger("infer.llama_cpp.utils") - - -@dataclass -class LlamaCppToolCallRenderer: - """Renders tool-aware chat prompts and counts tokens for a llama_cpp model. - - The chat template is pre-resolved on the driver (``_resolved_chat_template``) - so detection and rendering see the exact same string — no chance of drift - if the GGUF reader and llama-cpp-python's metadata loader interpret the - field differently. - - Rendering uses ``transformers.utils.chat_template_utils.render_jinja_template`` - so we inherit the polyfills HF templates rely on (``raise_exception``, - ``tojson``, ``strftime_now``, ``ChainableUndefined``). - - Token counting goes through ``Llama.tokenize`` so it reflects the actual - GGUF vocab — no second tokenizer to keep in sync. - """ - - chat_template: str - bos_token: str - eos_token: str - _llama: Llama - template_kwargs: dict[str, Any] = field(default_factory=dict) - - def render(self, messages: list[dict], tools: list[dict] | None) -> str: - from transformers.utils.chat_template_utils import render_jinja_template - - # Reasoning chat templates (e.g. Qwen3) test `'' in message.content`, - # which raises "argument of type 'NoneType' is not iterable" on assistant - # tool-call messages that legitimately carry content=None (replayed multi-turn - # tool calls). Coerce None to "" so the membership test is a safe no-op. - safe_messages = [{**m, "content": ""} if m.get("content") is None else m for m in messages] - - rendered, _ = render_jinja_template( - conversations=[safe_messages], - tools=tools, # type: ignore[arg-type] # HF types it as list[dict | Callable]; we only pass dicts - chat_template=self.chat_template, - add_generation_prompt=True, - bos_token=self.bos_token, - eos_token=self.eos_token, - **self.template_kwargs, - ) - return rendered[0] - - def count_tokens(self, text: str) -> int: - try: - tokens = self._llama.tokenize(text.encode("utf-8"), add_bos=False, special=True) - return len(tokens) - except Exception as e: - logger.warning("llama_cpp tokenize failed (%s); returning 0", e) - return 0 - - -def build_tool_call_renderer( - llama: Llama, chat_template: str, template_kwargs: dict[str, Any] | None = None -) -> LlamaCppToolCallRenderer: - """Build a renderer from a loaded ``Llama`` and the pre-resolved template.""" - return LlamaCppToolCallRenderer( - chat_template=chat_template, - bos_token=_detokenize_special(llama, llama.token_bos()), - eos_token=_detokenize_special(llama, llama.token_eos()), - _llama=llama, - template_kwargs=template_kwargs or {}, - ) - - -def _detokenize_special(llama: Llama, token_id: int) -> str: - if token_id < 0: - return "" - try: - return llama.detokenize([token_id], special=True).decode("utf-8", errors="replace") - except Exception: - return "" diff --git a/modelship/infer/llama_server/llama_server_infer.py b/modelship/infer/llama_server/llama_server_infer.py index 33e7c78..f563dc8 100644 --- a/modelship/infer/llama_server/llama_server_infer.py +++ b/modelship/infer/llama_server/llama_server_infer.py @@ -89,12 +89,11 @@ class _EarlyCrashError(RuntimeError): class LlamaServerInfer(BaseInfer): """Drives a `llama-server` subprocess over its native OpenAI-compatible - HTTP API. Unlike `llama_cpp` (in-process `Llama` bindings + modelship's - own tool-call/reasoning parsers), llama-server does its own chat - templating, tool-call, and reasoning parsing — this loader is a thin, - concurrency-safe proxy that projects its responses onto modelship's - protocol models (never relaying its JSON/SSE verbatim, which would leak - llama.cpp-only extension fields like `timings` to clients).""" + HTTP API. llama-server does its own chat templating, tool-call, and + reasoning parsing — this loader is a thin, concurrency-safe proxy that + projects its responses onto modelship's protocol models (never relaying + its JSON/SSE verbatim, which would leak llama.cpp-only extension fields + like `timings` to clients).""" def __init__(self, model_config: ModelshipModelConfig): super().__init__(model_config) diff --git a/modelship/infer/model_deployment.py b/modelship/infer/model_deployment.py index fc04cd8..5f6850a 100644 --- a/modelship/infer/model_deployment.py +++ b/modelship/infer/model_deployment.py @@ -165,10 +165,6 @@ async def __init__(self, config: ModelshipModelConfig): from modelship.infer.diffusers.diffusers_infer import DiffusersInfer self.infer = DiffusersInfer(config) - elif config.loader == ModelLoader.llama_cpp: - from modelship.infer.llama_cpp.llama_cpp_infer import LlamaCppInfer - - self.infer = LlamaCppInfer(config) elif config.loader == ModelLoader.llama_server: from modelship.infer.llama_server.llama_server_infer import LlamaServerInfer diff --git a/modelship/infer/model_resolver.py b/modelship/infer/model_resolver.py index c60b947..cd7d7f6 100644 --- a/modelship/infer/model_resolver.py +++ b/modelship/infer/model_resolver.py @@ -161,7 +161,7 @@ def resolve_model_source(model_ref: str, trust_remote_code: bool = False) -> str ) # Single GGUF in the repo: download it directly and return the file path. - # llama_cpp requires a file path, not a directory, so snapshot_download + # llama_server requires a file path, not a directory, so snapshot_download # would break it. The implicit "the only GGUF" is unambiguous. if len(ggufs) == 1: logger.info("HF repo %r has a single GGUF (%s); resolving to its file path", source, ggufs[0]) diff --git a/modelship/infer/stable_diffusion_cpp/stable_diffusion_cpp_infer.py b/modelship/infer/stable_diffusion_cpp/stable_diffusion_cpp_infer.py index 5646fef..1025970 100644 --- a/modelship/infer/stable_diffusion_cpp/stable_diffusion_cpp_infer.py +++ b/modelship/infer/stable_diffusion_cpp/stable_diffusion_cpp_infer.py @@ -23,7 +23,7 @@ class StableDiffusionCppInfer(BaseInfer): """CPU-only image-generation loader backed by stable-diffusion.cpp (via the stable-diffusion-cpp-python bindings). Loads GGUF-quantized single-file diffusion checkpoints (SD1.5/SDXL/SD-Turbo, all-in-one Flux) and serves the - OpenAI images endpoints. Structurally mirrors the llama_cpp loader.""" + OpenAI images endpoints.""" def __init__(self, model_config: ModelshipModelConfig): super().__init__(model_config) @@ -40,7 +40,7 @@ def __init__(self, model_config: ModelshipModelConfig): merged = merge_with_user_overrides(recommendation, user_overrides, model_name=model_config.name) self.config = user_config.model_copy(update=merged) - # Verbose native logging when MSHIP_LOG_LEVEL is TRACE (mirrors llama_cpp). + # Verbose native logging when MSHIP_LOG_LEVEL is TRACE. mship_log_level = os.environ.get("MSHIP_LOG_LEVEL", "INFO").upper() self._verbose = mship_log_level == "TRACE" diff --git a/modelship/logging.py b/modelship/logging.py index 4a99437..8eb8235 100644 --- a/modelship/logging.py +++ b/modelship/logging.py @@ -54,7 +54,6 @@ def format(self, record): "vllm", "transformers", "diffusers", - "llama_cpp", "flashinfer", "huggingface_hub", ) diff --git a/modelship/openai/parsers/streaming.py b/modelship/openai/parsers/streaming.py index 09fc723..f57d0bd 100644 --- a/modelship/openai/parsers/streaming.py +++ b/modelship/openai/parsers/streaming.py @@ -1,8 +1,7 @@ """Loader-agnostic chat-completion helpers for parser-aware responses. -Loaders produce text differently — HF ``Pipeline`` for transformers, raw -``llama.create_completion`` for llama_cpp, async iterators from plugins — -but the OpenAI response shapes (streaming and non-streaming) are the +Loaders produce text differently — HF ``Pipeline`` for transformers, async +iterators from plugins — but the OpenAI response shapes (streaming and non-streaming) are the same. This module owns those shapes plus the :class:`~modelship.openai.parsers.output.ChatOutputStreamer` driving loop, so the loaders only deal in plain text. @@ -232,10 +231,9 @@ def parse_chat_completion_text( ) -> ParsedChatOutput: """Run the shared :class:`ChatOutputStreamer` over a full completion text. - Public because non-streaming loader paths outside this module (e.g. - llama_cpp's ``_handle_with_parsers``) parse text themselves and feed - the result into :func:`modelship.openai.chat_utils.build_from_parsed` - rather than going through :func:`build_chat_completion_response`. + Public so a non-streaming loader that parses text itself can feed the + result into :func:`modelship.openai.chat_utils.build_from_parsed` rather + than going through :func:`build_chat_completion_response`. """ streamer = _make_streamer( parser_name=parser_name, diff --git a/modelship/openai/parsers/tool_calling/input.py b/modelship/openai/parsers/tool_calling/input.py index 01233ed..4b28f57 100644 --- a/modelship/openai/parsers/tool_calling/input.py +++ b/modelship/openai/parsers/tool_calling/input.py @@ -4,7 +4,7 @@ tool schemas use these helpers to interpret the request's ``tool_choice``: ``resolve_tools_for_request`` shapes the tools list passed into the prompt and ``request_forces_tool_call`` reports whether the request mandates a tool call. -Enforcement of that mandate is the loader's job (e.g. the llama_cpp tool-call +Enforcement of that mandate is the loader's job (e.g. a constrained-decoding grammar) — these helpers only express intent. """ diff --git a/modelship/openai/protocol/chat.py b/modelship/openai/protocol/chat.py index 29ae3d2..df08fbe 100644 --- a/modelship/openai/protocol/chat.py +++ b/modelship/openai/protocol/chat.py @@ -101,9 +101,9 @@ def _validate_tools_response_format_compat(self) -> "ChatCompletionRequest": # markers a model would use to emit a tool call (..., # <|python_tag|>..., etc.). The two cannot meaningfully coexist on any # loader we support: vLLM passes both through but the grammar dominates; - # llama-cpp-python silently drops json_schema; transformers has no - # native machinery to compose them. Reject upfront so callers don't - # discover the conflict by watching tool calls never fire in prod. + # transformers has no native machinery to compose them. Reject upfront + # so callers don't discover the conflict by watching tool calls never + # fire in prod. if not self.tools or not self.response_format: return self fmt_type = self.response_format.get("type") diff --git a/modelship/openai/protocol/responses/adapter.py b/modelship/openai/protocol/responses/adapter.py index d03fbe6..98e4207 100644 --- a/modelship/openai/protocol/responses/adapter.py +++ b/modelship/openai/protocol/responses/adapter.py @@ -334,7 +334,7 @@ def _usage_from_chat(usage: UsageInfo) -> ResponseUsage: caching / reasoning models) under the OpenAI-standard ``prompt_tokens_details`` / ``completion_tokens_details``. Responses uses the same sub-field names, so this is a direct field-to-field copy; loaders that report no details - (llama_cpp/transformers) leave them at the zero default. + (llama_server/transformers) leave them at the zero default. """ prompt_details = usage.prompt_tokens_details completion_details = usage.completion_tokens_details diff --git a/modelship/openai/protocol/responses/streaming.py b/modelship/openai/protocol/responses/streaming.py index a1a59a0..1cb6a69 100644 --- a/modelship/openai/protocol/responses/streaming.py +++ b/modelship/openai/protocol/responses/streaming.py @@ -3,7 +3,7 @@ The chat loaders all emit Server-Sent Events as ``data: {chunk}\\n\\n`` strings (``ChatCompletionStreamResponse`` payloads) terminated by ``data: [DONE]`` — this is the one wire shape vLLM and the ``ChatOutputStreamer`` loaders (transformers, -llama_cpp, plugins) share. The Responses API instead wants a *semantic* event +plugins) share. The Responses API instead wants a *semantic* event protocol: named events (``response.created``, ``response.output_text.delta``, …) each carrying a monotonically increasing ``sequence_number`` plus the relevant output index, wrapping each output item in explicit added/done brackets. diff --git a/modelship/preflight/base.py b/modelship/preflight/base.py index a3aa89b..66bb444 100644 --- a/modelship/preflight/base.py +++ b/modelship/preflight/base.py @@ -380,13 +380,6 @@ def recommend(self, config: ModelshipModelConfig, hw: HardwareProfile) -> dict[s def _ensure_registered() -> None: if ModelLoader.custom not in _REGISTRY: register(ModelLoader.custom, _CustomPluginPreflight()) - if ModelLoader.llama_cpp not in _REGISTRY: - try: - from modelship.preflight.llama_cpp import LlamaCppPreflight - - register(ModelLoader.llama_cpp, LlamaCppPreflight()) - except Exception: - logger.debug("preflight: LlamaCppPreflight registration skipped", exc_info=True) if ModelLoader.llama_server not in _REGISTRY: try: from modelship.preflight.llama_cpp import LlamaServerPreflight diff --git a/modelship/preflight/llama_cpp.py b/modelship/preflight/llama_cpp.py index 744f0ee..14f401d 100644 --- a/modelship/preflight/llama_cpp.py +++ b/modelship/preflight/llama_cpp.py @@ -10,7 +10,7 @@ logger = get_logger("preflight.llama_cpp") # Fraction of total system RAM the preflight will allocate. No equivalent of -# vLLM's `gpu_memory_utilization` exists in LlamaCppConfig; 0.8 leaves room +# vLLM's `gpu_memory_utilization` exists for this loader; 0.8 leaves room # for the OS, page cache, and other actors on the same node. _RAM_UTILIZATION = 0.8 @@ -65,7 +65,7 @@ def recommend(self, config: ModelshipModelConfig, hw: HardwareProfile) -> dict[s logger.info("preflight '%s': skipping — GGUF metadata unreadable at %s", config.name, model_path) return {} - kv_per_token = _kv_bytes_per_token(meta, config) + kv_per_token = _kv_bytes_per_token(meta) if kv_per_token is None: logger.warning( "preflight '%s': skipping — GGUF metadata missing KV-cache geometry " @@ -306,52 +306,10 @@ def _read_string(reader: Any, key: str) -> str | None: return str(val) -def _kv_bytes_per_token(meta: _GGUFMeta, config: ModelshipModelConfig) -> int | None: +def _kv_bytes_per_token(meta: _GGUFMeta) -> int | None: """Bytes of KV cache stored per token across all layers. `2 *` accounts for both K and V tensors. KV element size defaults to fp16 - (2 bytes); user can override via `model_kwargs.type_k`/`type_v`.""" - kv_dtype_bytes = _resolve_kv_dtype_bytes(config) - return 2 * meta.block_count * meta.head_count_kv * meta.head_dim * kv_dtype_bytes - - -def _resolve_kv_dtype_bytes(config: ModelshipModelConfig) -> int: - """llama.cpp's KV cache defaults to fp16 unless `type_k`/`type_v` are set - in `model_kwargs`. We accept either GGML type codes (ints) or string - aliases like `"f16"`, `"q8_0"`. Anything we don't recognize falls back to - fp16 — preflight stays conservative.""" - llama_cfg = config.llama_cpp_config - if llama_cfg is None: - return _DEFAULT_KV_DTYPE_BYTES - kwargs = llama_cfg.model_kwargs or {} - # Use the larger of the two if they disagree — KV cache bytes per token - # is the sum of K and V, both already counted with the `2 *` factor. - sizes = [_ggml_type_bytes(kwargs.get(k)) for k in ("type_k", "type_v") if k in kwargs] - sizes = [s for s in sizes if s is not None] - if not sizes: - return _DEFAULT_KV_DTYPE_BYTES - return max(sizes) - - -# GGML type aliases used by `type_k`/`type_v`. We only enumerate the formats -# people actually run as KV cache; anything else returns None and we fall back -# to fp16. Codes match `enum ggml_type` in llama.cpp/ggml.h. -_GGML_TYPE_BYTES_BY_NAME = { - "f32": 4, - "f16": 2, - "bf16": 2, - "q8_0": 1, # nominal bytes/elem — actual is 1.0625 with block overhead - "q4_0": 1, # nominal; ~0.5 bytes/elem incl. block overhead, round up safely -} - - -def _ggml_type_bytes(value: Any) -> int | None: - if value is None: - return None - if isinstance(value, int): - # Treat the GGML enum value as opaque — only the common float types - # are predictable. 0=F32, 1=F16, 30=BF16 in current ggml. - return {0: 4, 1: 2, 30: 2}.get(value) - if isinstance(value, str): - return _GGML_TYPE_BYTES_BY_NAME.get(value.lower()) - return None + (2 bytes) — `llama_server`'s config surface has no `type_k`/`type_v` + override, so preflight always assumes it.""" + return 2 * meta.block_count * meta.head_count_kv * meta.head_dim * _DEFAULT_KV_DTYPE_BYTES diff --git a/modelship/preflight/stable_diffusion_cpp.py b/modelship/preflight/stable_diffusion_cpp.py index 296ec25..5eb4c12 100644 --- a/modelship/preflight/stable_diffusion_cpp.py +++ b/modelship/preflight/stable_diffusion_cpp.py @@ -19,7 +19,7 @@ class StableDiffusionCppPreflight: """Hardware-aware defaults for the stable_diffusion_cpp loader. Conservative in v1: the only recommendation is enabling VAE tiling on low-RAM hosts. Sizing - by model footprint (like the llama_cpp preflight does for n_ctx) is a + by model footprint (like the llama_server preflight does for n_ctx) is a follow-up once split-file resolution lands.""" def recommend(self, config: ModelshipModelConfig, hw: HardwareProfile) -> dict[str, Any]: diff --git a/pyproject.toml b/pyproject.toml index 6b5716e..6bbb002 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,6 @@ gpu = [ "accelerate>=1.6.0", "bitsandbytes>=0.49.0", "diffusers>=0.31.0", - "llama-cpp-python>=0.3.32", "stable-diffusion-cpp-python>=0.4.7", "onnxruntime-gpu>=1.20.1", "nvidia-ml-py", @@ -58,7 +57,6 @@ cpu = [ "torchvision>=0.25.0", "transformers>=5.5.3", "sentence-transformers>=3.0.0", - "llama-cpp-python>=0.3.32", "stable-diffusion-cpp-python>=0.4.7", "onnxruntime>=1.20.1", "vllm==0.24.0", @@ -104,11 +102,6 @@ name = "pytorch-cu130" url = "https://download.pytorch.org/whl/cu130" explicit = true -[[tool.uv.index]] -name = "llama-cpp-cu130" -url = "https://abetlen.github.io/llama-cpp-python/whl/cu130" -explicit = true - # NOTE: this URL embeds the vllm version — a future bump of vllm==0.24.0 above # must update this to match, or the cpu extra silently falls back to whatever # (if anything) that path resolves to. @@ -130,9 +123,6 @@ torchvision = [ { index = "pytorch-cu130", extra = "gpu" }, { index = "pytorch-cpu", extra = "cpu" }, ] -llama-cpp-python = [ - { index = "llama-cpp-cu130", extra = "gpu" }, -] vllm = [ { index = "vllm-cpu", extra = "cpu" }, ] @@ -177,7 +167,6 @@ known-first-party = ["modelship"] [tool.pytest.ini_options] markers = [ "integration: tests that require a running Ray cluster and external models", - "llama_cpp: integration tests that exercise the llama_cpp loader", "llama_server: integration tests that exercise the llama_server loader", "vllm: integration tests that exercise the vllm loader", "diffusers: integration tests that exercise the diffusers loader", diff --git a/tests/test_config.py b/tests/test_config.py index a2e1a9f..14ee135 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,7 +5,6 @@ from modelship.infer.infer_config import ( AutoscalingConfig, - LlamaCppConfig, LlamaServerConfig, ModelLoader, ModelshipConfig, @@ -16,161 +15,6 @@ ) -class TestLlamaCppConfig: - def test_defaults(self): - config = LlamaCppConfig() - assert config.n_gpu_layers == -1 - assert config.n_ctx == 2048 - assert config.n_batch == 512 - assert config.chat_format is None - assert config.model_kwargs == {} - assert config.cache is None - - def test_cache_defaults(self): - config = LlamaCppConfig(cache={}) - assert config.cache is not None - assert config.cache.type == "ram" - assert config.cache.capacity == "2GiB" - assert config.cache.capacity_bytes == 2 << 30 - - def test_cache_disk(self): - config = LlamaCppConfig(cache={"type": "disk", "capacity": "512MB"}) - assert config.cache is not None - assert config.cache.type == "disk" - assert config.cache.capacity_bytes == 512 * 1000**2 - - @pytest.mark.parametrize( - ("capacity", "expected_bytes"), - [ - ("2GiB", 2 << 30), - ("512MB", 512 * 1000**2), - ("1.5gb", int(1.5 * 1000**3)), - ("100mib", 100 << 20), - ("4096", 4096), - (1024, 1024), - ], - ) - def test_cache_capacity_parsing(self, capacity, expected_bytes): - config = LlamaCppConfig(cache={"capacity": capacity}) - assert config.cache is not None - assert config.cache.capacity_bytes == expected_bytes - - def test_cache_rejects_invalid_type(self): - with pytest.raises(ValidationError): - LlamaCppConfig(cache={"type": "gpu"}) - - @pytest.mark.parametrize("capacity", ["0", "-5MB", "2PB", "abc", ""]) - def test_cache_rejects_invalid_capacity(self, capacity): - with pytest.raises(ValidationError): - LlamaCppConfig(cache={"capacity": capacity}) - - def _disk_cache_model(self, **overrides): - return ModelshipModelConfig( - name="qwen-gguf", - model="repo/Qwen-GGUF:*Q4_K_M.gguf", - usecase=ModelUsecase.generate, - loader=ModelLoader.llama_cpp, - llama_cpp_config=LlamaCppConfig(cache={"type": "disk"}), - **overrides, - ) - - def test_disk_cache_single_replica_allowed(self): - config = self._disk_cache_model() - assert config.llama_cpp_config.cache.type == "disk" - - def test_disk_cache_rejects_multiple_num_replicas(self): - with pytest.raises(ValidationError, match="not process-safe"): - self._disk_cache_model(num_replicas=2) - - def test_disk_cache_rejects_autoscaling_above_one(self): - with pytest.raises(ValidationError, match="not process-safe"): - self._disk_cache_model(autoscaling_config=AutoscalingConfig(min_replicas=1, max_replicas=3)) - - def test_disk_cache_allows_scale_to_zero_single_max(self): - config = self._disk_cache_model(autoscaling_config=AutoscalingConfig(min_replicas=0, max_replicas=1)) - assert config.llama_cpp_config.cache.type == "disk" - - def _num_gpus_model(self, num_gpus): - return ModelshipModelConfig( - name="qwen-gguf", - model="repo/Qwen-GGUF:*Q4_K_M.gguf", - usecase=ModelUsecase.generate, - loader=ModelLoader.llama_cpp, - num_gpus=num_gpus, - ) - - def test_num_gpus_integer_allowed(self): - config = self._num_gpus_model(1) - assert config.num_gpus == 1 - - def test_num_gpus_zero_allowed(self): - config = self._num_gpus_model(0) - assert config.num_gpus == 0 - - def test_num_gpus_fractional_rejected(self): - with pytest.raises(ValidationError, match="not allowed for the llama_cpp loader"): - self._num_gpus_model(0.5) - - def test_num_gpus_fractional_still_allowed_on_vllm(self): - config = ModelshipModelConfig( - name="qwen-vllm", - model="repo/Qwen", - usecase=ModelUsecase.generate, - loader=ModelLoader.vllm, - num_gpus=0.5, - ) - assert config.num_gpus == 0.5 - - def test_disk_cache_ignored_for_non_llama_cpp_loader(self): - # A leftover llama_cpp_config on a different loader is ignored, so the - # disk-cache multi-replica guard must not fire. - config = ModelshipModelConfig( - name="qwen-vllm", - model="Qwen/Qwen2.5-7B-Instruct", - usecase=ModelUsecase.generate, - loader=ModelLoader.vllm, - llama_cpp_config=LlamaCppConfig(cache={"type": "disk"}), - num_replicas=2, - ) - assert config.num_replicas == 2 - - def test_ram_cache_allows_multiple_replicas(self): - config = ModelshipModelConfig( - name="qwen-gguf", - model="repo/Qwen-GGUF:*Q4_K_M.gguf", - usecase=ModelUsecase.generate, - loader=ModelLoader.llama_cpp, - llama_cpp_config=LlamaCppConfig(cache={"type": "ram"}), - num_replicas=4, - ) - assert config.num_replicas == 4 - - def test_custom_values(self): - config = LlamaCppConfig( - n_gpu_layers=33, - n_ctx=4096, - n_batch=1024, - chat_format="llama-3", - model_kwargs={"seed": 42}, - ) - assert config.n_gpu_layers == 33 - assert config.n_ctx == 4096 - assert config.n_batch == 1024 - assert config.chat_format == "llama-3" - assert config.model_kwargs == {"seed": 42} - - def test_llama_cpp_model_config(self): - config = ModelshipModelConfig( - name="llama-3", - model="meta-llama/Llama-3-8B-Instruct-GGUF:*Q4_K_M.gguf", - usecase=ModelUsecase.generate, - loader=ModelLoader.llama_cpp, - llama_cpp_config=LlamaCppConfig(), - ) - assert config.loader == ModelLoader.llama_cpp - assert config.model == "meta-llama/Llama-3-8B-Instruct-GGUF:*Q4_K_M.gguf" - - class TestLlamaServerConfig: def test_defaults(self): config = LlamaServerConfig() @@ -251,7 +95,7 @@ def test_chat_template_kwargs_round_trips(self): "name": "qwen3", "model": "some-org/qwen3", "usecase": ModelUsecase.generate, - "loader": ModelLoader.llama_cpp, + "loader": ModelLoader.llama_server, "chat_template_kwargs": {"enable_thinking": False}, } ) diff --git a/tests/test_effective_config.py b/tests/test_effective_config.py index 4120c05..91541cc 100644 --- a/tests/test_effective_config.py +++ b/tests/test_effective_config.py @@ -19,7 +19,7 @@ def _model(name: str, **overrides) -> dict: """A minimal raw model dict (the form the store holds).""" - base = {"name": name, "model": f"org/{name}", "usecase": "generate", "loader": "llama_cpp"} + base = {"name": name, "model": f"org/{name}", "usecase": "generate", "loader": "llama_server"} base.update(overrides) return base diff --git a/tests/test_integration.py b/tests/test_integration.py index ebd691a..de56c47 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -68,34 +68,6 @@ "mm_processor_kwargs": {"min_pixels": 50176, "max_pixels": 200704}, }, }, - "chat-limited": { - "name": "chat-limited", - "model": "lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF:*Q4_K_M.gguf", - "usecase": "generate", - "loader": "llama_cpp", - "num_cpus": 1, - "llama_cpp_config": { - "tool_calls_enabled": False, - }, - }, - "chat-llama-mship": { - "name": "chat-llama-mship", - "model": "lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF:*Q4_K_M.gguf", - "usecase": "generate", - "loader": "llama_cpp", - "num_cpus": 1, - }, - "chat-llama-gpu": { - "name": "chat-llama-gpu", - # Same GGUF as chat-llama-mship, but on a whole GPU to exercise the - # cu130-wheel offload path end-to-end (actor GPU allocation, - # n_gpu_layers honored, preflight RAM sizing skipped). - "model": "lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF:*Q4_K_M.gguf", - "usecase": "generate", - "loader": "llama_cpp", - "num_gpus": 1, - "num_cpus": 1, - }, "autoscale-llama": { "name": "autoscale-llama", # Tiny CPU GGUF so the host can hold several replicas (1 cpu each, up to @@ -105,7 +77,7 @@ # keep the test's poll windows tractable. "model": "lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF:*Q4_K_M.gguf", "usecase": "generate", - "loader": "llama_cpp", + "loader": "llama_server", "num_cpus": 1, "autoscaling_config": { "min_replicas": 1, @@ -115,32 +87,16 @@ "downscale_delay_s": 10, }, }, - "chat-llama-reasoning": { - "name": "chat-llama-reasoning", - # Qwen3-0.6B in GGUF form: same family as the vLLM `chat-reasoning` - # deployment (which uses the safetensors checkpoint), so the - # model emits `...` and supports Hermes-style tool - # calls in the same chat template. Lets us exercise reasoning, - # tools, and reasoning+tools through the llama_cpp loader in one - # deployment. Reasoning chains need headroom, so n_ctx is bumped. - "model": "lmstudio-community/Qwen3-0.6B-GGUF:*Q4_K_M.gguf", - "usecase": "generate", - "loader": "llama_cpp", - "num_cpus": 1, - "llama_cpp_config": { - "n_ctx": 4096, - }, - }, "chat-llama-server": { "name": "chat-llama-server", - # Same Qwen3-0.6B GGUF as `chat-llama-reasoning`, but through the new - # llama_server loader: a llama-server subprocess doing its own chat - # templating, tool-call, and reasoning parsing (`--jinja - # --reasoning-format auto`) instead of modelship's ChatOutputStreamer. - # `parallel: 4` exercises the loader's headline capability over - # llama_cpp: true multi-slot concurrency instead of one asyncio.Lock - # serializing every request. n_ctx is per-slot (the loader launches - # with `-c n_ctx*parallel`), bumped for reasoning headroom. + # Qwen3-0.6B GGUF through the llama_server loader: a llama-server + # subprocess doing its own chat templating, tool-call, and reasoning + # parsing (`--jinja --reasoning-format auto`) instead of modelship's + # ChatOutputStreamer. `parallel: 4` exercises the loader's headline + # capability: true multi-slot concurrency instead of a single + # asyncio.Lock serializing every request. n_ctx is per-slot (the + # loader launches with `-c n_ctx*parallel`), bumped for reasoning + # headroom. "model": "lmstudio-community/Qwen3-0.6B-GGUF:*Q4_K_M.gguf", "usecase": "generate", "loader": "llama_server", @@ -152,10 +108,9 @@ }, "chat-llama-server-plain": { "name": "chat-llama-server-plain", - # Same non-reasoning Qwen2.5-0.5B GGUF as `chat-llama-mship`, through - # the llama_server loader instead of llama_cpp. Used for the - # response_format tests mirrored from `TestChatLlamaCpp`, which need - # a model that doesn't emit a `...` preamble. + # Same non-reasoning Qwen2.5-0.5B GGUF as chat-llama-server, through + # the llama_server loader. Used for the response_format tests, which + # need a model that doesn't emit a `...` preamble. "model": "lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF:*Q4_K_M.gguf", "usecase": "generate", "loader": "llama_server", @@ -163,9 +118,9 @@ }, "chat-llama-server-gpu": { "name": "chat-llama-server-gpu", - # Same GGUF as chat-llama-server-plain, on a whole GPU — mirrors - # `chat-llama-gpu`'s coverage of the offload path (actor GPU - # allocation, -ngl honored) for the llama_server loader. + # Same GGUF as chat-llama-server-plain, on a whole GPU — exercises the + # llama_server loader's offload path (actor GPU allocation, -ngl + # honored). "model": "lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF:*Q4_K_M.gguf", "usecase": "generate", "loader": "llama_server", @@ -247,7 +202,7 @@ "chat-transformers-reasoning": { "name": "chat-transformers-reasoning", # Qwen3-0.6B safetensors — same family as the vLLM `chat-reasoning` - # and llama_cpp `chat-llama-reasoning` deployments. Lets us + # and llama_server `chat-llama-server` deployments. Lets us # exercise reasoning, tools, and reasoning+tools through the # transformers loader's `ChatOutputStreamer` wiring on real model # output. CPU-only and float32 to match the rest of the @@ -978,447 +933,14 @@ def test_tool_calling_streaming_transformers_loader(self, client): assert collected["finish_reason"] == "tool_calls" -@pytest.mark.integration -@pytest.mark.llama_cpp -class TestChatLlamaCpp: - @pytest.fixture(autouse=True, scope="class") - def _deploy(self, model_deployer): - model_deployer.deploy("chat-llama-mship") - - def test_tool_calling_llama_cpp_loader(self, client): - """Round-trip a Hermes-style tool call through the llama_cpp loader. - - Same Qwen2.5-0.5B-Instruct weights as `chat-capable` (vLLM) and - `chat-transformers`, but in GGUF form via llama-cpp-python. Auto-detected - hermes parser renders the prompt with `tools=...` and parses the - `{...}` markers out of raw completion output. - """ - completion = client.chat.completions.create( - model="chat-llama-mship", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - ) - tool_calls = completion.choices[0].message.tool_calls - assert tool_calls, f"expected a tool call, got content={completion.choices[0].message.content!r}" - assert tool_calls[0].function.name == "get_weather" - assert "Paris" in tool_calls[0].function.arguments - assert completion.choices[0].finish_reason == "tool_calls" - - def test_tool_calling_streaming_llama_cpp_loader(self, client): - """Stream a tool call through the llama_cpp loader and verify the - delta sequence matches the OpenAI streaming contract. - - Same shape as `test_tool_calling_streaming_transformers_loader` — - asserts a single name delta, multiple incremental argument deltas, - valid JSON on concatenation, and final ``finish_reason="tool_calls"``. - """ - stream = client.chat.completions.create( - model="chat-llama-mship", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - stream=True, - ) - - collected = _collect_streaming_tool_call(stream) - - assert collected["tool_calls"], ( - f"expected at least one streamed tool call; got content={collected['content']!r}" - ) - call_0 = collected["tool_calls"][0] - assert call_0["id"], "expected an id on the first tool-call delta" - assert call_0["name"] == "get_weather" - assert collected["name_deltas"] == 1, f"expected one name delta, got {collected['name_deltas']}" - assert collected["args_deltas"] >= 2, ( - f"expected arguments to stream incrementally, got {collected['args_deltas']} args delta(s)" - ) - parsed_args = json.loads(call_0["arguments"]) - assert parsed_args.get("city") - assert "Paris" in parsed_args["city"] - assert collected["finish_reason"] == "tool_calls" - - def test_response_format_json_object_constrains_unprompted_output(self, client): - """Prompt is natural-language; the grammar constraint produces JSON.""" - completion = client.chat.completions.create( - model="chat-llama-mship", - messages=[{"role": "user", "content": "What is the capital of France?"}], - response_format={"type": "json_object"}, - max_tokens=64, - ) - content = completion.choices[0].message.content - assert content - parsed = json.loads(content) - assert isinstance(parsed, dict) - - def test_response_format_json_schema_constrains_unprompted_output(self, client): - """A natural-language question + json_schema → schema-conformant output.""" - schema = { - "type": "object", - "properties": { - "city": {"type": "string"}, - "country": {"type": "string"}, - }, - "required": ["city", "country"], - "additionalProperties": False, - } - completion = client.chat.completions.create( - model="chat-llama-mship", - messages=[{"role": "user", "content": "Where is the Eiffel Tower located?"}], - response_format={ - "type": "json_schema", - "json_schema": {"name": "location", "schema": schema, "strict": True}, - }, - max_tokens=64, - ) - content = completion.choices[0].message.content - assert content - parsed = json.loads(content) - assert set(parsed.keys()) == {"city", "country"} - assert isinstance(parsed["city"], str) and parsed["city"] - assert isinstance(parsed["country"], str) and parsed["country"] - - def test_response_format_json_schema_streaming_constrains_unprompted_output(self, client): - """Same intent on the streaming path.""" - schema = { - "type": "object", - "properties": {"answer": {"type": "string"}}, - "required": ["answer"], - "additionalProperties": False, - } - stream = client.chat.completions.create( - model="chat-llama-mship", - messages=[{"role": "user", "content": "What is the capital of France?"}], - response_format={ - "type": "json_schema", - "json_schema": {"name": "answer", "schema": schema, "strict": True}, - }, - max_tokens=64, - stream=True, - ) - chunks = [] - for chunk in stream: - if chunk.choices and chunk.choices[0].delta.content: - chunks.append(chunk.choices[0].delta.content) - content = "".join(chunks) - assert content - parsed = json.loads(content) - assert set(parsed.keys()) == {"answer"} - assert isinstance(parsed["answer"], str) and parsed["answer"] - - def test_response_format_coexists_with_tool_choice_none(self, client): - """tool_choice='none' is the safe escape valve: tools listed but inert, - schema enforced on content output. - """ - schema = { - "type": "object", - "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, - "required": ["city", "country"], - "additionalProperties": False, - } - completion = client.chat.completions.create( - model="chat-llama-mship", - messages=[{"role": "user", "content": "Where is the Eiffel Tower located?"}], - tools=[_WEATHER_TOOL], - tool_choice="none", - response_format={ - "type": "json_schema", - "json_schema": {"name": "location", "schema": schema, "strict": True}, - }, - max_tokens=64, - ) - assert not completion.choices[0].message.tool_calls - content = completion.choices[0].message.content - assert content - parsed = json.loads(content) - assert set(parsed.keys()) == {"city", "country"} - - def test_response_format_with_active_tools_rejected_by_gateway(self): - """Protocol-layer validator rejects tools + response_format when - tool_choice is anything other than 'none'. The schema grammar would - block tool-call markers from being emitted, so we surface the conflict - upfront rather than silently breaking tool calling. - """ - response = httpx.post( - f"{OPENAI_API_BASE}/chat/completions", - json={ - "model": "chat-llama-mship", - "messages": [{"role": "user", "content": "What is the weather in Paris?"}], - "tools": [_WEATHER_TOOL], - "tool_choice": "auto", - "response_format": { - "type": "json_schema", - "json_schema": { - "name": "unused", - "schema": {"type": "object", "properties": {"x": {"type": "string"}}}, - "strict": True, - }, - }, - }, - timeout=30, - ) - assert response.status_code in (400, 422), ( - f"expected client-error status, got {response.status_code}: {response.text}" - ) - assert "tool_choice='none'" in response.text - - -@pytest.mark.integration -@pytest.mark.llama_cpp -class TestChatLlamaCppGpu: - """End-to-end GPU offload through the llama_cpp loader. - - Same GGUF and tool-calling shape as `TestChatLlamaCpp` (CPU), but deployed - with `num_gpus=1` so the actor gets a whole GPU and the cu130 wheel's - `llama_supports_gpu_offload()` gate honors `n_gpu_layers` instead of - forcing CPU-only. Proves the offload path end-to-end rather than - re-covering the CPU loader's response-format/streaming behavior. - """ - - @pytest.fixture(autouse=True, scope="class") - def _deploy(self, model_deployer): - model_deployer.deploy("chat-llama-gpu") - - def test_chat_completion(self, client): - completion = client.chat.completions.create( - model="chat-llama-gpu", - messages=[{"role": "user", "content": "What is the capital of France?"}], - max_tokens=32, - ) - content = completion.choices[0].message.content - assert content - assert "Paris" in content - - def test_tool_calling_llama_cpp_gpu_loader(self, client): - completion = client.chat.completions.create( - model="chat-llama-gpu", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - ) - tool_calls = completion.choices[0].message.tool_calls - assert tool_calls, f"expected a tool call, got content={completion.choices[0].message.content!r}" - assert tool_calls[0].function.name == "get_weather" - assert "Paris" in tool_calls[0].function.arguments - assert completion.choices[0].finish_reason == "tool_calls" - - -@pytest.mark.integration -@pytest.mark.llama_cpp -class TestChatLlamaCppReasoning: - """End-to-end reasoning + tool calling through the llama_cpp loader. - - Same Qwen3-0.6B family as the vLLM `chat-reasoning` deployment but - in GGUF form, so the modelship-side ``ChatOutputStreamer`` is - actually exercised on real model output (vLLM has its own native - reasoning parser, llama_cpp does not). One deployment covers - three scenarios because Qwen3 emits ``...`` AND - supports Hermes-style tool calls in the same chat template. - """ - - @pytest.fixture(autouse=True, scope="class") - def _deploy(self, model_deployer): - model_deployer.deploy("chat-llama-reasoning") - - def test_reasoning_completion_llama_cpp(self): - """Non-streaming: ``...`` block routes to ``message.reasoning``, - the final answer lands in ``message.content``, no marker leakage.""" - response = httpx.post( - f"{OPENAI_API_BASE}/chat/completions", - json={ - "model": "chat-llama-reasoning", - "messages": [{"role": "user", "content": "Briefly: what is 7 times 8?"}], - "max_tokens": 1024, - }, - timeout=300, - ) - assert response.status_code == 200, response.text - message = response.json()["choices"][0]["message"] - assert message.get("reasoning"), f"expected reasoning content, got {message!r}" - assert "" not in (message.get("content") or "") - assert "" not in (message.get("content") or "") - assert "" not in message["reasoning"] - assert "" not in message["reasoning"] - - def test_reasoning_streaming_llama_cpp(self): - """Streaming: at least one delta carries ``reasoning``; concatenated - reasoning is non-empty; markers never leak into either field.""" - with httpx.stream( - "POST", - f"{OPENAI_API_BASE}/chat/completions", - json={ - "model": "chat-llama-reasoning", - "messages": [{"role": "user", "content": "Briefly: what is 7 times 8?"}], - "max_tokens": 1024, - "stream": True, - }, - timeout=300, - ) as response: - assert response.status_code == 200 - reasoning_parts: list[str] = [] - content_parts: list[str] = [] - reasoning_deltas = 0 - for line in response.iter_lines(): - if not line.startswith("data: "): - continue - payload = line[len("data: ") :] - if payload == "[DONE]": - break - chunk = json.loads(payload) - delta = chunk["choices"][0].get("delta") or {} - if delta.get("reasoning"): - reasoning_parts.append(delta["reasoning"]) - reasoning_deltas += 1 - if delta.get("content"): - content_parts.append(delta["content"]) - - assert reasoning_deltas >= 1, "expected at least one reasoning delta" - assert "".join(reasoning_parts).strip(), "expected non-empty reasoning content" - assert "" not in "".join(reasoning_parts) - assert "" not in "".join(reasoning_parts) - assert "" not in "".join(content_parts) - assert "" not in "".join(content_parts) - - def test_reasoning_with_tools_llama_cpp(self, client): - """Reasoning + tool calling in one round-trip. - - Asserts that when a reasoning model also emits a tool call, the - single-pass ``ChatOutputStreamer`` populates BOTH - ``message.reasoning`` and ``message.tool_calls``, and that - ``finish_reason="tool_calls"``. - """ - completion = client.chat.completions.create( - model="chat-llama-reasoning", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=1024, - ) - message = completion.choices[0].message - # The OpenAI Python SDK exposes unknown fields via ``model_extra``. - reasoning = getattr(message, "reasoning", None) or message.model_extra.get("reasoning") - assert reasoning, f"expected reasoning, got message={message!r}" - assert "" not in reasoning - tool_calls = message.tool_calls - assert tool_calls, f"expected a tool call, got content={message.content!r}, reasoning={reasoning!r}" - assert tool_calls[0].function.name == "get_weather" - assert "Paris" in tool_calls[0].function.arguments - assert completion.choices[0].finish_reason == "tool_calls" - - def test_tool_markers_inside_reasoning_not_double_counted(self, client): - """Tool-call markers emitted *inside* ```` must route to - reasoning, never become real tool calls. - - Coaxes the model into illustrating tool-call syntax inside its - reasoning and then making one actual call. The single-pass - ``ChatOutputStreamer`` must: - - - Surface the illustrative ``...`` text - inside ``message.reasoning`` (proving it was treated as - reasoning bytes, not a real call). - - Emit exactly ONE ``tool_calls`` entry (the actual post- - reasoning call), not multiples. - - Real models are non-deterministic; if the prompt fails to - produce literal markers in reasoning, we skip the - marker-routing assertion rather than flake — the - single-tool-call assertion still has value either way. The - deterministic equivalent is exercised in unit tests - (``tests/test_reasoning.py::TestComposition``). - """ - completion = client.chat.completions.create( - model="chat-llama-reasoning", - messages=[ - { - "role": "system", - "content": ( - "You are an assistant with access to tools. When you think inside " - "..., FIRST quote one example of tool-call syntax " - "verbatim inside angle brackets — e.g. write the literal text " - '{"name":"example","arguments":{}} as part ' - "of your reasoning to remind yourself of the format. THEN decide " - "which real tool to call." - ), - }, - {"role": "user", "content": "What is the weather in Paris?"}, - ], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=1024, - ) - message = completion.choices[0].message - reasoning = getattr(message, "reasoning", None) or message.model_extra.get("reasoning") or "" - tool_calls = message.tool_calls or [] - - # Hard assertions: regardless of prompt compliance, the streamer - # must produce exactly one real tool call for the weather query. - assert tool_calls, ( - f"expected exactly one real tool call, got content={message.content!r}, reasoning={reasoning!r}" - ) - assert len(tool_calls) == 1, ( - f"expected exactly one tool call (markers inside must not be double-counted); " - f"got {len(tool_calls)} calls={[tc.function.name for tc in tool_calls]}" - ) - assert tool_calls[0].function.name == "get_weather" - assert completion.choices[0].finish_reason == "tool_calls" - - # Soft assertion: only meaningful if the model actually quoted - # the marker syntax inside its reasoning. - if "" in reasoning: - # Reasoning carries the literal marker text — confirms the - # streamer routed it to the reasoning view rather than - # parsing it as a real call (which would have shown up as a - # second tool_calls entry). - assert "" in reasoning, ( - f"reasoning has an unmatched marker (open without close): {reasoning!r}" - ) - - def test_response_format_with_reasoning_deployment_rejected(self): - """A JSON grammar would exclude the `<` token, breaking the reasoning - parser's `...` emission. The loader rejects the combination at - request time rather than producing malformed output. - """ - response = httpx.post( - f"{OPENAI_API_BASE}/chat/completions", - json={ - "model": "chat-llama-reasoning", - "messages": [{"role": "user", "content": "What is 2+2?"}], - "response_format": { - "type": "json_schema", - "json_schema": { - "name": "answer", - "schema": { - "type": "object", - "properties": {"answer": {"type": "string"}}, - "required": ["answer"], - }, - "strict": True, - }, - }, - "max_tokens": 256, - }, - timeout=60, - ) - assert response.status_code == 400, f"expected 400, got {response.status_code}: {response.text}" - assert "reasoning" in response.text.lower() - - @pytest.mark.integration @pytest.mark.llama_server class TestChatLlamaServer: """End-to-end chat, tool calling, reasoning, and concurrency through the - new `llama_server` loader (a `llama-server` subprocess proxied over its - native OpenAI-compatible HTTP API, rather than in-process llama-cpp-python - bindings + modelship's own parsers). - - Same Qwen3-0.6B-GGUF weights as `TestChatLlamaCppReasoning`'s - `chat-llama-reasoning` deployment, so these tests double as a parity check - between the two loaders on identical model output — except reasoning and - tool-call parsing is llama-server's own (`--jinja --reasoning-format - auto`), not modelship's `ChatOutputStreamer`. + `llama_server` loader (a `llama-server` subprocess proxied over its + native OpenAI-compatible HTTP API). Reasoning and tool-call parsing is + llama-server's own (`--jinja --reasoning-format auto`), not modelship's + `ChatOutputStreamer`. """ @pytest.fixture(autouse=True, scope="class") @@ -1427,9 +949,9 @@ def _deploy(self, model_deployer): def test_chat_completion(self, client): # This deployment is Qwen3-0.6B (reasoning-capable), unlike - # `chat-llama-mship`'s plain Qwen2.5 — it always emits a `...` - # preamble before content, so the token budget needs headroom for - # reasoning to finish, not just the answer itself. + # `chat-llama-server-plain`'s plain Qwen2.5 — it always emits a + # `...` preamble before content, so the token budget needs + # headroom for reasoning to finish, not just the answer itself. completion = client.chat.completions.create( model="chat-llama-server", messages=[{"role": "user", "content": "What is the capital of France?"}], @@ -1458,7 +980,7 @@ def test_tool_calling_llama_server_loader(self, client): def test_tool_calling_streaming_llama_server_loader(self, client): """Stream a tool call through llama-server and verify the delta sequence matches the OpenAI streaming contract, same shape as the - llama_cpp/transformers loader streaming tests.""" + transformers loader streaming tests.""" stream = client.chat.completions.create( model="chat-llama-server", messages=[{"role": "user", "content": "What is the weather in Paris?"}], @@ -1564,12 +1086,10 @@ def test_reasoning_with_tools_llama_server(self, client): assert completion.choices[0].finish_reason == "tool_calls" def test_tool_markers_inside_reasoning_not_double_counted_llama_server(self, client): - """Mirrors `TestChatLlamaCppReasoning.test_tool_markers_inside_reasoning_not_double_counted`, - but for llama-server's own parser instead of modelship's - `ChatOutputStreamer` — the bug pattern (a `...` - illustration quoted inside `` reasoning getting parsed as a - second, real call) is plausible for any single-pass parser, not just - modelship's. + """Verifies llama-server's own parser doesn't double-count a + `...` illustration quoted inside `` + reasoning as a second, real call — a bug pattern plausible for any + single-pass parser, not just modelship's `ChatOutputStreamer`. Coaxes the model into illustrating tool-call syntax inside its reasoning before making one actual call, and asserts exactly one real @@ -1618,9 +1138,7 @@ def test_tool_markers_inside_reasoning_not_double_counted_llama_server(self, cli ) def test_response_format_with_reasoning_llama_server(self, client): - """Free win vs. `llama_cpp` (which rejects this combo outright — see - `TestChatLlamaCppReasoning.test_response_format_with_reasoning_deployment_rejected`): - llama-server handles a JSON-schema `response_format` combined with + """llama-server handles a JSON-schema `response_format` combined with reasoning natively, routing `...` to `message.reasoning` and the schema-conforming JSON to `message.content`.""" completion = client.chat.completions.create( @@ -1690,11 +1208,11 @@ def test_named_function_tool_choice_falls_back_to_auto(self, client): assert message.content, f"expected the free-text branch to stay reachable, got message={message!r}" def test_concurrent_requests_are_not_serialized(self, client): - """The loader's headline win over llama_cpp: llama-server's - `--parallel` slots let several requests run concurrently instead of - being serialized behind a single `asyncio.Lock` (as `LlamaCppInfer` - is). Time one request, then several at once, and assert the - concurrent batch finishes well under what full serialization would take. + """The loader's headline capability: llama-server's `--parallel` slots + let several requests run concurrently instead of being serialized + behind a single lock. Time one request, then several at once, and + assert the concurrent batch finishes well under what full + serialization would take. """ prompt = { "model": "chat-llama-server", @@ -1714,7 +1232,7 @@ def test_concurrent_requests_are_not_serialized(self, client): future.result() concurrent_elapsed = time.monotonic() - start - # Full serialization (llama_cpp's single-lock behavior) would take + # Full serialization (a single-lock loader's behavior) would take # roughly concurrency * baseline; llama-server's parallel slots should # keep this well under that. assert concurrent_elapsed < baseline * (concurrency - 0.5), ( @@ -1726,8 +1244,8 @@ def test_concurrent_requests_are_not_serialized(self, client): @pytest.mark.integration @pytest.mark.llama_server class TestResponsesLlamaServer: - """The /v1/responses adapter is loader-agnostic: same smoke test as - `TestResponsesLlamaCpp`, run over the llama_server loader.""" + """The /v1/responses adapter is loader-agnostic: same smoke test shape + as vLLM's, run over the llama_server loader.""" @pytest.fixture(autouse=True, scope="class") def _deploy(self, model_deployer): @@ -1767,12 +1285,11 @@ def test_streaming_response_through_llama_server(self, client): @pytest.mark.integration @pytest.mark.llama_server class TestChatLlamaServerResponseFormat: - """Mirrors `TestChatLlamaCpp`'s response_format cluster for the - llama_server loader. Uses `chat-llama-server-plain` (non-reasoning - Qwen2.5-0.5B) rather than `chat-llama-server` (Qwen3, always emits a - `...` preamble) so these tests read the same as their llama_cpp - counterparts — response_format + reasoning together is covered - separately by `TestChatLlamaServer.test_response_format_with_reasoning_llama_server`. + """response_format tests for the llama_server loader. Uses + `chat-llama-server-plain` (non-reasoning Qwen2.5-0.5B) rather than + `chat-llama-server` (Qwen3, always emits a `...` preamble) — + response_format + reasoning together is covered separately by + `TestChatLlamaServer.test_response_format_with_reasoning_llama_server`. """ @pytest.fixture(autouse=True, scope="class") @@ -1780,8 +1297,7 @@ def _deploy(self, model_deployer): model_deployer.deploy("chat-llama-server-plain") def test_response_format_json_object_without_schema_is_unconstrained(self, client): - """Real gap vs. `llama_cpp`, discovered while mirroring this test: llama-server's - own docs claim bare `{"type": "json_object"}` (no `schema` key) produces + """llama-server's own docs claim bare `{"type": "json_object"}` (no `schema` key) produces "plain JSON output" like other OpenAI-inspired providers, but verified directly against the b9859 binary (`curl` straight to `/v1/chat/completions`, bypassing modelship) this isn't enforced — the model answers in free @@ -1896,7 +1412,7 @@ class TestChatLlamaServerGpu: Same GGUF and tool-calling shape as `TestChatLlamaServerResponseFormat` (CPU), but deployed with `num_gpus=1` so the actor gets a whole GPU and the loader passes `-ngl` for real offload instead of the forced `-ngl 0` - it uses when `num_gpus` is `0`. Mirrors `TestChatLlamaCppGpu`. + it uses when `num_gpus` is `0`. """ @pytest.fixture(autouse=True, scope="class") @@ -1950,8 +1466,8 @@ class TestChatTransformersLlama3Json: auto-detector picks ``llama3_json`` from the chat template's ``<|python_tag|>`` reference. This class is the first end-to-end exercise of the ``llama3_json`` parser on real model output (vLLM - has its own native parser; transformers/llama_cpp run through the - cross-loader registry). + has its own native parser; transformers runs through the cross-loader + registry). """ @pytest.fixture(autouse=True, scope="class") @@ -2174,8 +1690,7 @@ def test_reasoning_streaming_transformers(self): def test_reasoning_with_tools_transformers(self, client): """Reasoning + tool calling in one round-trip through transformers. - Mirrors ``TestChatLlamaCppReasoning.test_reasoning_with_tools_llama_cpp``: - the single-pass ``ChatOutputStreamer`` must populate BOTH + The single-pass ``ChatOutputStreamer`` must populate BOTH ``message.reasoning`` and ``message.tool_calls``. """ completion = client.chat.completions.create( @@ -2196,30 +1711,6 @@ def test_reasoning_with_tools_transformers(self, client): assert completion.choices[0].finish_reason == "tool_calls" -@pytest.mark.integration -@pytest.mark.llama_cpp -class TestChatLimited: - @pytest.fixture(autouse=True, scope="class") - def _deploy(self, model_deployer): - model_deployer.deploy("chat-limited") - - def test_tool_calling_explicit_opt_out(self, client): - """Verifies that ``tool_calls_enabled: false`` disables tools even when the model's chat template supports them.""" - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, - }, - } - ] - completion = client.chat.completions.create( - model="chat-limited", messages=[{"role": "user", "content": "Weather in London?"}], tools=tools - ) - assert not completion.choices[0].message.tool_calls - - @pytest.mark.integration def test_embeddings(client, model_deployer): model_deployer.deploy("embed-model") @@ -2503,7 +1994,7 @@ def test_tool_calling_streaming_transformers_function_gemma_loader(self, client) # character (transformers/generation/streamers.py: ``text.rfind(" ")+1``), # and FunctionGemma's args body contains no internal spaces, so the # whole body arrives in a single chunk and produces exactly one args - # delta. vLLM / llama_cpp loaders emit per-token and still satisfy >= 2. + # delta. vLLM / llama_server loaders emit per-token and still satisfy >= 2. assert collected["args_deltas"] >= 1, f"expected at least one args delta, got {collected['args_deltas']}" parsed_args = json.loads(call_0["arguments"]) assert parsed_args.get("city") @@ -2695,46 +2186,6 @@ def test_streaming_emits_reasoning_summary_deltas(self, client): assert reasoning_items, "expected a reasoning output item in the completed response" -@pytest.mark.integration -@pytest.mark.llama_cpp -class TestResponsesLlamaCpp: - """The adapter is loader-agnostic: /v1/responses works over the llama_cpp - chat pipeline with no loader-side changes, same as it does over vLLM.""" - - @pytest.fixture(autouse=True, scope="class") - def _deploy(self, model_deployer): - model_deployer.deploy("chat-llama-mship") - - def test_basic_response_through_llama_cpp(self, client): - resp = client.responses.create( - model="chat-llama-mship", - input="Say hello in one word.", - max_output_tokens=20, - ) - assert resp.status in {"completed", "incomplete"} - assert resp.output_text.strip() - - def test_streaming_response_through_llama_cpp(self, client): - # The streaming translator is loader-agnostic too: it consumes the same - # chat SSE chunk stream llama_cpp emits. - stream = client.responses.create( - model="chat-llama-mship", - input="Say hello in one word.", - max_output_tokens=20, - stream=True, - ) - text_deltas: list[str] = [] - completed = None - for event in stream: - if event.type == "response.output_text.delta": - text_deltas.append(event.delta) - elif event.type == "response.completed": - completed = event.response - assert "".join(text_deltas).strip() - assert completed is not None - assert completed.status in {"completed", "incomplete"} - - def _running_replicas(model_name: str) -> int: """Count RUNNING replicas of the deployment serving `model_name`, read from the Serve REST status API. The app name is `-`, so @@ -2779,7 +2230,7 @@ def _hammer(client: OpenAI, model: str, stop: threading.Event, errors: list) -> @pytest.mark.integration -@pytest.mark.llama_cpp +@pytest.mark.llama_server @pytest.mark.autoscaling class TestAutoscaling: """End-to-end check that a model's autoscaling_config actually drives Ray @@ -2840,7 +2291,7 @@ def _poll(predicate, deadline_s: float) -> bool: @pytest.mark.integration -@pytest.mark.llama_cpp +@pytest.mark.llama_server @pytest.mark.gateway_ha class TestGatewayReplicaConsistency: """With 2 gateway replicas (the session starts --gateway-replicas 2), a deployed @@ -2854,18 +2305,18 @@ def test_add_and_remove_propagate_to_all_replicas(self, client, model_deployer): client.models.list() # Deploy: the model becomes routable on every replica. - model_deployer.deploy("chat-limited") - assert _poll(lambda: _model_in_all_samples(client, "chat-limited"), deadline_s=60), ( + model_deployer.deploy("chat-llama-server-plain") + assert _poll(lambda: _model_in_all_samples(client, "chat-llama-server-plain"), deadline_s=60), ( "deployed model did not become routable on all gateway replicas" ) completion = client.chat.completions.create( - model="chat-limited", messages=[{"role": "user", "content": "hi"}], max_tokens=5 + model="chat-llama-server-plain", messages=[{"role": "user", "content": "hi"}], max_tokens=5 ) assert completion.choices[0].message.content is not None - # Reconcile to a different model — chat-limited is removed everywhere. - model_deployer.deploy("chat-llama-mship") - assert _poll(lambda: _model_in_no_samples(client, "chat-limited"), deadline_s=60), ( + # Reconcile to a different model — chat-llama-server-plain is removed everywhere. + model_deployer.deploy("chat-llama-server") + assert _poll(lambda: _model_in_no_samples(client, "chat-llama-server-plain"), deadline_s=60), ( "removed model still routable on some gateway replica" ) @@ -2876,5 +2327,5 @@ def test_add_and_remove_propagate_to_all_replicas(self, client, model_deployer): for _ in range(20): with pytest.raises(openai.NotFoundError): client.chat.completions.create( - model="chat-limited", messages=[{"role": "user", "content": "hi"}], max_tokens=5 + model="chat-llama-server-plain", messages=[{"role": "user", "content": "hi"}], max_tokens=5 ) diff --git a/tests/test_integration_profiles.py b/tests/test_integration_profiles.py index 2ad8b6b..5b506cb 100644 --- a/tests/test_integration_profiles.py +++ b/tests/test_integration_profiles.py @@ -146,7 +146,7 @@ def _run_deploy_expecting_failure(profile: str, num_cpus: int, num_gpus: int, tm @pytest.mark.integration @pytest.mark.profiles -@pytest.mark.llama_cpp +@pytest.mark.llama_server def test_chat_on_2_cores_deploys_smallest_generate(tmp_path): # 2 cores: chat (generate + embed) can only afford the 1.5B generate rung # alongside embed — the smallest box gets the smallest model. @@ -157,7 +157,7 @@ def test_chat_on_2_cores_deploys_smallest_generate(tmp_path): @pytest.mark.integration @pytest.mark.profiles -@pytest.mark.llama_cpp +@pytest.mark.llama_server def test_chat_on_8_cores_scales_generate_up(tmp_path): # 8 cores: the knapsack must pick a larger generate than the 2-core box did # (at least the 3B), proving cpu headroom scales the selection up. @@ -168,7 +168,7 @@ def test_chat_on_8_cores_scales_generate_up(tmp_path): @pytest.mark.integration @pytest.mark.profiles -@pytest.mark.llama_cpp +@pytest.mark.llama_server def test_assistant_deploys_full_capability_set(tmp_path): # assistant = generate + transcription + tts. Needs the whispercpp + kokoroonnx # plugin wheels available (MSHIP_PLUGIN_WHEEL_DIR / installed extras). @@ -211,4 +211,4 @@ def test_studio_on_one_gpu_uses_gpu_loaders(tmp_path): by_uc = {m["usecase"]: m for m in doc["models"]} assert by_uc["generate"]["loader"] == "vllm" assert by_uc["image"]["loader"] == "diffusers" - assert by_uc["embed"]["loader"] == "llama_cpp" + assert by_uc["embed"]["loader"] == "llama_server" diff --git a/tests/test_llama_cpp_chat_trace.py b/tests/test_llama_cpp_chat_trace.py deleted file mode 100644 index deb8d86..0000000 --- a/tests/test_llama_cpp_chat_trace.py +++ /dev/null @@ -1,105 +0,0 @@ -"""TRACE-level logging of raw model output on the llama.cpp chat path. - -The raw completion text (before tool-call/reasoning parsing) is what's needed -to debug whether ```` tags are present, so it must be logged at -TRACE. Tests mock Ray Serve via ``__new__`` to bypass the deployment wrapper. -""" - -import asyncio -import inspect - -import pytest - -from modelship.infer.llama_cpp.openai.serving_chat import OpenAIServingChat -from modelship.logging import TRACE -from modelship.openai.protocol import ChatCompletionRequest - -LOGGER_NAME = "modelship.infer.llama_cpp.chat" - - -class _FakeRenderer: - def render(self, messages, tools): - return "PROMPT" - - def count_tokens(self, text): - return len(text.split()) - - -class _FakeLlama: - def __init__(self, text): - self._text = text - - def create_completion(self, *, stream=False, **kwargs): - if stream: - # Emit the text in a few pieces so the buffering logic in - # _raw_text_chunks is exercised, not just a single chunk. - return iter([{"choices": [{"text": piece}]} for piece in (self._text[:3], self._text[3:])]) - return { - "choices": [{"text": self._text}], - "usage": {"prompt_tokens": 3, "completion_tokens": 5}, - } - - -def _serving(text: str) -> OpenAIServingChat: - # Bypass __init__ / the @serve.deployment wrapper; wire up only what - # _handle_with_parsers touches on the non-streaming path. - from llama_cpp import Llama - - chat = OpenAIServingChat.__new__(OpenAIServingChat) - chat.model_name = "x" - chat._lock = asyncio.Lock() - chat._renderer = _FakeRenderer() - chat._llama = _FakeLlama(text) - chat.reasoning_parser = None - chat._logged_reasoning_unconstrained = False - chat._completion_accepted_params = set(inspect.signature(Llama.create_completion).parameters) - return chat - - -def _request() -> ChatCompletionRequest: - return ChatCompletionRequest(model="x", messages=[{"role": "user", "content": "hi"}]) - - -class TestTraceResponseLogging: - @pytest.mark.asyncio - async def test_non_streaming_logs_raw_response_at_trace(self, caplog): - text = '{"name": "get_weather"}' - chat = _serving(text) - with caplog.at_level(TRACE, logger=LOGGER_NAME): - await chat._handle_with_parsers( - _request(), - "chat-abc", - messages=[{"role": "user", "content": "hi"}], - tools=None, - tool_parser_name=None, - ) - - records = [r for r in caplog.records if r.name == LOGGER_NAME and "chat response" in r.message] - assert records, "expected a TRACE 'chat response' record" - rendered = records[0].getMessage() - assert "chat-abc" in rendered - assert text in rendered - - @pytest.mark.asyncio - async def test_streaming_logs_buffered_response_at_trace(self, caplog): - text = '{"name": "get_weather"}' - chat = _serving(text) - with caplog.at_level(TRACE, logger=LOGGER_NAME): - gen = chat._locked_stream_with_parsers( - request_id="chat-xyz", - completion_kwargs={}, - tool_parser_name=None, - reasoning_parser_name=None, - prompt_tokens=3, - max_tokens=None, - include_usage=False, - ) - async for _ in gen: - pass - - records = [r for r in caplog.records if r.name == LOGGER_NAME and "(stream)" in r.message] - assert records, "expected a TRACE 'chat response ... (stream)' record" - rendered = records[0].getMessage() - assert "chat-xyz" in rendered - # The buffered text reassembled across the streamed chunks. - assert text in rendered diff --git a/tests/test_llama_cpp_gpu_offload.py b/tests/test_llama_cpp_gpu_offload.py deleted file mode 100644 index 152fa13..0000000 --- a/tests/test_llama_cpp_gpu_offload.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Tests for llama_cpp GPU offload gating in LlamaCppInfer.__init__. - -The guard runs before any `Llama(...)` construction in `start()`, so these -tests build the config only and never touch a real GGUF. -""" - -from __future__ import annotations - -from unittest.mock import patch - -from modelship.infer.infer_config import LlamaCppConfig, ModelLoader, ModelshipModelConfig, ModelUsecase -from modelship.infer.llama_cpp.llama_cpp_infer import LlamaCppInfer - - -def _make_config(*, num_gpus: float, n_gpu_layers: int) -> ModelshipModelConfig: - return ModelshipModelConfig( - name="test-model", - model="org/test-model", - usecase=ModelUsecase.generate, - loader=ModelLoader.llama_cpp, - num_gpus=num_gpus, - llama_cpp_config=LlamaCppConfig(n_gpu_layers=n_gpu_layers), - ) - - -class TestLlamaCppGpuOffloadGating: - def test_honors_n_gpu_layers_when_gpu_capable_and_assigned(self): - with patch("modelship.infer.llama_cpp.llama_cpp_infer.llama_supports_gpu_offload", return_value=True): - infer = LlamaCppInfer(_make_config(num_gpus=1, n_gpu_layers=-1)) - assert infer._n_gpu_layers == -1 - - def test_forces_cpu_when_wheel_lacks_gpu_support(self): - with patch("modelship.infer.llama_cpp.llama_cpp_infer.llama_supports_gpu_offload", return_value=False): - infer = LlamaCppInfer(_make_config(num_gpus=1, n_gpu_layers=-1)) - assert infer._n_gpu_layers == 0 - - def test_forces_cpu_when_no_gpu_assigned_even_if_wheel_is_capable(self): - with patch("modelship.infer.llama_cpp.llama_cpp_infer.llama_supports_gpu_offload", return_value=True): - infer = LlamaCppInfer(_make_config(num_gpus=0, n_gpu_layers=-1)) - assert infer._n_gpu_layers == 0 - - def test_no_warning_for_cpu_only_deploy_with_default_n_gpu_layers(self, caplog): - # num_gpus=0 is an explicit CPU-only request; n_gpu_layers=-1 (the - # library default) shouldn't trigger a warning just because it's non-zero. - with ( - patch("modelship.infer.llama_cpp.llama_cpp_infer.llama_supports_gpu_offload", return_value=True), - caplog.at_level("WARNING"), - ): - LlamaCppInfer(_make_config(num_gpus=0, n_gpu_layers=-1)) - assert not any("n_gpu_layers" in record.message for record in caplog.records) - - def test_warns_when_gpu_assigned_but_wheel_lacks_support(self, caplog): - with ( - patch("modelship.infer.llama_cpp.llama_cpp_infer.llama_supports_gpu_offload", return_value=False), - caplog.at_level("WARNING"), - ): - LlamaCppInfer(_make_config(num_gpus=1, n_gpu_layers=-1)) - assert any("n_gpu_layers" in record.message for record in caplog.records) diff --git a/tests/test_llama_cpp_logprobs.py b/tests/test_llama_cpp_logprobs.py deleted file mode 100644 index 5a63059..0000000 --- a/tests/test_llama_cpp_logprobs.py +++ /dev/null @@ -1,53 +0,0 @@ -"""The llama.cpp loader cannot yet thread token logprobs into the OpenAI -response, so it must reject requests that ask for them rather than silently -dropping the fields (which left clients waiting for logprobs that never came). -""" - -import pytest - -from modelship.infer.infer_config import RawRequestProxy -from modelship.infer.llama_cpp.openai.serving_chat import OpenAIServingChat -from modelship.openai.protocol import ChatCompletionRequest, ErrorResponse - - -def _serving() -> OpenAIServingChat: - # The logprobs guard runs at the very top of create_chat_completion, before - # anything reads _caps / renderer / llama, so a bare instance is enough. - return OpenAIServingChat.__new__(OpenAIServingChat) - - -def _request(**overrides) -> ChatCompletionRequest: - payload = {"model": "x", "messages": [{"role": "user", "content": "hi"}], **overrides} - return ChatCompletionRequest(**payload) - - -def _raw_request() -> RawRequestProxy: - return RawRequestProxy(None, {}) - - -class TestLogprobsRejected: - @pytest.mark.asyncio - async def test_logprobs_true_returns_400(self): - chat = _serving() - result = await chat.create_chat_completion(_request(logprobs=True), _raw_request()) - assert isinstance(result, ErrorResponse) - assert result._http_status == 400 - assert "logprobs" in result.error.message - - @pytest.mark.asyncio - async def test_top_logprobs_alone_returns_400(self): - # A client may set top_logprobs without logprobs=True; still unsupported. - chat = _serving() - result = await chat.create_chat_completion(_request(top_logprobs=5), _raw_request()) - assert isinstance(result, ErrorResponse) - assert result._http_status == 400 - - @pytest.mark.asyncio - async def test_logprobs_false_is_not_rejected_here(self): - # The default (logprobs=False, top_logprobs=0) must fall through the guard - # so normal requests still work; it then proceeds past this point and - # touches _caps, which the bare instance lacks → AttributeError, not an - # ErrorResponse. That confirms the guard let it through. - chat = _serving() - with pytest.raises(AttributeError): - await chat.create_chat_completion(_request(), _raw_request()) diff --git a/tests/test_llama_cpp_renderer.py b/tests/test_llama_cpp_renderer.py deleted file mode 100644 index 6eb0d7f..0000000 --- a/tests/test_llama_cpp_renderer.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Prompt rendering on the llama.cpp loader (LlamaCppToolCallRenderer). - -Regression: reasoning chat templates (Qwen3) test ``'' in message.content``, -which raised "argument of type 'NoneType' is not iterable" on replayed assistant -tool-call messages whose content is legitimately ``None``. render() must coerce -None content to "" so the membership test is a safe no-op. -""" - -from modelship.infer.llama_cpp.utils import LlamaCppToolCallRenderer - -# Mirrors Qwen3's reasoning-strip: a membership test on content that explodes on None. -_REASONING_TEMPLATE = ( - "{%- for message in messages %}" - "{%- set content = message['content'] %}" - "{%- if '' in content %}{%- set content = content.split('')[-1] %}{%- endif %}" - "{{ message['role'] }}: {{ content }}\n" - "{%- endfor %}" -) - - -def _renderer() -> LlamaCppToolCallRenderer: - return LlamaCppToolCallRenderer( - chat_template=_REASONING_TEMPLATE, - bos_token="", - eos_token="", - _llama=None, # type: ignore[arg-type] # render() never touches _llama - ) - - -def test_render_tolerates_none_content_on_tool_call_message(): - messages = [ - {"role": "user", "content": "turn off the lamps"}, - { - "role": "assistant", - "content": None, # assistant tool-call turn carries null content - "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "HassTurnOff", "arguments": "{}"}}], - }, - {"role": "tool", "tool_call_id": "c1", "content": "done"}, - ] - out = _renderer().render(messages, tools=None) # pre-fix: raised TypeError -> 400 - assert "assistant:" in out - assert "turn off the lamps" in out - - -def test_render_does_not_mutate_caller_messages(): - messages = [{"role": "assistant", "content": None}] - _renderer().render(messages, tools=None) - assert messages[0]["content"] is None - - -# Template branching on a chat_template_kwargs variable, mirroring Qwen3's `enable_thinking`. -_THINKING_TEMPLATE = ( - "{%- if enable_thinking is defined and not enable_thinking %}NOTHINK\n{%- endif %}" - "{%- for message in messages %}{{ message['role'] }}: {{ message['content'] }}\n{%- endfor %}" -) - - -def _thinking_renderer(template_kwargs: dict) -> LlamaCppToolCallRenderer: - return LlamaCppToolCallRenderer( - chat_template=_THINKING_TEMPLATE, - bos_token="", - eos_token="", - _llama=None, # type: ignore[arg-type] # render() never touches _llama - template_kwargs=template_kwargs, - ) - - -def test_render_forwards_template_kwargs(): - messages = [{"role": "user", "content": "hi"}] - out = _thinking_renderer({"enable_thinking": False}).render(messages, tools=None) - assert "NOTHINK" in out - - -def test_render_omits_kwargs_takes_template_default(): - messages = [{"role": "user", "content": "hi"}] - out = _thinking_renderer({}).render(messages, tools=None) - assert "NOTHINK" not in out diff --git a/tests/test_llama_cpp_structured.py b/tests/test_llama_cpp_structured.py deleted file mode 100644 index 6bf5daf..0000000 --- a/tests/test_llama_cpp_structured.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Tests for the llama_cpp ``response_format`` → ``LlamaGrammar`` converter.""" - -from llama_cpp import LlamaGrammar - -from modelship.infer.llama_cpp.structured import build_llama_grammar - - -class TestBuildLlamaGrammar: - def test_none_returns_none(self): - assert build_llama_grammar(None) is None - - def test_empty_dict_returns_none(self): - assert build_llama_grammar({}) is None - - def test_text_type_returns_none(self): - assert build_llama_grammar({"type": "text"}) is None - - def test_missing_type_returns_none(self): - assert build_llama_grammar({"foo": "bar"}) is None - - def test_json_object_returns_grammar(self): - g = build_llama_grammar({"type": "json_object"}) - assert isinstance(g, LlamaGrammar) - - def test_json_object_returns_fresh_instance(self): - # LlamaGrammar wraps a stateful C-side parser pointer; sharing an - # instance across concurrent requests would corrupt sampling state. - # Each call must compile a fresh grammar. - g1 = build_llama_grammar({"type": "json_object"}) - g2 = build_llama_grammar({"type": "json_object"}) - assert g1 is not g2 - - def test_json_schema_compiles_schema(self): - schema = { - "type": "object", - "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, - "required": ["name", "age"], - } - g = build_llama_grammar( - {"type": "json_schema", "json_schema": {"name": "p", "schema": schema, "strict": True}}, - ) - assert isinstance(g, LlamaGrammar) - - def test_json_schema_missing_schema_returns_none_with_warning(self, caplog): - import logging - - target = logging.getLogger("modelship.infer.llama_cpp.structured") - target.addHandler(caplog.handler) - try: - caplog.set_level(logging.WARNING) - g = build_llama_grammar({"type": "json_schema", "json_schema": {"name": "p"}}) - finally: - target.removeHandler(caplog.handler) - assert g is None - assert any("missing a 'schema'" in r.message for r in caplog.records) - - def test_unknown_type_returns_none_with_warning(self, caplog): - import logging - - target = logging.getLogger("modelship.infer.llama_cpp.structured") - target.addHandler(caplog.handler) - try: - caplog.set_level(logging.WARNING) - g = build_llama_grammar({"type": "xml"}) - finally: - target.removeHandler(caplog.handler) - assert g is None - assert any("unsupported response_format" in r.message for r in caplog.records) - - def test_invalid_schema_returns_none_with_warning(self, caplog): - import logging - - target = logging.getLogger("modelship.infer.llama_cpp.structured") - target.addHandler(caplog.handler) - try: - caplog.set_level(logging.WARNING) - # Schema with an unresolvable $ref triggers LlamaGrammar.from_json_schema to raise. - bad_schema = {"$ref": "#/definitions/nope"} - g = build_llama_grammar( - {"type": "json_schema", "json_schema": {"name": "p", "schema": bad_schema}}, - ) - finally: - target.removeHandler(caplog.handler) - # Either compiles to something or fails gracefully — both are acceptable; - # this test just guarantees we never raise into the caller. - assert g is None or isinstance(g, LlamaGrammar) - - -class TestKwargBuilders: - """``_build_kwargs`` and ``_build_completion_kwargs`` must replace - ``response_format`` with a compiled ``grammar`` so neither field reaches - llama-cpp-python with a shape it would silently ignore. - """ - - def _serving(self): - from llama_cpp import Llama - - from modelship.infer.llama_cpp.openai.serving_chat import OpenAIServingChat - - # Bypass __init__ — we only need the kwarg builders, which only read - # _accepted_params / _completion_accepted_params. - chat = OpenAIServingChat.__new__(OpenAIServingChat) - import inspect - - chat._accepted_params = set(inspect.signature(Llama.create_chat_completion).parameters) - chat._completion_accepted_params = set(inspect.signature(Llama.create_completion).parameters) - return chat - - def _request(self, response_format=None): - from modelship.openai.protocol import ChatCompletionRequest - - payload = {"model": "x", "messages": [{"role": "user", "content": "hi"}]} - if response_format is not None: - payload["response_format"] = response_format - return ChatCompletionRequest(**payload) - - def test_build_kwargs_injects_grammar(self): - chat = self._serving() - req = self._request( - response_format={ - "type": "json_schema", - "json_schema": { - "name": "p", - "schema": {"type": "object", "properties": {"x": {"type": "string"}}}, - "strict": True, - }, - } - ) - kwargs = chat._build_kwargs(req, messages=[{"role": "user", "content": "hi"}]) - assert "response_format" not in kwargs - assert isinstance(kwargs.get("grammar"), LlamaGrammar) - - def test_build_kwargs_omits_grammar_when_no_response_format(self): - chat = self._serving() - req = self._request() - kwargs = chat._build_kwargs(req, messages=[{"role": "user", "content": "hi"}]) - assert "grammar" not in kwargs - assert "response_format" not in kwargs - - def test_build_completion_kwargs_injects_grammar(self): - chat = self._serving() - req = self._request(response_format={"type": "json_object"}) - kwargs = chat._build_completion_kwargs(req, prompt="hi") - assert "response_format" not in kwargs - assert isinstance(kwargs.get("grammar"), LlamaGrammar) - - def test_build_completion_kwargs_omits_grammar_when_no_response_format(self): - chat = self._serving() - req = self._request() - kwargs = chat._build_completion_kwargs(req, prompt="hi") - assert "grammar" not in kwargs - - def test_max_completion_tokens_mapped_and_dropped(self): - # ``max_completion_tokens`` is the modern OpenAI field. llama-cpp-python - # only accepts ``max_tokens``. We map the value over and must drop the - # original key so it doesn't end up in the "unsupported params" warning. - chat = self._serving() - from modelship.openai.protocol import ChatCompletionRequest - - req = ChatCompletionRequest( - model="x", - messages=[{"role": "user", "content": "hi"}], - max_completion_tokens=42, - ) - kwargs = chat._build_kwargs(req, messages=[{"role": "user", "content": "hi"}]) - assert kwargs.get("max_tokens") == 42 - assert "max_completion_tokens" not in kwargs - - completion_kwargs = chat._build_completion_kwargs(req, prompt="hi") - assert completion_kwargs.get("max_tokens") == 42 - assert "max_completion_tokens" not in completion_kwargs diff --git a/tests/test_llama_cpp_tool_choice_grammar.py b/tests/test_llama_cpp_tool_choice_grammar.py deleted file mode 100644 index 605d5ed..0000000 --- a/tests/test_llama_cpp_tool_choice_grammar.py +++ /dev/null @@ -1,107 +0,0 @@ -"""tool_choice drives the llama.cpp tool-call grammar. - -Format-constraining is automatic when a request carries tools and the model has a -usable parser. The grammar *root* is chosen per request from ``tool_choice``: - -- ``required`` / named-function force a tool-only root, which has no free-text branch - and so cannot emit ```` — it is applied even on a reasoning deployment. -- ``auto`` (default) keeps a free-text branch whose ``content ::= [^<]+`` rule - excludes ``<`` (which opens ````), so it yields to a reasoning deployment. -""" - -import asyncio -import inspect - -import pytest - -from modelship.infer.llama_cpp.openai.serving_chat import OpenAIServingChat -from modelship.openai.protocol import ChatCompletionRequest - -TOOLS = [ - { - "type": "function", - "function": { - "name": "HassTurnOn", - "parameters": {"type": "object", "properties": {"area": {"type": "string"}}}, - }, - } -] - - -class _FakeRenderer: - def render(self, messages, tools): - return "PROMPT" - - def count_tokens(self, text): - return len(text.split()) - - -class _CapturingLlama: - """Records the kwargs of the last non-streaming create_completion call.""" - - def __init__(self): - self.last_kwargs: dict | None = None - - def create_completion(self, *, stream=False, **kwargs): - self.last_kwargs = kwargs - return {"choices": [{"text": "hi"}], "usage": {"prompt_tokens": 3, "completion_tokens": 1}} - - -def _serving(*, reasoning_parser: str | None) -> tuple[OpenAIServingChat, _CapturingLlama]: - from llama_cpp import Llama - - chat = OpenAIServingChat.__new__(OpenAIServingChat) - chat.model_name = "x" - chat._lock = asyncio.Lock() - chat._renderer = _FakeRenderer() - llama = _CapturingLlama() - chat._llama = llama - chat.tool_call_parser = "hermes" - chat.reasoning_parser = reasoning_parser - chat._logged_reasoning_unconstrained = False - chat._completion_accepted_params = set(inspect.signature(Llama.create_completion).parameters) - return chat, llama - - -def _request(tool_choice=None) -> ChatCompletionRequest: - return ChatCompletionRequest( - model="x", messages=[{"role": "user", "content": "hi"}], tools=TOOLS, tool_choice=tool_choice - ) - - -async def _run(reasoning_parser: str | None, tool_choice=None) -> dict: - chat, llama = _serving(reasoning_parser=reasoning_parser) - await chat._handle_with_parsers( - _request(tool_choice), - "chat-1", - messages=[{"role": "user", "content": "hi"}], - tools=TOOLS, - tool_parser_name="hermes", - ) - assert llama.last_kwargs is not None - return llama.last_kwargs - - -@pytest.mark.asyncio -async def test_auto_constrains_without_reasoning_parser(): - kwargs = await _run(reasoning_parser=None, tool_choice="auto") - assert kwargs.get("grammar") is not None, "auto should auto-constrain a non-reasoning deployment" - - -@pytest.mark.asyncio -async def test_auto_yields_to_reasoning_parser(): - kwargs = await _run(reasoning_parser="deepseek_r1", tool_choice="auto") - assert "grammar" not in kwargs, "auto must not constrain a reasoning deployment" - - -@pytest.mark.asyncio -async def test_required_forces_grammar_even_with_reasoning_parser(): - kwargs = await _run(reasoning_parser="deepseek_r1", tool_choice="required") - assert kwargs.get("grammar") is not None, "required must force the grammar regardless of reasoning" - - -@pytest.mark.asyncio -async def test_named_function_forces_grammar_even_with_reasoning_parser(): - choice = {"type": "function", "function": {"name": "HassTurnOn"}} - kwargs = await _run(reasoning_parser="deepseek_r1", tool_choice=choice) - assert kwargs.get("grammar") is not None, "named-function must force the grammar regardless of reasoning" diff --git a/tests/test_llama_cpp_tool_grammar.py b/tests/test_llama_cpp_tool_grammar.py deleted file mode 100644 index bfab834..0000000 --- a/tests/test_llama_cpp_tool_grammar.py +++ /dev/null @@ -1,455 +0,0 @@ -"""Tests for the llama_cpp tool-call GBNF grammar builder.""" - -import json - -from llama_cpp import LlamaGrammar - -from modelship.infer.llama_cpp.tool_grammar import ( - _MAX_TOOL_CALLS, - build_tool_call_gbnf, - build_tool_call_grammar, -) -from modelship.openai.parsers.tool_calling import HermesToolCallParser, get_parser - -GEMMA_TOOLS = [ - { - "type": "function", - "function": { - "name": "HassTurnOn", - "parameters": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "area": {"enum": ["Living Room", "Small Bedroom"]}, - }, - "required": ["name"], - }, - }, - }, - { - # All-optional / no-property intent: must collapse to FUNC{}. - "type": "function", - "function": {"name": "HassGetState", "parameters": {"type": "object", "properties": {}}}, - }, - { - "type": "function", - "function": { - "name": "SetTimer", - "parameters": { - "type": "object", - "properties": { - "minutes": {"type": "integer"}, - "on": {"type": "boolean"}, - "tags": {"type": "array", "items": {"type": "string"}}, - }, - }, - }, - }, -] - -TOOLS = [ - { - "type": "function", - "function": { - "name": "HassTurnOn", - "parameters": { - "type": "object", - "properties": {"area": {"type": "string"}}, - "required": ["area"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "HassTurnOff", - "parameters": {"type": "object", "properties": {"area": {"type": "string"}}}, - }, - }, -] - - -class TestBuildToolCallGrammar: - def test_hermes_returns_grammar(self): - g = build_tool_call_grammar(HermesToolCallParser(), TOOLS) - assert isinstance(g, LlamaGrammar) - - def test_hermes_grammar_text_has_envelope_and_consts(self): - parser = HermesToolCallParser() - text = build_tool_call_gbnf(parser, TOOLS) - assert text is not None - # Top-level rules and the renamed inner entry rule. The root allows - # optional conversational text around the tool calls. - assert "root ::= ( content )? tool-calls ( content )? | content" in text - assert "tc-ws ::= [ \\t\\n\\r]*" in text - assert "tc-json ::=" in text - assert "root ::= alternative" not in text # inner root was renamed - # Envelope markers and the tool names appear as GBNF string literals - # (consts are escaped inside the GBNF, so match the bare name). - assert parser.start_marker in text - assert parser.end_marker in text - assert "HassTurnOn" in text - assert "HassTurnOff" in text - - def test_empty_tools_returns_none(self): - assert build_tool_call_grammar(HermesToolCallParser(), []) is None - assert build_tool_call_gbnf(HermesToolCallParser(), []) is None - - def test_qwen3_coder_returns_none(self): - # Shares Hermes markers but an XML body — must not be treated as JSON-family. - assert build_tool_call_grammar(get_parser("qwen3_coder"), TOOLS) is None - - def test_fresh_instance_per_call(self): - # LlamaGrammar wraps stateful C-side state; never share across requests. - g1 = build_tool_call_grammar(HermesToolCallParser(), TOOLS) - g2 = build_tool_call_grammar(HermesToolCallParser(), TOOLS) - assert g1 is not g2 - - def test_tool_without_parameters_still_builds(self): - tools = [{"type": "function", "function": {"name": "Now"}}] - g = build_tool_call_grammar(HermesToolCallParser(), tools) - assert isinstance(g, LlamaGrammar) - - def test_malformed_tools_are_filtered(self): - # Entries without a usable function name are dropped, but valid ones - # still build (a None const would be unsatisfiable). - tools = [ - {"type": "function", "function": {}}, # no name - {"type": "function"}, # no function body - {"type": "function", "function": {"name": "HassTurnOn"}}, - ] - text = build_tool_call_gbnf(HermesToolCallParser(), tools) - assert text is not None - assert "HassTurnOn" in text - g = build_tool_call_grammar(HermesToolCallParser(), tools) - assert isinstance(g, LlamaGrammar) - - def test_all_malformed_tools_returns_none(self): - tools = [{"type": "function", "function": {}}, {"type": "function"}] - assert build_tool_call_gbnf(HermesToolCallParser(), tools) is None - assert build_tool_call_grammar(HermesToolCallParser(), tools) is None - - def test_content_rule_excludes_start_marker_first_char(self): - # The content exclusion is derived from the marker's first char so a - # free-text answer yields to the tool call once the marker begins. - text = build_tool_call_gbnf(HermesToolCallParser(), TOOLS) - assert text is not None - assert "content ::= [^<]+" in text # hermes start marker is "" - - -class TestGrammarShapedOutputRoundTrips: - """A sample matching the grammar's envelope must parse back to the tools.""" - - def test_hermes_roundtrip(self): - parser = HermesToolCallParser() - sample = ( - f'{parser.start_marker}\n{{"name": "HassTurnOn", "arguments": {{"area": "kitchen"}}}}\n{parser.end_marker}' - ) - out = parser.parse(sample) - assert out.content is None - assert len(out.tool_calls) == 1 - call = out.tool_calls[0] - assert call.function.name == "HassTurnOn" - assert json.loads(call.function.arguments) == {"area": "kitchen"} - - def test_hermes_roundtrip_two_calls(self): - parser = HermesToolCallParser() - sample = ( - f"{parser.start_marker}\n" - '{"name": "HassTurnOn", "arguments": {"area": "kitchen"}}\n' - f"{parser.end_marker}\n" - f"{parser.start_marker}\n" - '{"name": "HassTurnOff", "arguments": {"area": "bedroom"}}\n' - f"{parser.end_marker}" - ) - out = parser.parse(sample) - assert [c.function.name for c in out.tool_calls] == ["HassTurnOn", "HassTurnOff"] - - -class TestGemmaToolCallGrammar: - """The hand-built emitter for the FunctionGemma / Gemma 4 ``call:FUNC{...}`` DSL.""" - - def test_function_gemma_returns_grammar(self): - g = build_tool_call_grammar(get_parser("function_gemma"), GEMMA_TOOLS) - assert isinstance(g, LlamaGrammar) - - def test_gemma4_returns_grammar(self): - g = build_tool_call_grammar(get_parser("gemma4"), GEMMA_TOOLS) - assert isinstance(g, LlamaGrammar) - - def test_envelope_and_markers(self): - # Default: free-text answers stay reachable around the tool calls. - parser = get_parser("function_gemma") - text = build_tool_call_gbnf(parser, GEMMA_TOOLS) - assert text is not None - assert f'tool-call ::= "{parser.start_marker}" "call:" call-choice "{parser.end_marker}"' in text - assert "root ::= ws tool-calls ws | content" in text - assert "content ::= [^<]+" in text # both gemma markers/delims lead with '<' - - def test_require_tool_call_drops_free_text_escape(self): - # Forced-call root: no `content` branch, so the model must emit a call. - parser = get_parser("function_gemma") - text = build_tool_call_gbnf(parser, GEMMA_TOOLS, require_tool_call=True) - assert text is not None - assert "root ::= ws tool-calls ws\n" in text - assert "content ::=" not in text - assert build_tool_call_grammar(parser, GEMMA_TOOLS, require_tool_call=True) is not None - - def test_require_tool_call_drops_free_text_escape_hermes(self): - # The flag is parser-agnostic — the JSON-family envelope honors it too. - parser = HermesToolCallParser() - text = build_tool_call_gbnf(parser, TOOLS, require_tool_call=True) - assert text is not None - assert "root ::= tool-calls\n" in text - assert "content ::=" not in text - assert build_tool_call_grammar(parser, TOOLS, require_tool_call=True) is not None - - def test_call_cap_is_structural(self): - # The cap is exactly _MAX_TOOL_CALLS calls, concatenated with no separator. - text = build_tool_call_gbnf(get_parser("function_gemma"), GEMMA_TOOLS) - assert text is not None - expected = "tool-calls ::= " + "tool-call" + " ( tool-call )?" * (_MAX_TOOL_CALLS - 1) - assert expected in text - - def test_every_tool_name_is_a_literal(self): - text = build_tool_call_gbnf(get_parser("function_gemma"), GEMMA_TOOLS) - assert text is not None - for name in ("HassTurnOn", "HassGetState", "SetTimer"): - assert f'"{name}"' in text - # All three tools are selectable. - assert "call-choice ::= tool-0 | tool-1 | tool-2" in text - - def test_enum_values_are_escaped_literals(self): - # FunctionGemma wraps string values in the delimiter. - text = build_tool_call_gbnf(get_parser("function_gemma"), GEMMA_TOOLS) - assert text is not None - assert '"" "Living Room" ""' in text - assert '"" "Small Bedroom" ""' in text - - def test_gemma4_uses_its_own_delimiter(self): - # Gemma 4 uses <|"|> rather than ; it appears as an escaped literal. - text = build_tool_call_gbnf(get_parser("gemma4"), GEMMA_TOOLS) - assert text is not None - assert '"<|\\"|>" "Living Room" "<|\\"|>"' in text - assert "" not in text - - def test_empty_property_tool_collapses_to_empty_braces(self): - text = build_tool_call_gbnf(get_parser("function_gemma"), GEMMA_TOOLS) - assert text is not None - assert 'tool-1 ::= "HassGetState" "{" "}"' in text - - def test_number_bool_array_value_rules(self): - text = build_tool_call_gbnf(get_parser("function_gemma"), GEMMA_TOOLS) - assert text is not None - assert '"minutes:" ( "-"? [0-9]+ )' in text - assert '"on:" ( "true" | "false" )' in text - assert '"[" (' in text # array rule for `tags` - - def test_empty_type_list_falls_back_to_generic(self): - # `type: []` must not emit an empty group `( )`, which is invalid GBNF. - tools = [ - { - "type": "function", - "function": {"name": "F", "parameters": {"type": "object", "properties": {"x": {"type": []}}}}, - } - ] - parser = get_parser("function_gemma") - text = build_tool_call_gbnf(parser, tools) - assert text is not None - assert "( )" not in text - assert '"x:" gv' in text - assert build_tool_call_grammar(parser, tools) is not None - - def test_generic_rules_omitted_when_unneeded(self): - # The fully-typed GEMMA_TOOLS never needs the generic fallback, so the - # gv-* rule block must not be emitted. - text = build_tool_call_gbnf(get_parser("function_gemma"), GEMMA_TOOLS) - assert text is not None - assert "gv ::=" not in text - - def test_free_form_object_uses_generic_object_rule(self): - # A property typed `object` with no `properties`, a schemaless property, - # and an array with no `items` must map to the generic value rules — not - # a delimited string, which would reject a legitimate {...} / [...] and - # deadlock constrained sampling. - tools = [ - { - "type": "function", - "function": { - "name": "SetState", - "parameters": { - "type": "object", - "properties": { - "attrs": {"type": "object"}, - "anything": {}, - "list": {"type": "array"}, - }, - }, - }, - } - ] - parser = get_parser("function_gemma") - text = build_tool_call_gbnf(parser, tools) - assert text is not None - assert '"attrs:" gv-obj' in text - assert '"anything:" gv' in text - assert "gv ::= gv-str | gv-num | gv-bool | gv-null | gv-arr | gv-obj" in text - # And it compiles + round-trips a nested object and a mixed array. - assert build_tool_call_grammar(parser, tools) is not None - d = parser.string_delim - sample = ( - parser.start_marker - + "call:SetState{attrs:{k:" - + d - + "v" - + d - + ",n:5},list:[" - + d - + "a" - + d - + ",true]}" - + parser.end_marker - ) - out = parser.parse(sample) - assert json.loads(out.tool_calls[0].function.arguments) == {"attrs": {"k": "v", "n": 5}, "list": ["a", True]} - - def test_all_malformed_tools_returns_none(self): - tools = [{"type": "function", "function": {}}, {"type": "function"}] - assert build_tool_call_gbnf(get_parser("function_gemma"), tools) is None - assert build_tool_call_grammar(get_parser("function_gemma"), tools) is None - - def test_empty_tools_returns_none(self): - assert build_tool_call_gbnf(get_parser("function_gemma"), []) is None - - def test_required_arg_is_mandatory(self): - # 1. Required arg is mandatory: with a tool `{name:string, area:enum}`, - # `required:["name"]`, assert the emitted `tool-0-args` rule equals the expected - # string (no outer `?`, p0 in every branch). Assert `build_tool_call_grammar` compiles. - tools = [ - { - "type": "function", - "function": { - "name": "HassTurnOn", - "parameters": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "area": {"enum": ["Living Room", "Small Bedroom"]}, - }, - "required": ["name"], - }, - }, - } - ] - parser = get_parser("function_gemma") - text = build_tool_call_gbnf(parser, tools) - assert text is not None - assert 'tool-0-args ::= ( tool-0-p0 ( "," tool-0-p1 )? | tool-0-p1 )' not in text - assert 'tool-0-args ::= ( tool-0-p0 ( "," tool-0-p1 )? )' in text - assert 'tool-0-args ::= ( tool-0-p0 ( "," tool-0-p1 )? )?' not in text - assert build_tool_call_grammar(parser, tools) is not None - - def test_no_required_regression(self): - # 2. No-required regression: a tool with the same props but no `required` still - # emits `tool-0-args ::= ( ... )?` (outer optional present) — byte-identical to pre-change output. - tools = [ - { - "type": "function", - "function": { - "name": "HassTurnOn", - "parameters": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "area": {"enum": ["Living Room", "Small Bedroom"]}, - }, - }, - }, - } - ] - parser = get_parser("function_gemma") - text = build_tool_call_gbnf(parser, tools) - assert text is not None - assert 'tool-0-args ::= ( tool-0-p0 ( "," tool-0-p1 )? | tool-0-p1 )?' in text - - def test_multiple_required_order_preserved(self): - # 3. Multiple required, order preserved: `[opt a, req b, opt c, req d]` produces the - # branch set from the worked examples (assert key substrings: `"," tool-0-p1` mandatory, - # `( "," tool-0-p2 )?` optional, `"," tool-0-p3` mandatory). - tools = [ - { - "type": "function", - "function": { - "name": "MultiReq", - "parameters": { - "type": "object", - "properties": { - "a": {"type": "string"}, - "b": {"type": "string"}, - "c": {"type": "string"}, - "d": {"type": "string"}, - }, - "required": ["b", "d"], - }, - }, - } - ] - parser = get_parser("function_gemma") - text = build_tool_call_gbnf(parser, tools) - assert text is not None - expected_rule = ( - 'tool-0-args ::= ( tool-0-p0 "," tool-0-p1 ( "," tool-0-p2 )? "," tool-0-p3 | ' - 'tool-0-p1 ( "," tool-0-p2 )? "," tool-0-p3 )' - ) - assert expected_rule in text - - -class TestGemmaGrammarParserAgreement: - """Canonical strings the grammar can emit must parse back to the right call. - - Guards the emitter and the parser against drifting apart — the grammar - constrains generation, the parser consumes it, and nothing checks they - agree except this. - """ - - def test_string_arg_roundtrips(self): - parser = get_parser("function_gemma") - sample = f"{parser.start_marker}call:HassTurnOn{{name:{parser.string_delim}small_bedroom_light{parser.string_delim}}}{parser.end_marker}" - out = parser.parse(sample) - assert len(out.tool_calls) == 1 - call = out.tool_calls[0] - assert call.function.name == "HassTurnOn" - assert json.loads(call.function.arguments) == {"name": "small_bedroom_light"} - - def test_enum_arg_roundtrips(self): - parser = get_parser("function_gemma") - d = parser.string_delim - sample = f"{parser.start_marker}call:HassTurnOn{{name:{d}x{d},area:{d}Living Room{d}}}{parser.end_marker}" - out = parser.parse(sample) - assert json.loads(out.tool_calls[0].function.arguments) == {"name": "x", "area": "Living Room"} - - def test_empty_args_roundtrips(self): - parser = get_parser("function_gemma") - sample = f"{parser.start_marker}call:HassGetState{{}}{parser.end_marker}" - out = parser.parse(sample) - assert out.tool_calls[0].function.name == "HassGetState" - assert json.loads(out.tool_calls[0].function.arguments) == {} - - def test_number_bool_array_roundtrips(self): - parser = get_parser("function_gemma") - d = parser.string_delim - sample = f"{parser.start_marker}call:SetTimer{{minutes:42,on:true,tags:[{d}a{d},{d}b{d}]}}{parser.end_marker}" - out = parser.parse(sample) - assert json.loads(out.tool_calls[0].function.arguments) == {"minutes": 42, "on": True, "tags": ["a", "b"]} - - def test_required_arg_parser_agreement(self): - # 4. Parser agreement: the minimal grammar-emittable call for `HassTurnOn` is - # `...call:HassTurnOn{name:x}...` and round-trips via - # `parser.parse(...)` to `{"name": "x"}` (mirrors `test_enum_arg_roundtrips`). - parser = get_parser("function_gemma") - d = parser.string_delim - sample = f"{parser.start_marker}call:HassTurnOn{{name:{d}x{d}}}{parser.end_marker}" - out = parser.parse(sample) - assert len(out.tool_calls) == 1 - assert out.tool_calls[0].function.name == "HassTurnOn" - assert json.loads(out.tool_calls[0].function.arguments) == {"name": "x"} diff --git a/tests/test_logging.py b/tests/test_logging.py index cc3565a..23b8fe4 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -136,7 +136,7 @@ class TestVllmConfigureLoggingOptOut: get_component_logger_file_path() later dereferences handler.target.baseFilename and raises AttributeError, killing the replica's is_allocated() health check. - This only bites the vLLM loader (llama_cpp etc. never import vllm) and only on + This only bites the vLLM loader (other loaders never import vllm) and only on the head-restart recovery path: on a cold deploy is_allocated() runs before the replica __init__ imports vLLM; on recovery the replica re-imports vLLM first, so the nulled target is hit. We set VLLM_CONFIGURE_LOGGING=0 in propagate_lib_log_env diff --git a/tests/test_mistral_specials_smoketest.py b/tests/test_mistral_specials_smoketest.py index 416be53..bb26f55 100644 --- a/tests/test_mistral_specials_smoketest.py +++ b/tests/test_mistral_specials_smoketest.py @@ -150,66 +150,3 @@ def test_real_tokenizer_strips_marker_with_skip_special_tokens_true(self, mistra assert "[TOOL_CALLS]" not in decoded, ( f"expected `[TOOL_CALLS]` stripped on the real Mistral tokenizer; got decoded={decoded!r}." ) - - -class TestLlamaCppHasSameBug: - """Confirm the same hypothesis on the llama_cpp loader path. - - ``Llama.detokenize`` defaults to ``special=False`` - (``inspect.signature`` proves it), and ``Llama.create_completion`` - calls ``self.detokenize(...)`` without overriding that. So any - GGUF Mistral model running on the llama_cpp loader will also have - ``[TOOL_CALLS]`` stripped before our parser sees it. - - This test is a *static* check — we don't need to load a real GGUF - Mistral to confirm the failure mode. The default value of the - detokenize parameter is the failure surface. - - Why static and not behavioral: a behavioral test would require - downloading a Mistral GGUF (multi-GB) and running inference. The - purpose of the smoke test is to confirm the hypothesis cheaply. - The transformers fix in this PR addresses the transformers loader; - the llama_cpp fix is deferred to a follow-up because it requires - bypassing ``create_completion``'s text-only chunk shape (no token - IDs are exposed in the chunk dict, so we can't re-detokenize with - ``special=True`` without restructuring the call site). - """ - - def test_llama_detokenize_defaults_to_stripping_specials(self): - import inspect - - from llama_cpp import Llama - - sig = inspect.signature(Llama.detokenize) - special_param = sig.parameters.get("special") - assert special_param is not None, "Llama.detokenize unexpectedly lacks a `special` parameter" - assert special_param.default is False, ( - f"hypothesis disproved: Llama.detokenize now defaults to special={special_param.default!r}; " - "llama_cpp may already preserve special tokens in its streaming text." - ) - - def test_llama_create_completion_does_not_override_special(self): - """``create_completion`` does not pass ``special=True`` when - detokenizing. Combined with the default-False above, this means - ``[TOOL_CALLS]`` is dropped before reaching ``chunk["choices"][0]["text"]``. - """ - import inspect - - from llama_cpp import Llama - - # Find the streaming generator. Implementation detail name has - # changed across llama-cpp-python versions, so try both. - for attr in ("_create_completion", "create_completion"): - fn = getattr(Llama, attr, None) - if fn is None: - continue - src = inspect.getsource(fn) - # If `detokenize(` is called anywhere with `special=True`, the - # bug doesn't apply — but we expect every call site to omit it. - assert "detokenize(" in src, f"unexpected: {attr} doesn't reference detokenize" - assert "special=True" not in src, ( - f"hypothesis disproved on {attr}: at least one detokenize call now passes special=True; " - "llama_cpp may emit specials in streamed text. Re-evaluate the loader fix scope." - ) - return - pytest.fail("could not locate the llama-cpp-python streaming completion function") diff --git a/tests/test_model_resolver.py b/tests/test_model_resolver.py index ff31f98..ae29c4d 100644 --- a/tests/test_model_resolver.py +++ b/tests/test_model_resolver.py @@ -209,7 +209,7 @@ def test_multi_variant_gguf_without_selector_raises(self): def test_single_gguf_without_selector_returns_file_path(self): # Single-GGUF repo: resolver must return the file path (not a snapshot - # dir), because llama_cpp requires a file path. + # dir), because llama_server requires a file path. files = ["model.gguf", "config.json"] with ( patch("modelship.infer.model_resolver.list_repo_files", return_value=files), diff --git a/tests/test_mship_deploy.py b/tests/test_mship_deploy.py index 2fa273a..3a87617 100644 --- a/tests/test_mship_deploy.py +++ b/tests/test_mship_deploy.py @@ -163,24 +163,24 @@ def test_with_plugin_wheel(self): opts = build_deployment_options(config, plugin_wheel=wheel_path) assert opts["ray_actor_options"]["runtime_env"]["pip"] == [str(wheel_path)] - def test_llama_cpp_honors_num_gpus(self): + def test_llama_server_honors_num_gpus(self): config = ModelshipModelConfig( name="test-model", model="some-model", usecase=ModelUsecase.generate, - loader=ModelLoader.llama_cpp, + loader=ModelLoader.llama_server, num_gpus=2, ) opts = build_deployment_options(config) assert opts["ray_actor_options"]["num_gpus"] == 2 assert "placement_group_bundles" not in opts - def test_llama_cpp_num_gpus_zero_stays_cpu(self): + def test_llama_server_num_gpus_zero_stays_cpu(self): config = ModelshipModelConfig( name="test-model", model="some-model", usecase=ModelUsecase.generate, - loader=ModelLoader.llama_cpp, + loader=ModelLoader.llama_server, num_gpus=0, ) opts = build_deployment_options(config) diff --git a/tests/test_parser_resolution.py b/tests/test_parser_resolution.py index 972040b..8696254 100644 --- a/tests/test_parser_resolution.py +++ b/tests/test_parser_resolution.py @@ -11,7 +11,6 @@ from modelship.deploy.config import resolve_all_reasoning_parsers, resolve_all_tool_parsers from modelship.infer.infer_config import ( - LlamaCppConfig, ModelLoader, ModelshipConfig, ModelshipModelConfig, @@ -135,15 +134,6 @@ def test_vllm_opt_out_leaves_none(self): resolve_all_tool_parsers(ModelshipConfig(models=[cfg])) assert cfg._resolved_tool_call_parser is None - def test_llama_cpp_chat_format_opts_out(self): - cfg = _make_cfg( - loader=ModelLoader.llama_cpp, - llama_cpp_config=LlamaCppConfig(chat_format="chatml-function-calling"), - ) - cfg._resolved_chat_template = "{% if tools %}{% endif %}" - resolve_all_tool_parsers(ModelshipConfig(models=[cfg])) - assert cfg._resolved_tool_call_parser is None - class TestResolveSkipSpecialTokens: """``_resolved_skip_special_tokens`` is pinned by the parser's flag. diff --git a/tests/test_preflight_llama_cpp.py b/tests/test_preflight_llama_cpp.py deleted file mode 100644 index 7d94060..0000000 --- a/tests/test_preflight_llama_cpp.py +++ /dev/null @@ -1,281 +0,0 @@ -"""Tests for the LlamaCppPreflight estimator.""" - -from __future__ import annotations - -from pathlib import Path -from unittest.mock import patch - -from modelship.infer.infer_config import ( - LlamaCppConfig, - ModelLoader, - ModelshipModelConfig, - ModelUsecase, -) -from modelship.preflight import HardwareProfile -from modelship.preflight.llama_cpp import ( - LlamaCppPreflight, - _ggml_type_bytes, - _GGUFMeta, - _read_int, - _read_string, - _resolve_kv_dtype_bytes, -) - - -def _make_config( - *, - resolved_path: str | None = None, - llama_cpp_kwargs: dict | None = None, - num_gpus: float = 0, -) -> ModelshipModelConfig: - cfg = ModelshipModelConfig( - name="test-model", - model="org/test-model", - usecase=ModelUsecase.generate, - loader=ModelLoader.llama_cpp, - llama_cpp_config=LlamaCppConfig(**(llama_cpp_kwargs or {})), - num_gpus=num_gpus, - ) - cfg._resolved_path = resolved_path - return cfg - - -def _write_dummy_gguf(tmp_path: Path) -> Path: - """Write a tiny placeholder file with a `.gguf` suffix. Existence + path - are all that's checked before parsing; tests mock both `_read_gguf_metadata` - and `_weight_bytes` to inject the scenarios that matter.""" - path = tmp_path / "model.gguf" - path.write_bytes(b"\0" * 1024) - return path - - -_LLAMA_META = _GGUFMeta(block_count=32, head_count_kv=8, head_dim=128, context_length=8192) - - -class TestLlamaCppPreflightSkips: - def test_no_ram_returns_empty(self, tmp_path): - cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) - assert LlamaCppPreflight().recommend(cfg, HardwareProfile(ram_bytes=0)) == {} - - def test_no_resolved_path_returns_empty(self): - cfg = _make_config() - hw = HardwareProfile(ram_bytes=64 * 1024**3) - assert LlamaCppPreflight().recommend(cfg, hw) == {} - - def test_missing_file_returns_empty(self, tmp_path): - cfg = _make_config(resolved_path=str(tmp_path / "nope.gguf")) - hw = HardwareProfile(ram_bytes=64 * 1024**3) - assert LlamaCppPreflight().recommend(cfg, hw) == {} - - def test_unparseable_gguf_returns_empty(self, tmp_path): - cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) - hw = HardwareProfile(ram_bytes=64 * 1024**3) - # Real GGUFReader will fail on our zero-bytes file; preflight must skip. - assert LlamaCppPreflight().recommend(cfg, hw) == {} - - def test_gpu_offload_skips_ram_sizing(self, tmp_path): - # num_gpus > 0 means weights live in VRAM; the RAM-based sizer must not - # run at all, regardless of GGUF metadata or RAM available. - cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path)), num_gpus=1) - hw = HardwareProfile(ram_bytes=64 * 1024**3) - assert LlamaCppPreflight().recommend(cfg, hw) == {} - - -class TestLlamaCppPreflightRecommends: - def test_roomy_budget_caps_at_context_length(self, tmp_path): - # 1 GiB weights, 64 GiB RAM → plenty of headroom; cap at model's max. - cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) - hw = HardwareProfile(ram_bytes=64 * 1024**3) - - with ( - patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=_LLAMA_META), - patch("modelship.preflight.llama_cpp._weight_bytes", return_value=1 * 1024**3), - ): - rec = LlamaCppPreflight().recommend(cfg, hw) - - assert rec == {"n_ctx": 8192} - - def test_constrained_budget_recommends_lower_nctx(self, tmp_path): - # Tight: 4 GiB RAM, 1.75 GiB weights → budget < model's max context. - cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) - hw = HardwareProfile(ram_bytes=4 * 1024**3) - - # Giant-context model so the cap doesn't kick in first. - big_meta = _GGUFMeta( - block_count=_LLAMA_META.block_count, - head_count_kv=_LLAMA_META.head_count_kv, - head_dim=_LLAMA_META.head_dim, - context_length=131072, - ) - with ( - patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=big_meta), - patch("modelship.preflight.llama_cpp._weight_bytes", return_value=int(1.75 * 1024**3)), - ): - rec = LlamaCppPreflight().recommend(cfg, hw) - - assert "n_ctx" in rec - assert rec["n_ctx"] < 131072 - assert rec["n_ctx"] % 256 == 0 - - def test_missing_context_length_applies_safety_cap(self, tmp_path): - # GGUF lacks `{arch}.context_length`. On a huge-RAM host the math - # would otherwise hand out a context far beyond the model's training - # window; the cap (32768) must kick in instead. - cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) - hw = HardwareProfile(ram_bytes=1024 * 1024**3) # 1 TiB - - no_ctx_meta = _GGUFMeta( - block_count=_LLAMA_META.block_count, - head_count_kv=_LLAMA_META.head_count_kv, - head_dim=_LLAMA_META.head_dim, - context_length=None, - ) - with ( - patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=no_ctx_meta), - patch("modelship.preflight.llama_cpp._weight_bytes", return_value=1 * 1024**3), - ): - rec = LlamaCppPreflight().recommend(cfg, hw) - - assert rec == {"n_ctx": 32768} - - def test_sizes_against_available_ram_not_total(self, tmp_path): - # Same total RAM, different *free* RAM → the box with more free RAM gets a - # larger n_ctx. This is the multi-model fix: the generate model sizes against - # what's left after the satellites, not the box's total. - cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) - big_meta = _GGUFMeta( - block_count=_LLAMA_META.block_count, - head_count_kv=_LLAMA_META.head_count_kv, - head_dim=_LLAMA_META.head_dim, - context_length=131072, # giant so the budget, not the cap, decides - ) - tight = HardwareProfile(ram_bytes=64 * 1024**3, available_ram_bytes=6 * 1024**3) - roomy = HardwareProfile(ram_bytes=64 * 1024**3, available_ram_bytes=24 * 1024**3) - with ( - patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=big_meta), - patch("modelship.preflight.llama_cpp._weight_bytes", return_value=int(1.5 * 1024**3)), - ): - rec_tight = LlamaCppPreflight().recommend(cfg, tight) - rec_roomy = LlamaCppPreflight().recommend(cfg, roomy) - assert rec_roomy["n_ctx"] > rec_tight["n_ctx"] - - def test_zero_available_falls_back_to_total(self, tmp_path): - # available_ram_bytes == 0 (probe read nothing) → size against total, matching - # the pre-change behaviour (no regression for single-model deploys). - cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) - only_total = HardwareProfile(ram_bytes=64 * 1024**3, available_ram_bytes=0) - with ( - patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=_LLAMA_META), - patch("modelship.preflight.llama_cpp._weight_bytes", return_value=1 * 1024**3), - ): - rec = LlamaCppPreflight().recommend(cfg, only_total) - assert rec == {"n_ctx": 8192} # caps at the model's context_length, as with total - - def test_oversubscribed_budget_returns_empty(self, tmp_path): - # Weights >> available RAM → no budget left; skip recommendation - # rather than ship something the user can't actually run. - cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) - hw = HardwareProfile(ram_bytes=8 * 1024**3) - - with ( - patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=_LLAMA_META), - patch("modelship.preflight.llama_cpp._weight_bytes", return_value=32 * 1024**3), - ): - rec = LlamaCppPreflight().recommend(cfg, hw) - - assert rec == {} - - -class TestKvDtypeResolution: - def test_default_is_fp16(self): - cfg = _make_config() - assert _resolve_kv_dtype_bytes(cfg) == 2 - - def test_string_alias_q8_0(self): - cfg = _make_config(llama_cpp_kwargs={"model_kwargs": {"type_k": "q8_0", "type_v": "q8_0"}}) - assert _resolve_kv_dtype_bytes(cfg) == 1 - - def test_mixed_types_uses_larger(self): - cfg = _make_config(llama_cpp_kwargs={"model_kwargs": {"type_k": "f16", "type_v": "q8_0"}}) - # Conservative: pick the larger of the two so we don't under-estimate. - assert _resolve_kv_dtype_bytes(cfg) == 2 - - def test_unknown_falls_back_to_fp16(self): - cfg = _make_config(llama_cpp_kwargs={"model_kwargs": {"type_k": "q3_k_xxs"}}) - assert _resolve_kv_dtype_bytes(cfg) == 2 - - def test_ggml_enum_int(self): - # 0 = F32, 1 = F16 in ggml's type enum - assert _ggml_type_bytes(0) == 4 - assert _ggml_type_bytes(1) == 2 - assert _ggml_type_bytes(999) is None - - -class _FakeField: - """Imitates gguf.gguf_reader.ReaderField just enough for _read_field_value.""" - - def __init__(self, value: object) -> None: - self._value = value - - def contents(self) -> object: - return self._value - - -class _FakeReader: - def __init__(self, fields: dict[str, object]) -> None: - self._fields = {k: _FakeField(v) for k, v in fields.items()} - - def get_field(self, key: str) -> _FakeField | None: - return self._fields.get(key) - - -class TestFieldExtraction: - """gguf hands back several shapes for `ReaderField.contents()`; these - tests pin the behavior so future gguf bumps don't silently regress.""" - - def test_python_int_scalar(self): - reader = _FakeReader({"llama.block_count": 32}) - assert _read_int(reader, "llama.block_count") == 32 - - def test_numpy_scalar(self): - import numpy as np - - reader = _FakeReader({"llama.block_count": np.uint32(32)}) - assert _read_int(reader, "llama.block_count") == 32 - - def test_numpy_array_single_element(self): - import numpy as np - - # Some gguf versions return numpy arrays even for scalar metadata. - reader = _FakeReader({"llama.block_count": np.array([32], dtype=np.uint32)}) - assert _read_int(reader, "llama.block_count") == 32 - - def test_python_list(self): - reader = _FakeReader({"llama.block_count": [32]}) - assert _read_int(reader, "llama.block_count") == 32 - - def test_string_as_bytes_array(self): - import numpy as np - - reader = _FakeReader({"general.architecture": np.array([b"llama"], dtype=object)}) - assert _read_string(reader, "general.architecture") == "llama" - - def test_missing_key_returns_none(self): - reader = _FakeReader({}) - assert _read_int(reader, "llama.block_count") is None - assert _read_string(reader, "general.architecture") is None - - -class TestRegistration: - def test_run_preflight_dispatches_to_llama_cpp(self, tmp_path): - cfg = _make_config(resolved_path=str(_write_dummy_gguf(tmp_path))) - hw = HardwareProfile(ram_bytes=64 * 1024**3) - - from modelship.preflight import run_preflight - - with ( - patch("modelship.preflight.llama_cpp._read_gguf_metadata", return_value=_LLAMA_META), - patch("modelship.preflight.llama_cpp._weight_bytes", return_value=1 * 1024**3), - ): - rec = run_preflight(cfg, hw) - assert rec == {"n_ctx": 8192} diff --git a/tests/test_profiles_generator.py b/tests/test_profiles_generator.py index eabef06..78ef8c7 100644 --- a/tests/test_profiles_generator.py +++ b/tests/test_profiles_generator.py @@ -156,9 +156,9 @@ def test_refuse_path_writes_nothing(tmp_path): def test_cpu_allocation_never_exceeds_budget_when_scaling_down(): # Adversarial tiny budget: naive per-term rounding would sum to 0.05 > 0.04. specs = [ - _spec("g", ModelLoader.llama_cpp, ModelUsecase.generate), + _spec("g", ModelLoader.llama_server, ModelUsecase.generate), _spec("i", ModelLoader.stable_diffusion_cpp, ModelUsecase.image), - _spec("e", ModelLoader.llama_cpp, ModelUsecase.embed), + _spec("e", ModelLoader.llama_server, ModelUsecase.embed), ] allocs = _cpu_allocation(specs, 0.04) assert round(sum(allocs), 2) <= 0.04 @@ -171,7 +171,7 @@ def test_cpu_allocation_leftover_goes_to_a_single_anchor(): specs = [ _spec("g1", ModelLoader.vllm, ModelUsecase.generate), _spec("g2", ModelLoader.vllm, ModelUsecase.generate), - _spec("e", ModelLoader.llama_cpp, ModelUsecase.embed), + _spec("e", ModelLoader.llama_server, ModelUsecase.embed), ] allocs = _cpu_allocation(specs, 16.0) assert sum(allocs) <= 16.0 + 1e-9 diff --git a/tests/test_profiles_selector.py b/tests/test_profiles_selector.py index 5be4cc0..d5e5998 100644 --- a/tests/test_profiles_selector.py +++ b/tests/test_profiles_selector.py @@ -162,7 +162,7 @@ def test_gpu_box_uses_gpu_loaders_for_generate_image_cpu_for_satellites(): emb = next(s for s in specs if s.usecase == ModelUsecase.embed) assert gen.loader == ModelLoader.vllm assert img.loader == ModelLoader.diffusers - assert emb.loader == ModelLoader.llama_cpp # satellite stays CPU + assert emb.loader == ModelLoader.llama_server # satellite stays CPU def test_studio_on_tiny_gpu_is_refused_on_vram(): diff --git a/tests/test_vllm_gguf_guard.py b/tests/test_vllm_gguf_guard.py index 27a3509..d1fe53c 100644 --- a/tests/test_vllm_gguf_guard.py +++ b/tests/test_vllm_gguf_guard.py @@ -36,8 +36,8 @@ def test_vllm_gguf_rejected(self): ): resolve_all_model_sources(ModelshipConfig(models=[cfg])) - def test_llama_cpp_gguf_allowed(self): - cfg = _make_cfg(loader=ModelLoader.llama_cpp, num_gpus=0) + def test_llama_server_gguf_allowed(self): + cfg = _make_cfg(loader=ModelLoader.llama_server, num_gpus=0) with patch( "modelship.deploy.config.resolve_model_source", return_value="/cache/model-Q4_K_M.gguf", diff --git a/uv.lock b/uv.lock index dcf514d..230d547 100644 --- a/uv.lock +++ b/uv.lock @@ -1459,56 +1459,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/ba/c63c5786dfee4c3417094c4b00966e61e4a63efecee22cb7b4c0387dda83/librosa-0.11.0-py3-none-any.whl", hash = "sha256:0b6415c4fd68bff4c29288abe67c6d80b587e0e1e2cfb0aad23e4559504a7fa1", size = 260749, upload-time = "2025-03-11T15:09:52.982Z" }, ] -[[package]] -name = "llama-cpp-python" -version = "0.3.32" -source = { registry = "https://abetlen.github.io/llama-cpp-python/whl/cu130" } -resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform == 'darwin'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -dependencies = [ - { name = "diskcache" }, - { name = "jinja2" }, - { name = "numpy" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.32-cu130/llama_cpp_python-0.3.32-py3-none-manylinux_2_35_x86_64.whl" }, - { url = "https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.32-cu130/llama_cpp_python-0.3.32-py3-none-win_amd64.whl" }, -] - -[[package]] -name = "llama-cpp-python" -version = "0.3.32" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "(platform_machine == 's390x' and sys_platform == 'win32') or (platform_machine == 'x86_64' and sys_platform == 'win32')", - "platform_machine == 'aarch64' and sys_platform == 'win32'", - "(platform_machine == 'ppc64le' and sys_platform == 'win32') or (platform_machine == 'riscv64' and sys_platform == 'win32')", - "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'win32'", - "(platform_machine == 's390x' and sys_platform == 'emscripten') or (platform_machine == 'x86_64' and sys_platform == 'emscripten')", - "platform_machine == 'aarch64' and sys_platform == 'emscripten'", - "(platform_machine == 'ppc64le' and sys_platform == 'emscripten') or (platform_machine == 'riscv64' and sys_platform == 'emscripten')", - "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'emscripten'", - "(platform_machine == 's390x' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", - "platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "(platform_machine == 'ppc64le' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'riscv64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')", - "platform_machine != 'aarch64' and platform_machine != 'ppc64le' and platform_machine != 'riscv64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "(platform_machine == 's390x' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'darwin')", - "platform_machine == 'aarch64' and sys_platform == 'darwin'", - "platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin'", -] -dependencies = [ - { name = "diskcache" }, - { name = "jinja2" }, - { name = "numpy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/c8/3eb9c10c138eaa9d6148471701476169322eafbd825fdc13ec326552b516/llama_cpp_python-0.3.32.tar.gz", hash = "sha256:b06502361770f82eb08b7f1a192eb084b9ead2b88fe32cda8c397a2782eabde6", size = 70308017, upload-time = "2026-06-29T05:59:48.626Z" } - [[package]] name = "llguidance" version = "1.7.6" @@ -1722,7 +1672,6 @@ bark = [ { name = "bark" }, ] cpu = [ - { name = "llama-cpp-python", version = "0.3.32", source = { registry = "https://pypi.org/simple" } }, { name = "onnxruntime" }, { name = "sentence-transformers" }, { name = "stable-diffusion-cpp-python" }, @@ -1746,7 +1695,6 @@ gpu = [ { name = "bitsandbytes" }, { name = "diffusers" }, { name = "flashinfer-python" }, - { name = "llama-cpp-python", version = "0.3.32", source = { registry = "https://abetlen.github.io/llama-cpp-python/whl/cu130" } }, { name = "nvidia-ml-py" }, { name = "onnxruntime-gpu" }, { name = "sentence-transformers" }, @@ -1785,8 +1733,6 @@ requires-dist = [ { name = "httpx", specifier = ">=0.28.1" }, { name = "kokoroonnx", marker = "extra == 'kokoroonnx'", editable = "plugins/kokoroonnx" }, { name = "librosa", specifier = ">=0.11.0" }, - { name = "llama-cpp-python", marker = "extra == 'cpu'", specifier = ">=0.3.32" }, - { name = "llama-cpp-python", marker = "extra == 'gpu'", specifier = ">=0.3.32", index = "https://abetlen.github.io/llama-cpp-python/whl/cu130", conflict = { package = "modelship", extra = "gpu" } }, { name = "numpy", specifier = ">=2.2.6" }, { name = "nvidia-ml-py", marker = "extra == 'gpu'" }, { name = "onnxruntime", marker = "extra == 'cpu'", specifier = ">=1.20.1" }, From 1c16643a1ab7d036d9220fdbdc32cce7ab0b2432 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Sat, 4 Jul 2026 18:10:27 +0000 Subject: [PATCH 34/35] feat!: remove transformers loader and bark plugin vLLM 0.24's Transformers fallback backend absorbs the long-tail architecture use case with continuous batching; every other transformers usecase already has a production home (embed/vision -> vllm, chat GGUF -> llama_server, STT -> vllm/whispercpp, TTS -> kokoroonnx/orpheus). bark was an unused experiment with no replacement. --- AGENTS.md | 2 +- CLAUDE.md | 2 +- README.md | 22 +- config/examples/README.md | 1 - config/examples/transformers-cpu.yaml | 39 -- docs/architecture.md | 17 +- docs/model-configuration.md | 94 +--- docs/monitoring.md | 7 +- docs/plugins.md | 4 +- helm/modelship/values.yaml | 2 +- modelship/deploy/config.py | 24 +- modelship/infer/infer_config.py | 18 +- modelship/infer/model_deployment.py | 4 - modelship/infer/transformers/capabilities.py | 25 - .../infer/transformers/openai/serving_chat.py | 349 ------------- .../transformers/openai/serving_embedding.py | 67 --- .../transformers/openai/serving_speech.py | 55 -- .../openai/serving_transcription.py | 122 ----- .../infer/transformers/transformers_infer.py | 205 -------- .../parsers/tool_calling/parsers/base.py | 6 +- .../parsers/tool_calling/parsers/gemma.py | 2 +- modelship/openai/protocol/chat.py | 7 +- plugins/bark/README.md | 60 --- plugins/bark/bark/__init__.py | 3 - plugins/bark/bark/bark.py | 78 --- plugins/bark/pyproject.toml | 17 - pyproject.toml | 5 - tests/test_config.py | 85 +-- tests/test_gemma4_parsers.py | 6 +- tests/test_integration.py | 494 +----------------- tests/test_mistral_specials_smoketest.py | 24 +- tests/test_parser_resolution.py | 24 +- tests/test_preflight.py | 2 +- tests/test_tool_calling.py | 5 +- tests/test_transformers_capabilities.py | 35 -- tests/test_transformers_chat_reasoning.py | 178 ------- tests/test_transformers_chat_tools.py | 228 -------- tests/test_vllm_vision.py | 2 +- uv.lock | 57 +- 39 files changed, 88 insertions(+), 2289 deletions(-) delete mode 100644 config/examples/transformers-cpu.yaml delete mode 100644 modelship/infer/transformers/capabilities.py delete mode 100644 modelship/infer/transformers/openai/serving_chat.py delete mode 100644 modelship/infer/transformers/openai/serving_embedding.py delete mode 100644 modelship/infer/transformers/openai/serving_speech.py delete mode 100644 modelship/infer/transformers/openai/serving_transcription.py delete mode 100644 modelship/infer/transformers/transformers_infer.py delete mode 100644 plugins/bark/README.md delete mode 100644 plugins/bark/bark/__init__.py delete mode 100644 plugins/bark/bark/bark.py delete mode 100644 plugins/bark/pyproject.toml delete mode 100644 tests/test_transformers_capabilities.py delete mode 100644 tests/test_transformers_chat_reasoning.py delete mode 100644 tests/test_transformers_chat_tools.py diff --git a/AGENTS.md b/AGENTS.md index d8179b9..317205d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,7 +69,7 @@ The Docker image's `CMD` is `uv run --no-sync mship_deploy.py` (against the venv - `modelship/openai/api.py` — FastAPI gateway. Uses `RequestWatcher` + a single shared `DisconnectRegistry` Ray actor (keyed by request id) to propagate client disconnects across process boundaries. - `modelship/infer/model_deployment.py` — the single `@serve.deployment` actor class; lazily imports the right backend based on `config.loader`. - `modelship/infer/infer_config.py` — pydantic config schemas **and** `RawRequestProxy` / `DisconnectRegistry`. `RawRequestProxy` exists because FastAPI `Request` cannot cross Ray process boundaries; any new attribute vLLM reads from `raw_request` must be added there. -- `modelship/infer/{vllm,transformers,diffusers,custom}/` — one subdir per loader. Each has an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. `modelship/infer/llama_server/llama_server_infer.py` is a flat file with no `openai/` subpackage — it proxies a `llama-server` subprocess's own OpenAI-compatible HTTP API rather than parsing output in-process. +- `modelship/infer/{vllm,diffusers,custom}/` — one subdir per loader. Each has an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. `modelship/infer/llama_server/llama_server_infer.py` is a flat file with no `openai/` subpackage — it proxies a `llama-server` subprocess's own OpenAI-compatible HTTP API rather than parsing output in-process. - `modelship/plugins/base_plugin.py` — `BasePlugin` ABC that plugin packages subclass as `ModelPlugin`. - `plugins/*` — workspace packages, each opt-in via a root extra. The plugin module name and the extra name must match (`ensure_plugin()` calls `importlib.import_module(config.plugin)` and the error message says `uv sync --extra `). diff --git a/CLAUDE.md b/CLAUDE.md index c161c1c..c7c5dcf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,7 +49,7 @@ Docker's `CMD` is `uv run --no-sync mship_deploy.py` (auto-detecting CPUs/GPUs u - `modelship/openai/api.py` — FastAPI gateway. Uses `RequestWatcher` + a single shared `DisconnectRegistry` Ray actor (keyed by request id) to propagate client disconnects across process boundaries and cancel in-flight inference. - `modelship/infer/model_deployment.py` — the single `@serve.deployment` actor class; lazily imports the right backend from `config.loader`. - `modelship/infer/infer_config.py` — pydantic config schemas plus `RawRequestProxy` / `DisconnectRegistry`. `RawRequestProxy` exists because FastAPI `Request` can't cross Ray process boundaries. **Any new attribute vLLM reads from `raw_request` must be added there.** -- `modelship/infer/{vllm,transformers,diffusers,custom}/` — one subdir per loader, each with an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. `modelship/infer/llama_server/llama_server_infer.py` is the exception: a single flat file (no `openai/` subpackage) — it proxies a `llama-server` subprocess's own OpenAI-compatible HTTP API rather than running modelship's parsers in-process. +- `modelship/infer/{vllm,diffusers,custom}/` — one subdir per loader, each with an `*_infer.py` and (for non-custom) an `openai/` adapter subpackage. `modelship/infer/llama_server/llama_server_infer.py` is the exception: a single flat file (no `openai/` subpackage) — it proxies a `llama-server` subprocess's own OpenAI-compatible HTTP API rather than running modelship's parsers in-process. - `modelship/plugins/base_plugin.py` — `BasePlugin` ABC that each plugin package subclasses as `ModelPlugin`. - `plugins/*` — workspace packages, each opt-in via a matching root extra. The plugin module name and extra name **must match** (`ensure_plugin()` does `importlib.import_module(config.plugin)`). diff --git a/README.md b/README.md index e814e88..45fa332 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Python 3.12+](https://img.shields.io/badge/python-3.12+-blue.svg)](https://www.python.org/downloads/) -Self-hosted, OpenAI-compatible inference for the agentic era. Modelship runs your whole AI stack — reasoning LLMs with universal tool calling and the Responses API, plus embeddings, speech-to-text, text-to-speech, and image generation — as many models sharing your GPUs (or CPU) behind a single gateway. Built on [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) with pluggable inference backends: [vLLM](https://github.com/vllm-project/vllm) for high-throughput GPU or CPU inference, [HuggingFace Transformers](https://github.com/huggingface/transformers) for CPU and lightweight GPU workloads, [llama.cpp](https://github.com/ggml-org/llama.cpp) for high-efficiency GGUF models on CPU or GPU, [Diffusers](https://github.com/huggingface/diffusers) for image generation, and a plugin system for custom backends. +Self-hosted, OpenAI-compatible inference for the agentic era. Modelship runs your whole AI stack — reasoning LLMs with universal tool calling and the Responses API, plus embeddings, speech-to-text, text-to-speech, and image generation — as many models sharing your GPUs (or CPU) behind a single gateway. Built on [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) with pluggable inference backends: [vLLM](https://github.com/vllm-project/vllm) for high-throughput GPU or CPU inference, [llama.cpp](https://github.com/ggml-org/llama.cpp) for high-efficiency GGUF models on CPU or GPU, [Diffusers](https://github.com/huggingface/diffusers) for image generation, and a plugin system for custom backends. ## Why Modelship? @@ -12,7 +12,7 @@ Most self-hosted inference tools focus on running a single model. Modelship is f - **One server, many models** — run a full AI stack (chat + TTS + STT + embeddings + image gen) on a single machine instead of juggling separate services - **GPU memory control** — allocate exact GPU fractions per model (e.g. 70% for the LLM, 5% for TTS) so everything fits on your hardware -- **Mix and match backends** — use vLLM for high-throughput GPU inference, Transformers or llama.cpp for CPU-only workloads, Diffusers for images, and plugins for custom backends — in the same deployment +- **Mix and match backends** — use vLLM for high-throughput GPU inference, vLLM or llama.cpp for CPU-only workloads, Diffusers for images, and plugins for custom backends — in the same deployment - **Agentic-ready** — reasoning (`` → `reasoning_content`), universal tool/function calling, and the `/v1/responses` API work across the vLLM and llama.cpp (`llama_server`) loaders — the OpenAI-spec surface agents already target - **Drop-in OpenAI replacement** — any OpenAI SDK client works out of the box, making it easy to integrate with existing apps and tools like [Home Assistant](docs/home-assistant.md) @@ -41,7 +41,7 @@ graph TD EMB["Embedding Deployment
e.g. Nomic Embed
50% GPU"] end - subgraph CPU["CPU — Transformers / llama-server"] + subgraph CPU["CPU — vLLM / llama-server"] LLM_CPU["LLM Deployment
e.g. Qwen3-0.6B
CPU-only"] STT_CPU["STT Deployment
e.g. Whisper Small
CPU-only"] end @@ -51,33 +51,32 @@ graph TD end ``` -Each model runs as an isolated [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) deployment with its own lifecycle, health checks, and resource budget. Five inference backends are available: +Each model runs as an isolated [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) deployment with its own lifecycle, health checks, and resource budget. Four inference backends are available: | Backend | Best for | GPU required | |---|---|---| | **vLLM** | High-throughput chat, embeddings, transcription | No — installs on GPU or CPU | | **llama.cpp** (`llama_server`) | High-efficiency quantized GGUF models (chat, embeddings, vision) | No | -| **Transformers** | Chat, embeddings, transcription, TTS on CPU or lightweight GPU | No | | **Diffusers** | Image generation | Yes | -| **Custom (plugins)** | TTS backends (Kokoro ONNX, Bark, Orpheus), STT backends (whisper.cpp) | No | +| **Custom (plugins)** | TTS backends (Kokoro ONNX, Orpheus), STT backends (whisper.cpp) | No | -Models can be deployed across multiple GPUs, run on CPU-only, or both — multiple deployments of the same model (e.g. one on GPU via vLLM, one on CPU via Transformers) are load-balanced with round-robin routing. Each deployment can also scale horizontally with `num_replicas`. +Models can be deployed across multiple GPUs, run on CPU-only, or both — multiple deployments of the same model (e.g. one on GPU via vLLM, one on CPU via vLLM or llama.cpp) are load-balanced with round-robin routing. Each deployment can also scale horizontally with `num_replicas`. ... ## Requirements - **Docker** (or Python 3.12+ with `uv` for local development) -- **NVIDIA GPU** (optional) — 16 GB+ VRAM recommended for a full stack (LLM + TTS + STT + embeddings) via vLLM; 8 GB is sufficient for lighter setups. Not required when using the Transformers backend on CPU +- **NVIDIA GPU** (optional) — 16 GB+ VRAM recommended for a full stack (LLM + TTS + STT + embeddings) via vLLM; 8 GB is sufficient for lighter setups. Not required when using the vLLM or llama.cpp backends on CPU - **[NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html)** — required only when running GPU models in Docker - **HuggingFace token** for gated models ## Features - **Multi-model, multi-GPU** — run chat, embedding, STT, TTS, and image generation models simultaneously across one or more GPUs with tunable per-model GPU memory allocation -- **CPU-only support** — run models without a GPU using the Transformers backend (chat, embeddings, transcription, TTS). Useful for development, testing, or small models that don't need GPU acceleration -- **Multiple inference backends** — vLLM for high-throughput GPU inference, HuggingFace Transformers for CPU and lightweight GPU workloads, Diffusers for image generation, and a plugin system for custom backends +- **CPU-only support** — run models without a GPU using the vLLM or llama.cpp (`llama_server`) backends (chat, embeddings, transcription, vision). Useful for development, testing, or small models that don't need GPU acceleration +- **Multiple inference backends** — vLLM for high-throughput GPU or CPU inference, llama.cpp for efficient quantized GGUF models on CPU or GPU, Diffusers for image generation, and a plugin system for custom backends - **Zero-downtime hot-reloads** — modify your `models.yaml` and run a cluster reconcile; changes are applied incrementally without interrupting the API gateway or unchanged models -- **Advanced agentic capabilities** — native support for DeepSeek-style reasoning (`` blocks parsed into `reasoning_content`) and universal tool/function calling across the vLLM, GGUF (`llama_server`), and Transformers backends +- **Advanced agentic capabilities** — native support for DeepSeek-style reasoning (`` blocks parsed into `reasoning_content`) and universal tool/function calling across the vLLM and GGUF (`llama_server`) backends - **Per-model isolated deployments** — each model runs in its own Ray Serve deployment with independent lifecycle, health checks, failure isolation, and configurable replica count - **OpenAI-compatible API** — drop-in replacement for any OpenAI SDK client - **Streaming** — SSE streaming for chat completions and TTS audio @@ -160,7 +159,6 @@ Modelship's TTS and STT systems are built around a plugin architecture — each Built-in plugins: - [Kokoro ONNX](plugins/kokoroonnx/README.md) — lightweight TTS via ONNX Runtime (CPU or GPU) -- [Bark](plugins/bark/README.md) — multilingual TTS by Suno (GPU recommended) - [Orpheus](plugins/orpheus/README.md) — expressive TTS - [whisper.cpp](plugins/whispercpp/README.md) — CPU-only STT via `pywhispercpp` diff --git a/config/examples/README.md b/config/examples/README.md index 364b2f1..4237b51 100644 --- a/config/examples/README.md +++ b/config/examples/README.md @@ -6,7 +6,6 @@ Ready-to-run `models.yaml` configs for common scenarios. Mount one into the cont |---|---|---| | [llama-server.yaml](llama-server.yaml) | Quantized GGUF chat (concurrent), vision, embeddings via a llama-server subprocess | CPU or NVIDIA GPU | | [vllm-cpu.yaml](vllm-cpu.yaml) | Quantized (AWQ/GPTQ) chat via vLLM's CPU backend — the path for gemma tool calling, which llama-server's parsers can't handle | CPU | -| [transformers-cpu.yaml](transformers-cpu.yaml) | Llama 3.2 1B + Nomic embed + Whisper + MMS-TTS | CPU | | [vllm.yaml](vllm.yaml) | High-throughput chat with tool calling, embeddings, Whisper | NVIDIA GPU | | [diffusers.yaml](diffusers.yaml) | SDXL Turbo image generation | NVIDIA GPU | | [kokoro-tts.yaml](kokoro-tts.yaml) | Kokoro ONNX TTS with GPU + CPU fallback replicas | Mixed | diff --git a/config/examples/transformers-cpu.yaml b/config/examples/transformers-cpu.yaml deleted file mode 100644 index f15b701..0000000 --- a/config/examples/transformers-cpu.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Transformers loader examples — CPU-only, no GPU required. -# Great for development, small models, and environments without CUDA. - -models: - # Small chat model - - name: "llm" - model: "meta-llama/Llama-3.2-1B-Instruct" - usecase: "generate" - loader: "transformers" - num_cpus: 4 - transformers_config: - torch_dtype: "float32" - - # Embeddings - - name: "nomic-embed" - model: "nomic-ai/nomic-embed-text-v1.5" - usecase: "embed" - loader: "transformers" - num_cpus: 1 - transformers_config: - trust_remote_code: true - - # Speech-to-text (Whisper) with timestamps - - name: "whisper" - model: "openai/whisper-small" - usecase: "transcription" - loader: "transformers" - num_cpus: 2 - transformers_config: - torch_dtype: "float32" - pipeline_kwargs: - return_timestamps: true - - # TTS - - name: "facebook" - model: "facebook/mms-tts-eng" - usecase: "tts" - loader: "transformers" - num_cpus: 1 diff --git a/docs/architecture.md b/docs/architecture.md index 3405b6e..c26e87a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -4,10 +4,10 @@ Modelship is built on [Ray Serve](https://docs.ray.io/en/latest/serve/) for deployment orchestration and a **FastAPI gateway** that exposes an OpenAI-compatible API. Multiple inference backends are supported: -- **[vLLM](https://github.com/vllm-project/vllm)** — high-throughput GPU inference with continuous batching and PagedAttention -- **[HuggingFace Transformers](https://github.com/huggingface/transformers)** — CPU and lightweight GPU inference for chat, embeddings, transcription, and TTS +- **[vLLM](https://github.com/vllm-project/vllm)** — high-throughput inference with continuous batching and PagedAttention, on GPU or CPU +- **llama-server** — a proxied `llama-server` subprocess for quantized GGUF chat, embeddings, and vision on CPU or GPU - **[HuggingFace Diffusers](https://github.com/huggingface/diffusers)** — image generation via `AutoPipelineForText2Image` -- **Plugin system** — custom TTS and STT backends (Kokoro ONNX, Bark, Orpheus, whisper.cpp) +- **Plugin system** — custom TTS and STT backends (Kokoro ONNX, Orpheus, whisper.cpp) ## Request Lifecycle @@ -15,7 +15,7 @@ Modelship is built on [Ray Serve](https://docs.ray.io/en/latest/serve/) for depl 2. The gateway identifies the target model from the request body 3. A `RequestWatcher` begins monitoring the client connection for disconnects 4. The request is forwarded to the model's Ray Serve deployment via a `RawRequestProxy` (serializable headers + cancellation event) -5. The model deployment runs inference (vLLM, transformers, or plugin) +5. The model deployment runs inference (vLLM, llama-server, diffusers, or plugin) 6. Response streams back as JSON or SSE 7. If the client disconnects mid-inference, the watcher fires the cancellation event, freeing GPU resources immediately @@ -38,12 +38,11 @@ Each deployment uses one of the following loaders: |--------|---------|-----------|--------------| | `vllm` | vLLM engine | Chat/generation, embeddings, transcription, translation | No — installs on GPU or CPU | | `llama_server` | llama-server subprocess | Chat/generation, embeddings, vision (GGUF models) | No — runs on CPU or GPU (GGUF offload) | -| `transformers` | PyTorch + HuggingFace | Chat/generation, embeddings, transcription, translation, TTS | No — runs on CPU or GPU | | `diffusers` | HuggingFace Diffusers | Image generation (any `AutoPipelineForText2Image` model) | Yes | | `stable_diffusion_cpp` | stable-diffusion.cpp | Image generation (GGUF models: SD1.5/SDXL/SD-Turbo, all-in-one Flux) | No — currently CPU-only | -| `custom` | Plugin system | TTS backends (Kokoro ONNX, Bark, Orpheus), STT backends (whisper.cpp) | No | +| `custom` | Plugin system | TTS backends (Kokoro ONNX, Orpheus), STT backends (whisper.cpp) | No | -The `transformers` loader is ideal for CPU-only deployments, smaller models, or development/testing without a GPU. It uses HuggingFace `pipeline()` under the hood and handles audio resampling automatically for speech-to-text models. The `llama_server` loader provides high-efficiency inference for quantized GGUF models on CPU or GPU (`n_gpu_layers` offload, whole GPUs only — fractional `num_gpus` is rejected) by proxying a `llama-server` subprocess's own OpenAI-compatible API. The `vllm` loader provides higher throughput with continuous batching and PagedAttention, on GPU or CPU. +The `llama_server` loader provides high-efficiency inference for quantized GGUF models on CPU or GPU (`n_gpu_layers` offload, whole GPUs only — fractional `num_gpus` is rejected) by proxying a `llama-server` subprocess's own OpenAI-compatible API. The `vllm` loader provides higher throughput with continuous batching and PagedAttention, on GPU or CPU. ## Responses API (`/v1/responses`) @@ -54,7 +53,7 @@ The `transformers` loader is ideal for CPU-only deployments, smaller models, or **Supported:** text, reasoning (as a first-class `reasoning` output item), and client-driven tool calling (`function_call` / `function_call_output` round-trip), streaming and non-streaming — on the `vllm` and `llama_server` loaders. -**404s on other loaders** (`transformers`, `diffusers`, `custom`) — there is no generic fallback; a loader must implement `create_response` itself. +**404s on other loaders** (`diffusers`, `custom`) — there is no generic fallback; a loader must implement `create_response` itself. **Rejected with a clear 400** (rather than silently dropped): `previous_response_id`, `background`, and hosted built-in tools (e.g. `web_search`). These require server-side conversation state, which `/v1/responses` does not yet keep — `store` is accepted but never persisted (the response echoes `store: false`). Encrypted reasoning (`reasoning.encrypted_content`) is also not implemented. @@ -101,7 +100,7 @@ See [Plugin Development](plugins.md) for details. | `modelship/infer/model_deployment.py` | Ray Serve deployment actor | | `modelship/infer/infer_config.py` | Pydantic config models and protocols | | `modelship/infer/vllm/vllm_infer.py` | vLLM engine wrapper | -| `modelship/infer/transformers/transformers_infer.py` | Transformers pipeline wrapper (CPU/GPU) | +| `modelship/infer/llama_server/llama_server_infer.py` | llama-server subprocess proxy (GGUF chat/embed/vision) | | `modelship/infer/diffusers/diffusers_infer.py` | Diffusers pipeline wrapper | | `modelship/infer/stable_diffusion_cpp/stable_diffusion_cpp_infer.py` | stable-diffusion.cpp wrapper (CPU image gen) | | `modelship/plugins/base_plugin.py` | Plugin base classes | diff --git a/docs/model-configuration.md b/docs/model-configuration.md index 4780e4d..aba4412 100644 --- a/docs/model-configuration.md +++ b/docs/model-configuration.md @@ -112,7 +112,7 @@ Selection is **all-or-nothing**: the highest tier whose *complete* stack fits is | `name` | string | Model identifier used in API requests | | `model` | string | HuggingFace repo ID, local path, or `repo:filename` (see [Model source](#model-source)). Required for built-in loaders; optional for `loader: custom` | | `usecase` | string | `generate`, `embed`, `transcription`, `translation`, `tts`, or `image` | -| `loader` | string | `vllm`, `transformers`, `diffusers`, `llama_server`, `stable_diffusion_cpp`, or `custom` | +| `loader` | string | `vllm`, `diffusers`, `llama_server`, `stable_diffusion_cpp`, or `custom` | | `plugin` | string | Plugin module name (required when `loader: custom`); automatically loaded from wheels when referenced | | `num_gpus` | float \| int | GPU allocation. Fractional `< 1` shares one GPU (also sets vLLM `gpu_memory_utilization`); integer `≥ 1` requests that many whole GPUs (for `vllm`, this auto-sets `tensor_parallel_size = num_gpus` unless tp/pp is already specified). | | `num_cpus` | float | CPU units to allocate (default `0.1`) | @@ -120,12 +120,11 @@ Selection is **all-or-nothing**: the highest tier whose *complete* stack fits is | `autoscaling_config` | object | Autoscale replicas with load instead of a fixed `num_replicas` (see [Autoscaling](#autoscaling)). Mutually exclusive with `num_replicas`. | | `max_ongoing_requests` | int | Per-replica Ray Serve concurrency cap (default: Ray Serve's own default of `100`). Streaming requests hold a slot for the whole generation, so a low cap throttles upstream of the engine; raise it for high-concurrency models. Omit to inherit the default. | | `vllm_engine_kwargs` | object | Passed directly to the vLLM engine (see below) | -| `transformers_config` | object | Transformers loader options (see below) | | `diffusers_config` | object | Diffusers pipeline options (see below) | | `llama_server_config` | object | llama-server loader options (see below) | | `stable_diffusion_cpp_config` | object | stable-diffusion.cpp loader options (see below) | | `plugin_config` | object | Plugin-specific options passed through to the plugin | -| `chat_template_kwargs` | object | Extra variables forwarded into the chat-template render on text loaders (`vllm`, `transformers`) — e.g. `enable_thinking: false` for Qwen3. Only has an effect if the model's template branches on the key. A per-request `chat_template_kwargs` overrides the model default on `vllm`. | +| `chat_template_kwargs` | object | Extra variables forwarded into the chat-template render on text loaders (`vllm`) — e.g. `enable_thinking: false` for Qwen3. Only has an effect if the model's template branches on the key. A per-request `chat_template_kwargs` overrides the model default on `vllm`. | ## Model source @@ -308,94 +307,6 @@ models: trust_remote_code: true ``` -## Transformers Loader - -The `transformers` loader uses PyTorch with HuggingFace Transformers. Supports chat/generation, embeddings, transcription, translation, and TTS. Unlike the vLLM loader, it can run entirely on CPU — making it ideal for smaller models, development, or environments without a GPU. - -| Field | Type | Default | Description | -|---|---|---|---| -| `device` | string | `cpu` | Device to run on (`cpu`, `cuda`, `cuda:0`, etc.) | -| `torch_dtype` | string | `auto` | Model dtype (`auto`, `float16`, `bfloat16`, `float32`) | -| `trust_remote_code` | bool | `false` | Allow remote code execution | -| `model_kwargs` | object | `{}` | Extra keyword arguments passed to the model constructor | -| `pipeline_kwargs` | object | `{}` | Extra keyword arguments passed to the pipeline at inference time | -| `tool_call_parser` | string | auto | Parser used to turn raw model output into OpenAI `tool_calls`. Currently supported: `hermes` (Hermes-2-Pro / Qwen2.5-Instruct / many community fine-tunes that emit `{...}` markers), `qwen3_coder` (Qwen3-Coder family — same `` envelope but an XML body of `value`; parameter values are returned as strings), `mistral` (Mistral 7B Instruct v0.3+ / Mistral Small/Large that emit `[TOOL_CALLS][...]`), `llama3_json` (Llama-3.1 / Llama-3.2 Instruct emitting bare `{"name": "...", "parameters": {...}}`). Auto-detected from the chat template when omitted. | - -### Chat / Text Generation (CPU) - -```yaml -models: - - name: qwen - model: Qwen/Qwen3-0.6B - usecase: generate - loader: transformers - num_gpus: 0 - transformers_config: - device: "cpu" -``` - -### Chat with Tool Calling (CPU) - -The transformers loader renders `tools` into the prompt via the model's chat -template and parses the output back into OpenAI `tool_calls`. The model must -have been trained on a Hermes-style tool format (Qwen2.5-Instruct, Hermes-2, -many community fine-tunes); the parser is selected via `tool_call_parser`. - -```yaml -models: - - name: qwen-tools - model: Qwen/Qwen2.5-0.5B-Instruct - usecase: generate - loader: transformers - num_cpus: 2 - transformers_config: - device: "cpu" - tool_call_parser: hermes # this is the default; shown for clarity -``` - -### Speech-to-Text (CPU) - -Audio is automatically decoded and resampled to the model's expected sample rate (e.g. 16kHz for Whisper). - -```yaml -models: - - name: whisper - model: openai/whisper-small - usecase: transcription - loader: transformers - num_gpus: 0 - transformers_config: - device: "cpu" -``` - -### Embeddings (CPU) - -Uses `sentence-transformers` under the hood. - -```yaml -models: - - name: embeddings - model: sentence-transformers/all-MiniLM-L6-v2 - usecase: embed - loader: transformers - num_gpus: 0 - transformers_config: - device: "cpu" -``` - -### TTS (GPU) - -```yaml -models: - - name: my-tts - model: some-org/some-tts-model - usecase: tts - loader: transformers - num_gpus: 0.20 - transformers_config: - device: "cuda:0" -``` - ## Diffusers Loader The `diffusers` loader uses HuggingFace Diffusers for image generation. Any model supported by `AutoPipelineForText2Image` works out of the box. @@ -521,7 +432,6 @@ The `custom` loader delegates to a plugin module. The `plugin` field is required See each plugin's README for configuration details: - [Kokoro ONNX TTS](../plugins/kokoroonnx/README.md) -- [Bark TTS](../plugins/bark/README.md) - [Orpheus TTS](../plugins/orpheus/README.md) - [whisper.cpp STT](../plugins/whispercpp/README.md) diff --git a/docs/monitoring.md b/docs/monitoring.md index 98bb6e4..779818e 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -50,15 +50,10 @@ JSON format example: | `modelship.infer` | Base inference layer | | `modelship.infer.deployment` | Ray Serve model deployment actor | | `modelship.infer.vllm` | vLLM inference backend | -| `modelship.infer.transformers` | Transformers inference backend | -| `modelship.infer.transformers.transcription` | Transformers speech-to-text/translation | -| `modelship.infer.transformers.chat` | Transformers chat/generation | -| `modelship.infer.transformers.embedding` | Transformers embeddings | -| `modelship.infer.transformers.speech` | Transformers TTS | | `modelship.infer.diffusers` | Diffusers inference backend | | `modelship.infer.diffusers.image` | Diffusers image generation | | `modelship.infer.custom` | Custom/plugin inference backend | -| `modelship.plugin.` | Individual plugins (kokoroonnx, bark, orpheus, whispercpp) | +| `modelship.plugin.` | Individual plugins (kokoroonnx, orpheus, whispercpp) | ### Syslog diff --git a/docs/plugins.md b/docs/plugins.md index d40ae93..4196d1b 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -61,7 +61,7 @@ module-root = "" #### Dependency contract -Plugins assume the host environment provides `modelship` itself plus the full core/gpu/cpu stack: `torch`, `torchvision`, `transformers`, `sentence-transformers`, `numpy`, `scipy`, `librosa`, `soundfile`, `onnxruntime[-gpu]`, `diffusers`, `accelerate`, `vllm`. **Do not redeclare any of these in your plugin's `dependencies`.** +Plugins assume the host environment provides `modelship` itself plus the full core/gpu/cpu stack: `torch`, `torchvision`, `transformers`, `numpy`, `scipy`, `librosa`, `soundfile`, `onnxruntime[-gpu]`, `diffusers`, `vllm`. **Do not redeclare any of these in your plugin's `dependencies`.** Why: plugin wheels are shipped to Ray workers via `runtime_env`, which installs them into a per-job venv layered over a base image that already has the core stack baked in. Redeclaring `modelship` or `torch` causes pip to either pull a second copy from PyPI (version drift, two packages on `sys.path`) or error on the layered install. @@ -189,7 +189,7 @@ Every plugin must include a `README.md` in its package root (`plugins/myplugin/R - **Voices / options** — any model-specific choices (voice presets, providers, etc.) - **Example request** — a working `curl` command -See the built-in plugins for reference: [Kokoro ONNX](../plugins/kokoroonnx/README.md), [Bark](../plugins/bark/README.md), [Orpheus](../plugins/orpheus/README.md), [whisper.cpp](../plugins/whispercpp/README.md). +See the built-in plugins for reference: [Kokoro ONNX](../plugins/kokoroonnx/README.md), [Orpheus](../plugins/orpheus/README.md), [whisper.cpp](../plugins/whispercpp/README.md). ## Submitting to this repo diff --git a/helm/modelship/values.yaml b/helm/modelship/values.yaml index b44ff1a..f4eaeb1 100644 --- a/helm/modelship/values.yaml +++ b/helm/modelship/values.yaml @@ -72,7 +72,7 @@ secrets: # Comma-separated keys; when non-empty the gateway requires Authorization. apiKeys: "" -# -- Shared model-weight cache. vLLM/transformers/diffusers download weights here. +# -- Shared model-weight cache. vLLM/diffusers download weights here. # Multi-node clusters REQUIRE ReadWriteMany so every worker shares one copy; # a single-node cluster can use ReadWriteOnce. cache: diff --git a/modelship/deploy/config.py b/modelship/deploy/config.py index 51300c8..b5e2505 100644 --- a/modelship/deploy/config.py +++ b/modelship/deploy/config.py @@ -24,12 +24,9 @@ def _is_explicit_tool_opt_out(cfg) -> bool: that excludes our parser: - vllm: ``enable_auto_tool_choice: false`` — user disabled tool calling. - - transformers: ``tool_calls_enabled: false`` — user disabled tool calling. """ if cfg.loader == ModelLoader.vllm: return cfg.vllm_engine_kwargs is not None and cfg.vllm_engine_kwargs.enable_auto_tool_choice is False - if cfg.loader == ModelLoader.transformers: - return cfg.transformers_config is not None and cfg.transformers_config.tool_calls_enabled is False return False @@ -139,10 +136,7 @@ def resolve_all_model_sources(yml_conf: ModelshipConfig) -> None: if cfg.loader == ModelLoader.custom: continue assert cfg.model is not None # validator guarantees this for built-in loaders - trust_remote_code = bool( - (cfg.vllm_engine_kwargs and cfg.vllm_engine_kwargs.trust_remote_code) - or (cfg.transformers_config and cfg.transformers_config.trust_remote_code) - ) + trust_remote_code = bool(cfg.vllm_engine_kwargs and cfg.vllm_engine_kwargs.trust_remote_code) logger.info("Resolving model source for '%s': %s", cfg.name, cfg.model) cfg._resolved_path = resolve_model_source(cfg.model, trust_remote_code=trust_remote_code) logger.info("Resolved '%s' -> %s", cfg.name, cfg._resolved_path) @@ -176,9 +170,9 @@ def resolve_all_tool_parsers(yml_conf: ModelshipConfig) -> None: onto `_resolved_tool_call_parser` so loader code has a single source of truth and never re-implements the precedence. - Auto-detection runs for vllm and transformers. llama_server does its own - tool-call detection internally, unrelated to this registry; diffusers has - no chat path; custom is plugin-managed. + Auto-detection runs for vllm. llama_server does its own tool-call + detection internally, unrelated to this registry; diffusers has no chat + path; custom is plugin-managed. Behavior per model: - Loader-specific opt-out (see `_is_explicit_tool_opt_out`): leaves @@ -191,7 +185,7 @@ def resolve_all_tool_parsers(yml_conf: ModelshipConfig) -> None: """ registered = set(available_parsers()) for cfg in yml_conf.models: - if cfg.loader not in (ModelLoader.vllm, ModelLoader.transformers): + if cfg.loader != ModelLoader.vllm: continue if cfg.usecase != ModelUsecase.generate: continue @@ -199,11 +193,7 @@ def resolve_all_tool_parsers(yml_conf: ModelshipConfig) -> None: logger.info("Tool-call resolution skipped for '%s' (explicit opt-out).", cfg.name) continue - explicit = None - if cfg.loader == ModelLoader.transformers and cfg.transformers_config: - explicit = cfg.transformers_config.tool_call_parser - elif cfg.loader == ModelLoader.vllm and cfg.vllm_engine_kwargs: - explicit = cfg.vllm_engine_kwargs.tool_call_parser + explicit = cfg.vllm_engine_kwargs.tool_call_parser if cfg.vllm_engine_kwargs else None if explicit is not None: if explicit not in registered: @@ -291,7 +281,7 @@ def resolve_all_reasoning_parsers(yml_conf: ModelshipConfig) -> None: - Not detected: leaves None (reasoning disabled). """ for cfg in yml_conf.models: - if cfg.loader not in (ModelLoader.vllm, ModelLoader.transformers): + if cfg.loader != ModelLoader.vllm: continue if cfg.usecase != ModelUsecase.generate: continue diff --git a/modelship/infer/infer_config.py b/modelship/infer/infer_config.py index 44bff3b..b0b06a0 100644 --- a/modelship/infer/infer_config.py +++ b/modelship/infer/infer_config.py @@ -47,7 +47,6 @@ class ModelUsecase(StrEnum): class ModelLoader(StrEnum): vllm = "vllm" - transformers = "transformers" diffusers = "diffusers" llama_server = "llama_server" stable_diffusion_cpp = "stable_diffusion_cpp" @@ -84,18 +83,6 @@ class VllmEngineConfig(BaseModel): mm_processor_kwargs: dict[str, Any] | None = None -class TransformersConfig(BaseModel): - device: str = "cpu" - torch_dtype: str = "auto" - trust_remote_code: bool = False - model_kwargs: dict[str, Any] = Field(default_factory=dict) - pipeline_kwargs: dict[str, Any] = Field(default_factory=dict) - tool_call_parser: str | None = None - # Explicit opt-out from auto-detected tool calling. None -> auto-detect; False -> disabled - # even if the model's chat template advertises tools; True is a no-op (auto runs anyway). - tool_calls_enabled: bool | None = None - - class DiffusersConfig(BaseModel): torch_dtype: str = "float16" num_inference_steps: int = 30 @@ -215,7 +202,6 @@ class ModelshipModelConfig(BaseModel): # Ray Serve's per-replica concurrency cap. max_ongoing_requests: int | None = None vllm_engine_kwargs: VllmEngineConfig = Field(default_factory=VllmEngineConfig) - transformers_config: TransformersConfig | None = None diffusers_config: DiffusersConfig | None = None llama_server_config: LlamaServerConfig | None = None stable_diffusion_cpp_config: StableDiffusionCppConfig | None = None @@ -231,8 +217,8 @@ class ModelshipModelConfig(BaseModel): _resolved_chat_template: str | None = PrivateAttr(default=None) # Pinned at startup from the resolved tool-call parser's # ``markers_are_specials`` flag. Loaders that detokenize raw model - # output (transformers' ``TextIteratorStreamer``) consult this to - # decide whether to flip ``skip_special_tokens=False`` — required for + # output consult this to decide whether to flip + # ``skip_special_tokens=False`` — required for # parsers like Mistral whose ``[TOOL_CALLS]`` marker is registered as a # special token in the tokenizer and would otherwise be stripped before # the parser sees it. ``None`` means the loader should keep its own diff --git a/modelship/infer/model_deployment.py b/modelship/infer/model_deployment.py index 5f6850a..f82fdc4 100644 --- a/modelship/infer/model_deployment.py +++ b/modelship/infer/model_deployment.py @@ -157,10 +157,6 @@ async def __init__(self, config: ModelshipModelConfig): from modelship.infer.vllm.vllm_infer import VllmInfer self.infer = VllmInfer(config) - elif config.loader == ModelLoader.transformers: - from modelship.infer.transformers.transformers_infer import TransformersInfer - - self.infer = TransformersInfer(config) elif config.loader == ModelLoader.diffusers: from modelship.infer.diffusers.diffusers_infer import DiffusersInfer diff --git a/modelship/infer/transformers/capabilities.py b/modelship/infer/transformers/capabilities.py deleted file mode 100644 index 5639f05..0000000 --- a/modelship/infer/transformers/capabilities.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Modality capability detection for a HuggingFace ``Pipeline`` instance.""" - -from dataclasses import dataclass - -from transformers import Pipeline - -# HF pipeline tasks whose ``__call__`` accepts image inputs alongside chat messages. -_IMAGE_TASKS = frozenset({"image-text-to-text", "visual-question-answering"}) - - -@dataclass(frozen=True) -class TransformersCapabilities: - """Modalities the underlying HF pipeline can ingest.""" - - supports_image: bool - supports_audio: bool = False # no audio-aware chat pipelines today - - @classmethod - def detect(cls, pipeline: Pipeline) -> "TransformersCapabilities": - # The HF pipeline's ``task`` attribute is the authoritative signal: a - # text-generation pipeline cannot ingest image_url parts even if the - # underlying weights are a VLM. Multimodal chat needs an explicit task - # like "image-text-to-text". - task = getattr(pipeline, "task", None) - return cls(supports_image=task in _IMAGE_TASKS) diff --git a/modelship/infer/transformers/openai/serving_chat.py b/modelship/infer/transformers/openai/serving_chat.py deleted file mode 100644 index 17df961..0000000 --- a/modelship/infer/transformers/openai/serving_chat.py +++ /dev/null @@ -1,349 +0,0 @@ -import asyncio -import time -from collections.abc import AsyncGenerator, AsyncIterator -from threading import Thread -from typing import Any - -from transformers import Pipeline, PreTrainedTokenizerBase, TextIteratorStreamer - -from modelship.infer.base_serving import OpenAIServing -from modelship.infer.infer_config import RawRequestProxy, TransformersConfig -from modelship.infer.transformers.capabilities import TransformersCapabilities -from modelship.logging import TRACE, get_logger -from modelship.openai.chat_utils import UnsupportedContentError, normalize_chat_messages -from modelship.openai.parsers.reasoning import get_parser as get_reasoning_parser -from modelship.openai.parsers.streaming import build_chat_completion_response, stream_chat_completion -from modelship.openai.parsers.tool_calling import get_parser, request_forces_tool_call, resolve_tools_for_request -from modelship.openai.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, - ErrorResponse, - create_error_response, -) -from modelship.utils import base_request_id, drop_reserved_kwargs - -logger = get_logger("infer.transformers.chat") - - -class OpenAIServingChat(OpenAIServing): - request_id_prefix = "chatcmpl" - - def __init__( - self, - pipeline: Pipeline, - model_name: str, - config: TransformersConfig, - capabilities: TransformersCapabilities, - tool_call_parser: str | None = None, - reasoning_parser: str | None = None, - skip_special_tokens: bool | None = None, - chat_template_kwargs: dict[str, Any] | None = None, - ): - self.pipeline = pipeline - self.model_name = model_name - self.config = config - self.capabilities = capabilities - self.tool_call_parser = tool_call_parser - self.reasoning_parser = reasoning_parser - # Drop keys we pass to apply_chat_template ourselves — a collision would - # crash _render_prompt (duplicate kwarg) or flip tokenize in the counter. - # `conversation` is apply_chat_template's first positional arg (passed - # positionally here); `messages` is the same data inside the template - # context. The rest are keyword args we set ourselves. - self.chat_template_kwargs = drop_reserved_kwargs( - chat_template_kwargs or {}, - {"conversation", "messages", "tokenize", "tools", "add_generation_prompt"}, - logger=logger, - context=f"model '{model_name}'", - ) - assert pipeline.tokenizer is not None, "text-generation pipeline must have a tokenizer" - self.tokenizer: PreTrainedTokenizerBase = pipeline.tokenizer - self._lock = asyncio.Lock() - self._logged_force_unenforceable = False - # Validate configured parsers at startup so misconfiguration surfaces - # before the first request rather than mid-generation. None means - # auto-detection found no usable parser; tool calls in requests will be - # dropped with a warning at request time, and reasoning will simply - # not be extracted. - tool_parser = get_parser(tool_call_parser) if tool_call_parser is not None else None - reasoning = get_reasoning_parser(reasoning_parser) if reasoning_parser is not None else None - # Resolve the streamer's ``skip_special_tokens`` setting. Default - # ``True`` matches HF's example code and is correct for parsers - # whose markers are regular text (Hermes ````, - # llama3_json ``{"name"``). For parsers whose markers are - # registered specials (Mistral ``[TOOL_CALLS]``), the resolver - # passes ``False`` so the marker survives detokenization, and we - # noise-strip every OTHER registered special from each chunk - # ourselves so clients never see ``<|im_end|>`` / - # ``<|eot_id|>`` / etc. leak into content. Reasoning-parser - # markers go in the keep set too — a family could register - # ````/```` as specials, and the reasoning parser - # needs to see them. - self._skip_special_tokens = True if skip_special_tokens is None else skip_special_tokens - self._noise_specials: tuple[str, ...] = () - if not self._skip_special_tokens: - keep = { - m - for m in ( - tool_parser.start_marker if tool_parser else "", - tool_parser.end_marker if tool_parser else "", - reasoning.start_marker if reasoning else "", - reasoning.end_marker if reasoning else "", - ) - if m - } - # ``added_tokens_decoder`` is the authoritative list of every - # token (special + ordinary) registered on the tokenizer, with - # a per-entry ``special`` flag. We strip the ``special=True`` - # entries that are NOT in the keep set. Sort by length - # descending so a longer marker (e.g. ``<|start_header_id|>``) - # is replaced before a substring of it could cause false - # partial matches. Use ``getattr`` because some non-fast or - # custom tokenizers may not expose this attribute — in that - # case noise stripping silently degrades to a no-op rather - # than blowing up at startup. - added_tokens = getattr(self.tokenizer, "added_tokens_decoder", {}) or {} - self._noise_specials = tuple( - sorted( - ( - tok.content - for tok in added_tokens.values() - if tok.special and tok.content and tok.content not in keep - ), - key=len, - reverse=True, - ) - ) - - async def warmup(self) -> None: - logger.info("Warming up chat model: %s", self.model_name) - await self.run_in_executor( - self._run, - [{"role": "user", "content": "warmup"}], - None, - 1, - ) - logger.info("Warmup chat done for %s", self.model_name) - - async def create_chat_completion( - self, request: ChatCompletionRequest, raw_request: RawRequestProxy - ) -> ChatCompletionResponse | AsyncGenerator[str, None] | ErrorResponse: - request_id = f"{self.request_id_prefix}-{base_request_id(raw_request)}" - logger.info("chat completion request %s: stream=%s", request_id, request.stream) - logger.log( - TRACE, "chat request %s: messages=%s, max_tokens=%s", request_id, request.messages, request.max_tokens - ) - - try: - messages = normalize_chat_messages( - request.messages, - supports_image=self.capabilities.supports_image, - supports_audio=self.capabilities.supports_audio, - ) - except UnsupportedContentError as e: - logger.warning("chat request %s rejected: %s", request_id, e) - return create_error_response(e) - - tools = resolve_tools_for_request(request.tools, request.tool_choice) - if tools and self.tool_call_parser is None: - logger.warning( - "chat request %s asks for %d tool(s) but model %r has no usable tool-call parser; ignoring tools", - request_id, - len(tools), - self.model_name, - ) - tools = None - if tools and request_forces_tool_call(request.tool_choice) and not self._logged_force_unenforceable: - self._logged_force_unenforceable = True - logger.info( - "tool_choice forces a tool call but the transformers loader has no constrained decoding; " - "passing tools to the model and trusting it to comply (best-effort)" - ) - - max_tokens = request.max_tokens - if max_tokens is None and request.max_completion_tokens is not None: - max_tokens = request.max_completion_tokens - - parser_name = self.tool_call_parser if tools else None - reasoning_parser_name = self.reasoning_parser - - if request.stream: - include_usage = bool(request.stream_options and request.stream_options.include_usage) - return self._locked_stream( - request_id, - messages, - tools, - max_tokens, - parser_name=parser_name, - reasoning_parser_name=reasoning_parser_name, - include_usage=include_usage, - ) - - async with self._lock: - try: - result = await self.run_in_executor(self._run, messages, tools, max_tokens) - except Exception: - logger.exception("chat completion inference failed for %s", request_id) - return create_error_response("chat completion inference failed") - - prompt_tokens = self._count_prompt_tokens(messages, tools) - completion_text = self._extract_completion_text(result) - completion_tokens = len(self.tokenizer.encode(completion_text, add_special_tokens=False)) - - response = build_chat_completion_response( - request_id=request_id, - model_name=self.model_name, - text=completion_text, - parser_name=parser_name, - reasoning_parser_name=reasoning_parser_name, - noise_specials=self._noise_specials, - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - max_tokens=max_tokens, - created=int(time.time()), - ) - logger.log( - TRACE, - "chat response %s: text=%r, tool_calls=%d, prompt_tokens=%d, completion_tokens=%d", - request_id, - completion_text, - len(response.choices[0].message.tool_calls), - prompt_tokens, - completion_tokens, - ) - return response - - @staticmethod - def _extract_completion_text(result: list) -> str: - generated = result[0]["generated_text"] - if isinstance(generated, list): - return generated[-1]["content"] - return generated - - def _count_prompt_tokens(self, messages: list[dict], tools: list[dict[str, Any]] | None) -> int: - # apply_chat_template returns a string by default (character count!) — force tokenize=True. - kwargs: dict[str, Any] = {"tokenize": True, **self.chat_template_kwargs} - if tools: - kwargs["tools"] = tools - token_ids = self.tokenizer.apply_chat_template(messages, **kwargs) - return len(token_ids) - - def _render_prompt(self, messages: list[dict], tools: list[dict[str, Any]] | None = None) -> str: - rendered = self.tokenizer.apply_chat_template( - messages, - tools=tools, # type: ignore[arg-type] - tokenize=False, - add_generation_prompt=True, - **self.chat_template_kwargs, - ) - assert isinstance(rendered, str), "apply_chat_template(tokenize=False) must return str" - return rendered - - def _run(self, messages: list[dict], tools: list[dict[str, Any]] | None, max_tokens: int | None) -> list: - kwargs = {**self.config.pipeline_kwargs} - if max_tokens is not None: - kwargs["max_new_tokens"] = max_tokens - # Mirror the streamer's setting on the non-streaming path so the - # pipeline's internal detokenize doesn't pre-strip the parser's - # marker (the pipeline defaults ``skip_special_tokens=True``). - if not self._skip_special_tokens: - kwargs["skip_special_tokens"] = False - # The pipeline applies the chat template internally and forwards neither - # `tools` nor our `chat_template_kwargs`. Render ourselves whenever either - # is in play so both reach `apply_chat_template`; otherwise let the - # pipeline handle the plain message list. - if tools or self.chat_template_kwargs: - prompt = self._render_prompt(messages, tools) - return self.pipeline(prompt, return_full_text=False, **kwargs) # type: ignore[return-value] - return self.pipeline(messages, return_full_text=False, **kwargs) # type: ignore[return-value] - - async def _locked_stream( - self, - request_id: str, - messages: list[dict], - tools: list[dict[str, Any]] | None, - max_tokens: int | None, - *, - parser_name: str | None, - reasoning_parser_name: str | None, - include_usage: bool, - ) -> AsyncGenerator[str, None]: - async with self._lock: - async for chunk in self._stream( - request_id, - messages, - tools, - max_tokens, - parser_name=parser_name, - reasoning_parser_name=reasoning_parser_name, - include_usage=include_usage, - ): - yield chunk - - async def _stream( - self, - request_id: str, - messages: list[dict], - tools: list[dict[str, Any]] | None, - max_tokens: int | None, - *, - parser_name: str | None, - reasoning_parser_name: str | None, - include_usage: bool, - ) -> AsyncGenerator[str, None]: - streamer = TextIteratorStreamer( # type: ignore[arg-type] - self.tokenizer, - skip_prompt=True, - skip_special_tokens=self._skip_special_tokens, - ) - - kwargs = {**self.config.pipeline_kwargs} - if max_tokens is not None: - kwargs["max_new_tokens"] = max_tokens - - # Mirror _run: render ourselves whenever tools or chat_template_kwargs are - # set, since the pipeline forwards neither to apply_chat_template. - if tools or self.chat_template_kwargs: - prompt: Any = self._render_prompt(messages, tools) - else: - prompt = messages - - thread = Thread( - target=self.pipeline, - args=(prompt,), - kwargs={"streamer": streamer, "return_full_text": False, **kwargs}, - ) - thread.start() - - try: - async for chunk in stream_chat_completion( - request_id=request_id, - model_name=self.model_name, - text_chunks=_async_iter(streamer), - parser_name=parser_name, - noise_specials=self._noise_specials, - reasoning_parser_name=reasoning_parser_name, - count_tokens=lambda text: len(self.tokenizer.encode(text, add_special_tokens=False)), - prompt_tokens=self._count_prompt_tokens(messages, tools), - max_tokens=max_tokens, - include_usage=include_usage, - created=int(time.time()), - ): - yield chunk - finally: - thread.join() - - -async def _async_iter(streamer: TextIteratorStreamer) -> AsyncIterator[str]: - """Adapt a synchronous ``TextIteratorStreamer`` to an async iterator. - - HF's streamer is driven by a background thread and exposes a blocking - ``__next__``. We hop to a thread per pull so the event loop keeps spinning - (cancellation, request-watcher polls) while tokens are produced. - """ - sentinel = object() - while True: - item = await asyncio.to_thread(next, streamer, sentinel) - if item is sentinel: - return - yield item # type: ignore[misc] diff --git a/modelship/infer/transformers/openai/serving_embedding.py b/modelship/infer/transformers/openai/serving_embedding.py deleted file mode 100644 index d1e3ac9..0000000 --- a/modelship/infer/transformers/openai/serving_embedding.py +++ /dev/null @@ -1,67 +0,0 @@ -import time - -from sentence_transformers import SentenceTransformer - -from modelship.infer.base_serving import OpenAIServing -from modelship.infer.infer_config import RawRequestProxy -from modelship.logging import TRACE, get_logger -from modelship.openai.protocol import ( - EmbeddingRequest, - EmbeddingResponse, - EmbeddingResponseData, - ErrorResponse, - UsageInfo, - create_error_response, -) -from modelship.utils import base_request_id - -logger = get_logger("infer.transformers.embedding") - - -class OpenAIServingEmbedding(OpenAIServing): - request_id_prefix = "embd" - - def __init__(self, model: SentenceTransformer, model_name: str): - self.model = model - self.model_name = model_name - - async def warmup(self) -> None: - logger.info("Warming up embedding model: %s", self.model_name) - await self.run_in_executor(self._run, ["warmup"]) - logger.info("Warmup embedding done for %s", self.model_name) - - def _run(self, inputs: list[str]): - return self.model.encode(inputs) - - async def create_embedding( - self, request: EmbeddingRequest, raw_request: RawRequestProxy - ) -> EmbeddingResponse | ErrorResponse: - request_id = f"{self.request_id_prefix}-{base_request_id(raw_request)}" - logger.info("embedding request %s", request_id) - logger.log(TRACE, "embedding request %s: input=%s", request_id, request.input) # type: ignore[union-attr] - - inputs = request.input # type: ignore[union-attr] - if isinstance(inputs, str): - inputs = [inputs] - - try: - embeddings = await self.run_in_executor(self._run, inputs) - except Exception: - logger.exception("embedding inference failed for %s", request_id) - return create_error_response("embedding inference failed") - - tokenized = self.model.tokenize(inputs) # type: ignore[arg-type] - prompt_tokens = sum(len(ids) for ids in tokenized["input_ids"]) - - data = [EmbeddingResponseData(index=i, embedding=emb.tolist()) for i, emb in enumerate(embeddings)] - - response = EmbeddingResponse( - model=self.model_name, - data=data, - usage=UsageInfo(prompt_tokens=prompt_tokens, total_tokens=prompt_tokens), - created=int(time.time()), - ) - logger.log( - TRACE, "embedding response %s: num_embeddings=%d, prompt_tokens=%d", request_id, len(data), prompt_tokens - ) - return response diff --git a/modelship/infer/transformers/openai/serving_speech.py b/modelship/infer/transformers/openai/serving_speech.py deleted file mode 100644 index f381910..0000000 --- a/modelship/infer/transformers/openai/serving_speech.py +++ /dev/null @@ -1,55 +0,0 @@ -from transformers import Pipeline - -from modelship.infer.base_serving import OpenAIServing -from modelship.infer.infer_config import RawRequestProxy, TransformersConfig -from modelship.logging import TRACE, get_logger -from modelship.openai.protocol import ( - ErrorResponse, - RawSpeechResponse, - SpeechRequest, - create_error_response, -) -from modelship.utils import base_request_id -from modelship.utils.audio import to_wav - -logger = get_logger("infer.transformers.speech") - - -class OpenAIServingSpeech(OpenAIServing): - request_id_prefix = "tts" - - def __init__(self, pipeline: Pipeline, model_name: str, config: TransformersConfig): - self.pipeline = pipeline - self.model_name = model_name - self.config = config - - async def warmup(self) -> None: - logger.info("Warming up TTS model: %s", self.model_name) - await self.run_in_executor(self._run, "warmup") - logger.info("Warmup TTS done for %s", self.model_name) - - async def create_speech( - self, request: SpeechRequest, raw_request: RawRequestProxy - ) -> RawSpeechResponse | ErrorResponse: - request_id = f"{self.request_id_prefix}-{base_request_id(raw_request)}" - logger.info("speech request %s", request_id) - logger.log(TRACE, "speech request %s: input=%r", request_id, request.input) - - try: - result = await self.run_in_executor(self._run, request.input) - except Exception: - logger.exception("speech inference failed for %s", request_id) - return create_error_response("speech inference failed") - - wav_bytes = to_wav(result["audio"], result["sampling_rate"]) - logger.log( - TRACE, - "speech response %s: audio_bytes=%d, sample_rate=%d", - request_id, - len(wav_bytes), - result["sampling_rate"], - ) - return RawSpeechResponse(audio=wav_bytes) - - def _run(self, text: str) -> dict: - return self.pipeline(text, **self.config.pipeline_kwargs) # type: ignore[return-value] diff --git a/modelship/infer/transformers/openai/serving_transcription.py b/modelship/infer/transformers/openai/serving_transcription.py deleted file mode 100644 index 746a059..0000000 --- a/modelship/infer/transformers/openai/serving_transcription.py +++ /dev/null @@ -1,122 +0,0 @@ -from transformers import Pipeline - -from modelship.infer.base_infer import MINIMAL_WAV -from modelship.infer.base_serving import OpenAIServing -from modelship.infer.infer_config import RawRequestProxy, TransformersConfig -from modelship.logging import TRACE, get_logger -from modelship.openai.protocol import ( - ErrorResponse, - TranscriptionRequest, - TranscriptionResponse, - TranscriptionResponseVerbose, - TranscriptionUsageAudio, - TranslationRequest, - TranslationResponse, - TranslationResponseVerbose, - create_error_response, -) -from modelship.utils import base_request_id -from modelship.utils.audio import decode_audio - - -def _pipeline_input(audio_data: bytes, target_sr: int) -> tuple[dict, int]: - samples, duration_seconds = decode_audio(audio_data, target_sr) - return {"raw": samples, "sampling_rate": target_sr}, duration_seconds - - -logger = get_logger("infer.transformers.transcription") - - -class OpenAIServingTranscription(OpenAIServing): - request_id_prefix = "asr" - - def __init__(self, pipeline: Pipeline, model_name: str, config: TransformersConfig): - self.pipeline = pipeline - self.model_name = model_name - self.config = config - - @property - def _target_sr(self) -> int: - return self.pipeline.feature_extractor.sampling_rate # type: ignore[union-attr] - - async def warmup(self) -> None: - logger.info("Warming up transcription model: %s", self.model_name) - audio_input, _ = _pipeline_input(MINIMAL_WAV, self._target_sr) - await self.run_in_executor(self._run, audio_input, None) - logger.info("Warmup transcription done for %s", self.model_name) - - async def create_transcription( - self, audio_data: bytes, request: TranscriptionRequest, raw_request: RawRequestProxy - ) -> TranscriptionResponse | TranscriptionResponseVerbose | ErrorResponse: - request_id = f"{self.request_id_prefix}-{base_request_id(raw_request)}" - language = request.language if hasattr(request, "language") else None - logger.info("transcription request %s", request_id) - logger.log( - TRACE, "transcription request %s: language=%s, audio_bytes=%d", request_id, language, len(audio_data) - ) - - try: - audio_input, duration_seconds = _pipeline_input(audio_data, self._target_sr) - result = await self.run_in_executor(self._run, audio_input, language) - except Exception: - logger.exception("transcription inference failed for %s", request_id) - return create_error_response("transcription inference failed") - - text = result["text"].strip() - logger.log(TRACE, "transcription response %s: text=%r, duration=%ds", request_id, text, duration_seconds) - - return TranscriptionResponse( - text=text, - usage=TranscriptionUsageAudio(seconds=duration_seconds), - ) - - def _run(self, audio_input: dict, language: str | None = None) -> dict: - kwargs = {**self.config.pipeline_kwargs} - if language: - kwargs.setdefault("generate_kwargs", {}) - kwargs["generate_kwargs"]["language"] = language - return self.pipeline(audio_input, **kwargs) # type: ignore[return-value] - - -class OpenAIServingTranslation(OpenAIServing): - request_id_prefix = "translate" - - def __init__(self, pipeline: Pipeline, model_name: str, config: TransformersConfig): - self.pipeline = pipeline - self.model_name = model_name - self.config = config - - @property - def _target_sr(self) -> int: - return self.pipeline.feature_extractor.sampling_rate # type: ignore[union-attr] - - async def warmup(self) -> None: - logger.info("Warming up translation model: %s", self.model_name) - audio_input, _ = _pipeline_input(MINIMAL_WAV, self._target_sr) - await self.run_in_executor(self._run, audio_input) - logger.info("Warmup translation done for %s", self.model_name) - - async def create_translation( - self, audio_data: bytes, request: TranslationRequest, raw_request: RawRequestProxy - ) -> TranslationResponse | TranslationResponseVerbose | ErrorResponse: - request_id = f"{self.request_id_prefix}-{base_request_id(raw_request)}" - logger.info("translation request %s", request_id) - logger.log(TRACE, "translation request %s: audio_bytes=%d", request_id, len(audio_data)) - - try: - audio_input, _ = _pipeline_input(audio_data, self._target_sr) - result = await self.run_in_executor(self._run, audio_input) - except Exception: - logger.exception("translation inference failed for %s", request_id) - return create_error_response("translation inference failed") - - text = result["text"].strip() - logger.log(TRACE, "translation response %s: text=%r", request_id, text) - - return TranslationResponse(text=text) - - def _run(self, audio_input: dict) -> dict: - kwargs = {**self.config.pipeline_kwargs} - kwargs.setdefault("generate_kwargs", {}) - kwargs["generate_kwargs"]["task"] = "translate" - return self.pipeline(audio_input, **kwargs) # type: ignore[return-value] diff --git a/modelship/infer/transformers/transformers_infer.py b/modelship/infer/transformers/transformers_infer.py deleted file mode 100644 index 3333f97..0000000 --- a/modelship/infer/transformers/transformers_infer.py +++ /dev/null @@ -1,205 +0,0 @@ -from collections.abc import AsyncGenerator - -import torch - -from modelship.infer.base_infer import BaseInfer -from modelship.infer.base_serving import OpenAIServing -from modelship.infer.infer_config import ModelshipModelConfig, ModelUsecase, RawRequestProxy, TransformersConfig -from modelship.logging import get_logger -from modelship.openai.parsers.reasoning import resolve_active_reasoning_parser -from modelship.openai.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, - EmbeddingRequest, - EmbeddingResponse, - ErrorResponse, - RawSpeechResponse, - SpeechRequest, - TranscriptionRequest, - TranscriptionResponse, - TranscriptionResponseVerbose, - TranslationRequest, - TranslationResponse, - TranslationResponseVerbose, -) - -logger = get_logger("infer.transformers") - -_TORCH_DTYPES = { - "float16": torch.float16, - "float32": torch.float32, - "bfloat16": torch.bfloat16, -} - - -class TransformersInfer(BaseInfer): - def __init__(self, model_config: ModelshipModelConfig): - super().__init__(model_config) - self.config = model_config.transformers_config or TransformersConfig() - self.device = self.config.device - self.torch_dtype = _TORCH_DTYPES.get(self.config.torch_dtype) if self.config.torch_dtype != "auto" else "auto" - - mem_frac = self._get_memory_fraction() - if torch.cuda.is_available() and mem_frac is not None: - torch.cuda.set_per_process_memory_fraction(mem_frac) - - self._serving: list[OpenAIServing] = [] - - def shutdown(self) -> None: - pass - - def __del__(self): - try: - for serving in getattr(self, "_serving", []): - del serving - if torch.cuda.is_available(): - torch.cuda.empty_cache() - except Exception: - from modelship.metrics import RESOURCE_CLEANUP_ERRORS_TOTAL - - RESOURCE_CLEANUP_ERRORS_TOTAL.inc(tags={"model": self.model_config.name, "component": "transformers_model"}) - - async def start(self): - usecase = self.model_config.usecase - model_name = self.model_config._resolved_path - if not model_name: - raise ValueError( - f"Transformers deployment '{self.model_config.name}' is missing a resolved model path. " - f"Check driver logs for resolution errors." - ) - - if usecase is ModelUsecase.generate: - from transformers import pipeline - - from modelship.infer.transformers.capabilities import TransformersCapabilities - from modelship.infer.transformers.openai.serving_chat import OpenAIServingChat - - pipe = pipeline( - "text-generation", - model=model_name, - device=self.device, - torch_dtype=self.torch_dtype, - trust_remote_code=self.config.trust_remote_code, - model_kwargs=self.config.model_kwargs, - ) - capabilities = TransformersCapabilities.detect(pipe) - if capabilities.supports_image: - logger.info("Multimodal (vision) capability detected for model: %s", self.model_config.name) - # `resolve_all_tool_parsers` / `resolve_all_reasoning_parsers` - # already honored loader-specific opt-outs (leaving the resolved - # field None) and captured explicit overrides. Read both directly. - tool_call_parser = self.model_config._resolved_tool_call_parser - reasoning_parser = self.model_config._resolved_reasoning_parser - skip_special_tokens = self.model_config._resolved_skip_special_tokens - self.serving_chat = OpenAIServingChat( - pipe, - self.model_config.name, - self.config, - capabilities, - tool_call_parser=tool_call_parser, - reasoning_parser=reasoning_parser, - skip_special_tokens=skip_special_tokens, - chat_template_kwargs=self.model_config.chat_template_kwargs, - ) - # Confirm the capability-detected reasoning parser against the real render: - # a deployment that suppressed reasoning via chat_template_kwargs downgrades - # to None. Rendered through the same path real requests use. - self.serving_chat.reasoning_parser = resolve_active_reasoning_parser( - reasoning_parser, - lambda: self.serving_chat._render_prompt([{"role": "user", "content": "hi"}]), - ) - self._serving.append(self.serving_chat) - - elif usecase is ModelUsecase.embed: - from sentence_transformers import SentenceTransformer - - from modelship.infer.transformers.openai.serving_embedding import OpenAIServingEmbedding - - model = SentenceTransformer( - model_name, - device=self.device, - trust_remote_code=self.config.trust_remote_code, - model_kwargs=self.config.model_kwargs, - ) - self.serving_embedding = OpenAIServingEmbedding(model, self.model_config.name) - self._serving.append(self.serving_embedding) - - elif usecase in (ModelUsecase.transcription, ModelUsecase.translation): - from transformers import pipeline - - from modelship.infer.transformers.openai.serving_transcription import ( - OpenAIServingTranscription, - OpenAIServingTranslation, - ) - - pipe = pipeline( - "automatic-speech-recognition", - model=model_name, - device=self.device, - torch_dtype=self.torch_dtype, - trust_remote_code=self.config.trust_remote_code, - model_kwargs=self.config.model_kwargs, - ) - self.serving_transcription = OpenAIServingTranscription(pipe, self.model_config.name, self.config) - self.serving_translation = OpenAIServingTranslation(pipe, self.model_config.name, self.config) - self._serving.append(self.serving_transcription) - self._serving.append(self.serving_translation) - - elif usecase is ModelUsecase.tts: - from transformers import pipeline - - from modelship.infer.transformers.openai.serving_speech import OpenAIServingSpeech - - pipe = pipeline( - "text-to-audio", - model=model_name, - device=self.device, - torch_dtype=self.torch_dtype, - trust_remote_code=self.config.trust_remote_code, - model_kwargs=self.config.model_kwargs, - ) - self.serving_speech = OpenAIServingSpeech(pipe, self.model_config.name, self.config) - self._serving.append(self.serving_speech) - - logger.info( - "TransformersInfer started for %s (usecase=%s, device=%s)", self.model_config.name, usecase, self.device - ) - - async def warmup(self) -> None: - for serving in self._serving: - await serving.warmup() - - async def create_chat_completion( - self, request: ChatCompletionRequest, raw_request: RawRequestProxy - ) -> ErrorResponse | ChatCompletionResponse | AsyncGenerator[str, None]: - if not hasattr(self, "serving_chat"): - return await super().create_chat_completion(request, raw_request) - return await self.serving_chat.create_chat_completion(request, raw_request) - - async def create_embedding( - self, request: EmbeddingRequest, raw_request: RawRequestProxy - ) -> EmbeddingResponse | ErrorResponse: - if not hasattr(self, "serving_embedding"): - return await super().create_embedding(request, raw_request) - return await self.serving_embedding.create_embedding(request, raw_request) - - async def create_transcription( - self, audio_data: bytes, request: TranscriptionRequest, raw_request: RawRequestProxy - ) -> ErrorResponse | TranscriptionResponse | TranscriptionResponseVerbose | AsyncGenerator[str, None]: - if not hasattr(self, "serving_transcription"): - return await super().create_transcription(audio_data, request, raw_request) - return await self.serving_transcription.create_transcription(audio_data, request, raw_request) - - async def create_translation( - self, audio_data: bytes, request: TranslationRequest, raw_request: RawRequestProxy - ) -> ErrorResponse | TranslationResponse | TranslationResponseVerbose | AsyncGenerator[str, None]: - if not hasattr(self, "serving_translation"): - return await super().create_translation(audio_data, request, raw_request) - return await self.serving_translation.create_translation(audio_data, request, raw_request) - - async def create_speech( - self, request: SpeechRequest, raw_request: RawRequestProxy - ) -> ErrorResponse | RawSpeechResponse | AsyncGenerator[str, None]: - if not hasattr(self, "serving_speech"): - return await super().create_speech(request, raw_request) - return await self.serving_speech.create_speech(request, raw_request) diff --git a/modelship/openai/parsers/tool_calling/parsers/base.py b/modelship/openai/parsers/tool_calling/parsers/base.py index 0100799..e98c854 100644 --- a/modelship/openai/parsers/tool_calling/parsers/base.py +++ b/modelship/openai/parsers/tool_calling/parsers/base.py @@ -48,9 +48,9 @@ class ToolCallParser(ABC): # (``added_tokens_decoder`` with ``special=True``). Loaders that decode # with ``skip_special_tokens=True`` by default would otherwise strip the # marker before this parser ever sees it, leaving the parser permanently - # idle. The transformers loader reads this flag at startup, flips its - # streamer's ``skip_special_tokens`` to ``False`` when set, and noise- - # strips every OTHER registered special from chunks itself. Hermes and + # idle. Loaders that detokenize raw output read this flag at startup, + # flip ``skip_special_tokens`` to ``False`` when set, and noise-strip + # every OTHER registered special from chunks themselves. Hermes and # llama3_json keep the default — their markers are regular text. markers_are_specials: bool = False # Delimiter token wrapping string values in custom-syntax (non-JSON) call diff --git a/modelship/openai/parsers/tool_calling/parsers/gemma.py b/modelship/openai/parsers/tool_calling/parsers/gemma.py index 959a688..e109991 100644 --- a/modelship/openai/parsers/tool_calling/parsers/gemma.py +++ b/modelship/openai/parsers/tool_calling/parsers/gemma.py @@ -136,7 +136,7 @@ class FunctionGemmaToolCallParser(Gemma4ToolCallParser): The envelope and string-delim tokens (````, ````, ````) are registered as special tokens on the FunctionGemma tokenizer, so ``markers_are_specials = True`` so - the transformers loader keeps them visible to the parser. + loaders that detokenize raw output keep them visible to the parser. """ name = "function_gemma" diff --git a/modelship/openai/protocol/chat.py b/modelship/openai/protocol/chat.py index df08fbe..fbdeec3 100644 --- a/modelship/openai/protocol/chat.py +++ b/modelship/openai/protocol/chat.py @@ -100,10 +100,9 @@ def _validate_tools_response_format_compat(self) -> "ChatCompletionRequest": # grammar that excludes any token outside the schema — including the # markers a model would use to emit a tool call (..., # <|python_tag|>..., etc.). The two cannot meaningfully coexist on any - # loader we support: vLLM passes both through but the grammar dominates; - # transformers has no native machinery to compose them. Reject upfront - # so callers don't discover the conflict by watching tool calls never - # fire in prod. + # loader we support: vLLM passes both through but the grammar dominates. + # Reject upfront so callers don't discover the conflict by watching + # tool calls never fire in prod. if not self.tools or not self.response_format: return self fmt_type = self.response_format.get("type") diff --git a/plugins/bark/README.md b/plugins/bark/README.md deleted file mode 100644 index e9e52cc..0000000 --- a/plugins/bark/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# Bark TTS Plugin - -Text-to-speech using [Bark](https://github.com/suno-ai/bark) by Suno. Supports multiple languages and speaker presets. - -## Installation - -For local development: - -```bash -uv sync --extra bark -``` - -For Docker, no extra setup is needed — plugins referenced in `models.yaml` are loaded automatically from pre-built wheels via Ray's `runtime_env`. - -## Configuration - -```yaml -models: - - name: bark - model: suno/bark - usecase: tts - loader: custom - plugin: bark - num_gpus: 0.30 -``` - -### plugin_config Options - -None. Bark has no plugin-specific configuration options. - -## Voices - -Voice presets follow the pattern `v2/_speaker_<0-9>`: - -| Language | Preset pattern | -|---|---| -| English | `v2/en_speaker_0` ... `v2/en_speaker_9` | -| Chinese | `v2/zh_speaker_0` ... `v2/zh_speaker_9` | -| French | `v2/fr_speaker_0` ... `v2/fr_speaker_9` | -| German | `v2/de_speaker_0` ... `v2/de_speaker_9` | -| Spanish | `v2/es_speaker_0` ... `v2/es_speaker_9` | -| Hindi | `v2/hi_speaker_0` ... `v2/hi_speaker_9` | -| Italian | `v2/it_speaker_0` ... `v2/it_speaker_9` | -| Japanese | `v2/ja_speaker_0` ... `v2/ja_speaker_9` | -| Korean | `v2/ko_speaker_0` ... `v2/ko_speaker_9` | -| Polish | `v2/pl_speaker_0` ... `v2/pl_speaker_9` | -| Portuguese | `v2/pt_speaker_0` ... `v2/pt_speaker_9` | -| Russian | `v2/ru_speaker_0` ... `v2/ru_speaker_9` | -| Turkish | `v2/tr_speaker_0` ... `v2/tr_speaker_9` | - -## Example Request - -```bash -curl http://localhost:8000/v1/audio/speech \ - -H "Content-Type: application/json" \ - -d '{"model": "bark", "input": "Hello world", "voice": "v2/en_speaker_6", "response_format": "wav"}' \ - --output speech.wav -``` - -Single-response mode only. SSE streaming is not supported. diff --git a/plugins/bark/bark/__init__.py b/plugins/bark/bark/__init__.py deleted file mode 100644 index 833684b..0000000 --- a/plugins/bark/bark/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bark.bark import ModelPlugin - -__all__ = ["ModelPlugin"] diff --git a/plugins/bark/bark/bark.py b/plugins/bark/bark/bark.py deleted file mode 100644 index 1af8c3a..0000000 --- a/plugins/bark/bark/bark.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Bark TTS plugin for Modelship. - -Voice presets follow the pattern: v2/_speaker_<0-9> - - English: v2/en_speaker_0 … v2/en_speaker_9 - Chinese: v2/zh_speaker_0 … v2/zh_speaker_9 - French: v2/fr_speaker_0 … v2/fr_speaker_9 - German: v2/de_speaker_0 … v2/de_speaker_9 - Spanish: v2/es_speaker_0 … v2/es_speaker_9 - Hindi: v2/hi_speaker_0 … v2/hi_speaker_9 - Italian: v2/it_speaker_0 … v2/it_speaker_9 - Japanese: v2/ja_speaker_0 … v2/ja_speaker_9 - Korean: v2/ko_speaker_0 … v2/ko_speaker_9 - Polish: v2/pl_speaker_0 … v2/pl_speaker_9 - Portuguese: v2/pt_speaker_0 … v2/pt_speaker_9 - Russian: v2/ru_speaker_0 … v2/ru_speaker_9 - Turkish: v2/tr_speaker_0 … v2/tr_speaker_9 - -Example request: - - curl http://localhost:8000/v1/audio/speech \\ - -H "Content-Type: application/json" \\ - -d '{"model": "bark", "input": "Hello world", "voice": "v2/en_speaker_6", "response_format": "wav"}' \\ - --output speech.wav -""" - -from collections.abc import AsyncGenerator - -import torch -from transformers import BarkModel, BarkProcessor - -from modelship.infer.infer_config import ModelshipModelConfig -from modelship.logging import get_logger -from modelship.openai.protocol import ErrorResponse, RawSpeechResponse, create_error_response -from modelship.plugins.base_plugin import BasePlugin -from modelship.utils.audio import to_wav - -logger = get_logger("plugin.bark") - - -class ModelPlugin(BasePlugin): - def __init__(self, model_config: ModelshipModelConfig): - self.device = "cuda:0" if torch.cuda.is_available() else "cpu" - self.model = BarkModel.from_pretrained(pretrained_model_name_or_path=model_config.model).to(self.device) # type: ignore[arg-type] - self.processor = BarkProcessor.from_pretrained(model_config.model) - - def __del__(self): - try: - if model := getattr(self, "model", None): - del model - self.model = None - if torch.cuda.is_available(): - torch.cuda.empty_cache() - except Exception: - pass - - async def start(self): - pass - - async def create_speech( - self, - input: str, - voice: str | None = None, - speed: float | None = None, - stream: bool = False, - request_id: str | None = None, - ) -> RawSpeechResponse | AsyncGenerator[tuple[bytes, int], None] | ErrorResponse: - logger.info("started generation: %s with voice: %s to device %s", input, voice, self.device) - - if stream: - return create_error_response("bark does not support streaming") - - inputs = self.processor(input, voice_preset=voice).to(device=self.device) - sample_rate = getattr(self.model.generation_config, "sample_rate", 24000) # type: ignore[union-attr] - speech_output = self.model.generate(**inputs).cpu().numpy().squeeze() # type: ignore[call-arg] - - return RawSpeechResponse(audio=to_wav(speech_output, sample_rate), media_type="audio/wav") diff --git a/plugins/bark/pyproject.toml b/plugins/bark/pyproject.toml deleted file mode 100644 index 92f323b..0000000 --- a/plugins/bark/pyproject.toml +++ /dev/null @@ -1,17 +0,0 @@ -[project] -name = "bark" -version = "0.1.0" -description = "Bark TTS plugin for Modelship" -requires-python = "==3.12.10" -dependencies = [] - -[build-system] -requires = ["uv_build"] -build-backend = "uv_build" - -[tool.uv.sources] -modelship = { workspace = true } - -[tool.uv.build-backend] -module-name = "bark" -module-root = "" diff --git a/pyproject.toml b/pyproject.toml index 6bbb002..013ab23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,11 +41,9 @@ gpu = [ "torch>=2.10.0", "torchvision>=0.25.0", "transformers>=5.5.3", - "sentence-transformers>=3.0.0", "flashinfer-python>=0.6.1", "vllm==0.24.0", "vllm[audio]==0.24.0", - "accelerate>=1.6.0", "bitsandbytes>=0.49.0", "diffusers>=0.31.0", "stable-diffusion-cpp-python>=0.4.7", @@ -56,14 +54,12 @@ cpu = [ "torch>=2.10.0", "torchvision>=0.25.0", "transformers>=5.5.3", - "sentence-transformers>=3.0.0", "stable-diffusion-cpp-python>=0.4.7", "onnxruntime>=1.20.1", "vllm==0.24.0", "vllm[audio]==0.24.0", ] kokoroonnx = ["kokoroonnx"] -bark = ["bark"] orpheus = ["orpheus"] whispercpp = ["whispercpp"] otel = [ @@ -112,7 +108,6 @@ explicit = true [tool.uv.sources] kokoroonnx = { workspace = true } -bark = { workspace = true } orpheus = { workspace = true } whispercpp = { workspace = true } torch = [ diff --git a/tests/test_config.py b/tests/test_config.py index 14ee135..1501505 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -10,7 +10,6 @@ ModelshipConfig, ModelshipModelConfig, ModelUsecase, - TransformersConfig, VllmEngineConfig, ) @@ -349,8 +348,8 @@ def test_num_gpus_redundant_with_tp_logs_warning(self, caplog): assert any("redundant" in rec.message for rec in caplog.records) def test_non_vllm_loader_skips_tp_derivation(self): - # transformers has no parallelism config; num_gpus stays as-is for the - # loader to interpret directly (whole GPUs are fine for that path). + # Non-vllm loaders have no parallelism config; num_gpus stays as-is + # for the loader to interpret directly. config = ModelshipModelConfig( name="test-tts", model="some-model", @@ -509,7 +508,8 @@ def test_unaffected_by_num_replicas(self): def test_changes_when_loader_differs(self): assert ( - self._cfg(loader=ModelLoader.vllm).fingerprint() != self._cfg(loader=ModelLoader.transformers).fingerprint() + self._cfg(loader=ModelLoader.vllm).fingerprint() + != self._cfg(loader=ModelLoader.llama_server, num_gpus=0).fingerprint() ) def test_deployment_name_combines_name_and_fingerprint(self): @@ -527,83 +527,6 @@ def test_fingerprint_distinct_per_gateway(self): assert cfg.fingerprint() == cfg.fingerprint("") -class TestTransformersConfig: - def test_defaults(self): - config = TransformersConfig() - assert config.device == "cpu" - assert config.torch_dtype == "auto" - assert config.trust_remote_code is False - assert config.model_kwargs == {} - assert config.pipeline_kwargs == {} - - def test_custom_values(self): - config = TransformersConfig( - device="cuda:0", - torch_dtype="float16", - trust_remote_code=True, - model_kwargs={"attn_implementation": "flash_attention_2"}, - ) - assert config.device == "cuda:0" - assert config.torch_dtype == "float16" - assert config.trust_remote_code is True - assert config.model_kwargs == {"attn_implementation": "flash_attention_2"} - - def test_transformers_generate_model(self): - config = ModelshipModelConfig( - name="llm-cpu", - model="meta-llama/Llama-3.2-1B-Instruct", - usecase=ModelUsecase.generate, - loader=ModelLoader.transformers, - num_cpus=4, - transformers_config=TransformersConfig(torch_dtype="float32"), - ) - assert config.loader == ModelLoader.transformers - assert config.usecase == ModelUsecase.generate - assert config.transformers_config.torch_dtype == "float32" - - def test_transformers_embed_model(self): - config = ModelshipModelConfig( - name="embed", - model="nomic-ai/nomic-embed-text-v1.5", - usecase=ModelUsecase.embed, - loader=ModelLoader.transformers, - num_cpus=2, - transformers_config=TransformersConfig(trust_remote_code=True), - ) - assert config.usecase == ModelUsecase.embed - assert config.transformers_config.trust_remote_code is True - - def test_transformers_transcription_model(self): - config = ModelshipModelConfig( - name="whisper-cpu", - model="openai/whisper-base", - usecase=ModelUsecase.transcription, - loader=ModelLoader.transformers, - num_cpus=2, - ) - assert config.usecase == ModelUsecase.transcription - assert config.transformers_config is None - - def test_transformers_tts_model(self): - config = ModelshipModelConfig( - name="tts", - model="microsoft/speecht5_tts", - usecase=ModelUsecase.tts, - loader=ModelLoader.transformers, - num_cpus=1, - ) - assert config.usecase == ModelUsecase.tts - - def test_transformers_config_not_required(self): - config = ModelshipModelConfig( - name="test", - model="some-model", - usecase=ModelUsecase.generate, - loader=ModelLoader.transformers, - ) - assert config.transformers_config is None - - class TestNumReplicas: def test_default_num_replicas(self): config = ModelshipModelConfig( diff --git a/tests/test_gemma4_parsers.py b/tests/test_gemma4_parsers.py index 88305a7..10d2a5b 100644 --- a/tests/test_gemma4_parsers.py +++ b/tests/test_gemma4_parsers.py @@ -106,9 +106,9 @@ def test_streaming_tool_call(self): def test_markers_are_specials_is_true(self): # FunctionGemma's envelope and string-delim tokens are registered - # specials on the tokenizer; the transformers loader keys off this - # flag to switch ``skip_special_tokens=False`` and keep the markers - # visible to the parser. + # specials on the tokenizer; loaders that detokenize raw output + # key off this flag to switch ``skip_special_tokens=False`` and + # keep the markers visible to the parser. assert FunctionGemmaToolCallParser.markers_are_specials is True def test_accepts_space_separator_instead_of_colon(self): diff --git a/tests/test_integration.py b/tests/test_integration.py index de56c47..1a2b67d 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -131,118 +131,32 @@ "name": "embed-model-llama-server", # Real embeddings through a live llama-server subprocess (`--embedding`) # — the existing `test_embeddings` integration test only exercises the - # transformers loader; llama_server's B4 embeddings support was - # otherwise only unit-tested against a mocked httpx transport. + # vllm loader; llama_server's B4 embeddings support was otherwise + # only unit-tested against a mocked httpx transport. "model": "nomic-ai/nomic-embed-text-v1.5-GGUF:nomic-embed-text-v1.5.Q4_K_M.gguf", "usecase": "embed", "loader": "llama_server", "num_cpus": 1, }, - "chat-transformers": { - "name": "chat-transformers", - "model": "Qwen/Qwen2.5-0.5B-Instruct", - "usecase": "generate", - "loader": "transformers", - "num_cpus": 2, - "transformers_config": { - "device": "cpu", - "torch_dtype": "float32", - }, - }, - "chat-transformers-llama3-json": { - "name": "chat-transformers-llama3-json", - # Llama-3.1-8B-Instruct emits the JSON-format tool call defined by - # Meta's spec — bare ``{"name": "...", "parameters": {...}}``. The - # chat template references ``<|python_tag|>``, so auto-detection - # resolves to ``llama3_json``. 8B is the smallest Llama-3.x - # variant Meta certifies for tool / function calling — Llama-3.2-1B - # has the same chat template but does not reliably emit the - # JSON-call shape on ``tool_choice="auto"``. Like the Mistral-7B - # transformers deployment, 8B in float32 is too memory-heavy for - # CPU+float32, so this is the second exception pinned to a full - # GPU (``device=cuda``, native bfloat16 via ``torch_dtype="auto"``). - # Requires ``HF_TOKEN`` for the gated repo. - "model": "meta-llama/Llama-3.1-8B-Instruct", - "usecase": "generate", - "loader": "transformers", - "num_cpus": 5, - "num_gpus": 1, - "transformers_config": { - "device": "cuda", - }, - }, - "chat-transformers-mistral": { - "name": "chat-transformers-mistral", - # Mistral-7B-Instruct-v0.3 emits ``[TOOL_CALLS]`` followed by a JSON - # array of tool calls. The marker is registered as a *special added - # token* on the tokenizer, so by default ``skip_special_tokens=True`` - # in ``TextIteratorStreamer`` would strip it before the parser sees - # it. The fix in this PR detects this case via - # ``markers_are_specials`` on the parser, pins - # ``_resolved_skip_special_tokens=False`` on the model config, and - # the transformers loader flips both ``TextIteratorStreamer`` and - # the non-streaming ``pipeline()`` call accordingly. - # - # 7B is the smallest official Mistral that ships the v3+ tool - # protocol; the rest of the transformers suite runs on CPU+float32 - # but a 7B model in float32 (~28GB) creates too much memory - # pressure on the integration host, so this deployment is the - # exception — pinned to a full GPU. ``torch_dtype`` is left at the - # ``"auto"`` default which picks the model's native bfloat16. - # Requires ``HF_TOKEN`` (gated repo). - "model": "mistralai/Mistral-7B-Instruct-v0.3", - "usecase": "generate", - "loader": "transformers", - "num_cpus": 5, - "num_gpus": 1, - "transformers_config": { - "device": "cuda", - }, - }, - "chat-transformers-reasoning": { - "name": "chat-transformers-reasoning", - # Qwen3-0.6B safetensors — same family as the vLLM `chat-reasoning` - # and llama_server `chat-llama-server` deployments. Lets us - # exercise reasoning, tools, and reasoning+tools through the - # transformers loader's `ChatOutputStreamer` wiring on real model - # output. CPU-only and float32 to match the rest of the - # transformers integration suite. - "model": "Qwen/Qwen3-0.6B", - "usecase": "generate", - "loader": "transformers", - "num_cpus": 2, - "transformers_config": { - "device": "cpu", - "torch_dtype": "float32", - }, - }, - "chat-transformers-function-gemma": { - "name": "chat-transformers-function-gemma", - # Google's FunctionGemma (Gemma 2 family) uses the `` - # and `` syntax. It's a 270M parameter model, small enough - # to run on CPU+float32. - "model": "google/functiongemma-270m-it", - "usecase": "generate", - "loader": "transformers", - "num_cpus": 2, - "transformers_config": { - "device": "cpu", - "torch_dtype": "float32", - }, - }, "embed-model": { "name": "embed-model", "model": "nomic-ai/nomic-embed-text-v1.5", "usecase": "embed", - "loader": "transformers", - "num_cpus": 1, + "loader": "vllm", + "num_gpus": 0.15, + "vllm_engine_kwargs": { + "trust_remote_code": True, + }, }, "stt-model": { "name": "stt-model", "model": "openai/whisper-tiny", "usecase": "transcription", - "loader": "transformers", - "num_cpus": 1, + "loader": "vllm", + "num_gpus": 0.15, + "vllm_engine_kwargs": { + "trust_remote_code": True, + }, }, "tts-model": { "name": "tts-model", @@ -865,74 +779,6 @@ def test_image_url_streaming(self, client): assert len(chunks) > 0 -@pytest.mark.integration -class TestChatTransformers: - @pytest.fixture(autouse=True, scope="class") - def _deploy(self, model_deployer): - model_deployer.deploy("chat-transformers") - - def test_tool_calling_transformers_loader(self, client): - """Round-trip a Hermes-style tool call through the transformers loader. - - Uses the same Qwen2.5-0.5B-Instruct weights as the vLLM `chat-capable` - deployment but goes through the modelship-side tool-calling toolkit - (apply_chat_template(tools=...) on input, hermes parser on output). - """ - completion = client.chat.completions.create( - model="chat-transformers", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - ) - tool_calls = completion.choices[0].message.tool_calls - assert tool_calls, f"expected a tool call, got content={completion.choices[0].message.content!r}" - assert tool_calls[0].function.name == "get_weather" - assert "Paris" in tool_calls[0].function.arguments - assert completion.choices[0].finish_reason == "tool_calls" - - def test_tool_calling_streaming_transformers_loader(self, client): - """Stream a tool call through the transformers loader and verify the - delta sequence matches the OpenAI streaming contract. - - Asserts: - - the function name arrives in exactly one delta; - - arguments arrive across multiple deltas (incremental, not buffered); - - concatenated arguments form valid JSON containing the expected key; - - the final delta carries ``finish_reason="tool_calls"``. - """ - stream = client.chat.completions.create( - model="chat-transformers", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - stream=True, - ) - - collected = _collect_streaming_tool_call(stream) - - assert collected["tool_calls"], ( - f"expected at least one streamed tool call; got content={collected['content']!r}" - ) - call_0 = collected["tool_calls"][0] - assert call_0["id"], "expected an id on the first tool-call delta" - assert call_0["name"] == "get_weather" - # Name must be sent exactly once (not on every delta). - assert collected["name_deltas"] == 1, f"expected one name delta, got {collected['name_deltas']}" - # Arguments must arrive incrementally across multiple deltas — that's the - # whole point of switching from block-level buffering to vLLM-style - # diff streaming. Exact count depends on the model, but it must be > 1. - assert collected["args_deltas"] >= 2, ( - f"expected arguments to stream incrementally, got {collected['args_deltas']} args delta(s)" - ) - # Concatenated args must form valid JSON containing the city. - parsed_args = json.loads(call_0["arguments"]) - assert parsed_args.get("city") - assert "Paris" in parsed_args["city"] - assert collected["finish_reason"] == "tool_calls" - - @pytest.mark.integration @pytest.mark.llama_server class TestChatLlamaServer: @@ -980,7 +826,7 @@ def test_tool_calling_llama_server_loader(self, client): def test_tool_calling_streaming_llama_server_loader(self, client): """Stream a tool call through llama-server and verify the delta sequence matches the OpenAI streaming contract, same shape as the - transformers loader streaming tests.""" + vLLM loader streaming tests.""" stream = client.chat.completions.create( model="chat-llama-server", messages=[{"role": "user", "content": "What is the weather in Paris?"}], @@ -1448,7 +1294,7 @@ def test_tool_calling_llama_server_gpu_loader(self, client): @pytest.mark.llama_server def test_embeddings_llama_server(client, model_deployer): """Real embeddings through a live llama-server subprocess (`--embedding`). - `test_embeddings` only exercises the transformers loader; this is the + `test_embeddings` only exercises the vllm loader; this is the first live-binary coverage of llama_server's B4 embeddings support (previously only unit-tested against a mocked httpx transport).""" model_deployer.deploy("embed-model-llama-server") @@ -1457,260 +1303,6 @@ def test_embeddings_llama_server(client, model_deployer): assert len(response.data[0].embedding) > 0 -@pytest.mark.integration -class TestChatTransformersLlama3Json: - """End-to-end llama3_json tool calling through the transformers loader. - - Llama-3.1-8B-Instruct on GPU emits the JSON-format tool call defined - by Meta's spec — bare ``{"name": "...", "parameters": {...}}``. The - auto-detector picks ``llama3_json`` from the chat template's - ``<|python_tag|>`` reference. This class is the first end-to-end - exercise of the ``llama3_json`` parser on real model output (vLLM - has its own native parser; transformers runs through the cross-loader - registry). - """ - - @pytest.fixture(autouse=True, scope="class") - def _deploy(self, model_deployer): - model_deployer.deploy("chat-transformers-llama3-json") - - def test_tool_calling_transformers_llama3_json_loader(self, client): - """Round-trip a bare-JSON tool call through the transformers loader. - - Verifies the parser surfaces ``message.tool_calls`` from a - ``{"name": "...", "parameters": {...}}`` response and canonicalizes - the ``parameters`` field bytes into ``arguments`` for the OpenAI - contract. - """ - completion = client.chat.completions.create( - model="chat-transformers-llama3-json", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - ) - tool_calls = completion.choices[0].message.tool_calls - assert tool_calls, f"expected a tool call, got content={completion.choices[0].message.content!r}" - assert tool_calls[0].function.name == "get_weather" - assert "Paris" in tool_calls[0].function.arguments - assert completion.choices[0].finish_reason == "tool_calls" - - def test_tool_calling_streaming_transformers_llama3_json_loader(self, client): - """Stream a bare-JSON tool call and verify the OpenAI delta contract. - - Same shape as the Hermes/Mistral streaming tests: exactly one name - delta, multiple incremental argument deltas, valid JSON on - concatenation, final ``finish_reason="tool_calls"``. - """ - stream = client.chat.completions.create( - model="chat-transformers-llama3-json", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - stream=True, - ) - - collected = _collect_streaming_tool_call(stream) - - assert collected["tool_calls"], ( - f"expected at least one streamed tool call; got content={collected['content']!r}" - ) - call_0 = collected["tool_calls"][0] - assert call_0["id"], "expected an id on the first tool-call delta" - assert call_0["name"] == "get_weather" - assert collected["name_deltas"] == 1, f"expected one name delta, got {collected['name_deltas']}" - assert collected["args_deltas"] >= 2, ( - f"expected arguments to stream incrementally, got {collected['args_deltas']} args delta(s)" - ) - parsed_args = json.loads(call_0["arguments"]) - assert parsed_args.get("city") - assert "Paris" in parsed_args["city"] - assert collected["finish_reason"] == "tool_calls" - - -@pytest.mark.integration -class TestChatTransformersMistral: - """End-to-end Mistral tool calling through the transformers loader. - - Regression coverage for the ``markers_are_specials`` fix. Mistral - tokenizers register ``[TOOL_CALLS]`` as a special added token, so - the default ``TextIteratorStreamer(skip_special_tokens=True)`` would - strip the marker before the parser sees it — the parser would - silently miss every tool call. The fix pins - ``_resolved_skip_special_tokens=False`` from the parser flag and - the transformers loader honors it (plus a streamer-side noise - stripper for the other specials that now leak through). - - If this class fails with empty ``tool_calls`` and the marker text - visible in ``message.content``, the loader plumbing has regressed. - """ - - @pytest.fixture(autouse=True, scope="class") - def _deploy(self, model_deployer): - model_deployer.deploy("chat-transformers-mistral") - - def test_tool_calling_transformers_mistral_loader(self, client): - """Round-trip a ``[TOOL_CALLS]``-prefixed tool call through the - transformers loader. - - Verifies the parser surfaces ``message.tool_calls`` from a - ``[TOOL_CALLS][{"name": "...", "arguments": {...}}]`` response — - which only works if the marker survived the tokenizer's - special-token stripping. - """ - completion = client.chat.completions.create( - model="chat-transformers-mistral", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - ) - tool_calls = completion.choices[0].message.tool_calls - assert tool_calls, ( - f"expected a tool call (Mistral [TOOL_CALLS] marker likely stripped before parser saw it); " - f"got content={completion.choices[0].message.content!r}" - ) - assert tool_calls[0].function.name == "get_weather" - assert "Paris" in tool_calls[0].function.arguments - assert completion.choices[0].finish_reason == "tool_calls" - # The marker itself must never leak into content. - assert "[TOOL_CALLS]" not in (completion.choices[0].message.content or "") - - def test_tool_calling_streaming_transformers_mistral_loader(self, client): - """Stream a Mistral tool call and verify the OpenAI delta contract. - - Same shape as the Hermes/llama3_json streaming tests: exactly one - name delta, multiple incremental argument deltas, valid JSON on - concatenation, final ``finish_reason="tool_calls"``. The marker - must not leak into any content delta. - """ - stream = client.chat.completions.create( - model="chat-transformers-mistral", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - stream=True, - ) - - collected = _collect_streaming_tool_call(stream) - - assert collected["tool_calls"], ( - f"expected at least one streamed tool call; got content={collected['content']!r}" - ) - call_0 = collected["tool_calls"][0] - assert call_0["id"], "expected an id on the first tool-call delta" - assert call_0["name"] == "get_weather" - assert collected["name_deltas"] == 1, f"expected one name delta, got {collected['name_deltas']}" - assert collected["args_deltas"] >= 2, ( - f"expected arguments to stream incrementally, got {collected['args_deltas']} args delta(s)" - ) - parsed_args = json.loads(call_0["arguments"]) - assert parsed_args.get("city") - assert "Paris" in parsed_args["city"] - assert collected["finish_reason"] == "tool_calls" - assert "[TOOL_CALLS]" not in collected["content"] - - -@pytest.mark.integration -class TestChatTransformersReasoning: - """End-to-end reasoning + tool calling through the transformers loader. - - Qwen3-0.6B safetensors driven through the HF text-generation pipeline. - Verifies that ``transformers/openai/serving_chat.py`` plumbs - ``_resolved_reasoning_parser`` into the unified ``ChatOutputStreamer`` - and that reasoning, tools, and reasoning+tools all surface correctly - on real model output. - """ - - @pytest.fixture(autouse=True, scope="class") - def _deploy(self, model_deployer): - model_deployer.deploy("chat-transformers-reasoning") - - def test_reasoning_completion_transformers(self): - """Non-streaming: ``...`` block routes to ``message.reasoning``, - the final answer lands in ``message.content``, no marker leakage.""" - response = httpx.post( - f"{OPENAI_API_BASE}/chat/completions", - json={ - "model": "chat-transformers-reasoning", - "messages": [{"role": "user", "content": "Briefly: what is 7 times 8?"}], - "max_tokens": 1024, - }, - timeout=300, - ) - assert response.status_code == 200, response.text - message = response.json()["choices"][0]["message"] - assert message.get("reasoning"), f"expected reasoning content, got {message!r}" - assert "" not in (message.get("content") or "") - assert "" not in (message.get("content") or "") - assert "" not in message["reasoning"] - assert "" not in message["reasoning"] - - def test_reasoning_streaming_transformers(self): - """Streaming: at least one delta carries ``reasoning``; concatenated - reasoning is non-empty; markers never leak into either field.""" - with httpx.stream( - "POST", - f"{OPENAI_API_BASE}/chat/completions", - json={ - "model": "chat-transformers-reasoning", - "messages": [{"role": "user", "content": "Briefly: what is 7 times 8?"}], - "max_tokens": 1024, - "stream": True, - }, - timeout=300, - ) as response: - assert response.status_code == 200 - reasoning_parts: list[str] = [] - content_parts: list[str] = [] - reasoning_deltas = 0 - for line in response.iter_lines(): - if not line.startswith("data: "): - continue - payload = line[len("data: ") :] - if payload == "[DONE]": - break - chunk = json.loads(payload) - delta = chunk["choices"][0].get("delta") or {} - if delta.get("reasoning"): - reasoning_parts.append(delta["reasoning"]) - reasoning_deltas += 1 - if delta.get("content"): - content_parts.append(delta["content"]) - - assert reasoning_deltas >= 1, "expected at least one reasoning delta" - assert "".join(reasoning_parts).strip(), "expected non-empty reasoning content" - assert "" not in "".join(reasoning_parts) - assert "" not in "".join(reasoning_parts) - assert "" not in "".join(content_parts) - assert "" not in "".join(content_parts) - - def test_reasoning_with_tools_transformers(self, client): - """Reasoning + tool calling in one round-trip through transformers. - - The single-pass ``ChatOutputStreamer`` must populate BOTH - ``message.reasoning`` and ``message.tool_calls``. - """ - completion = client.chat.completions.create( - model="chat-transformers-reasoning", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=1024, - ) - message = completion.choices[0].message - reasoning = getattr(message, "reasoning", None) or message.model_extra.get("reasoning") - assert reasoning, f"expected reasoning, got message={message!r}" - assert "" not in reasoning - tool_calls = message.tool_calls - assert tool_calls, f"expected a tool call, got content={message.content!r}, reasoning={reasoning!r}" - assert tool_calls[0].function.name == "get_weather" - assert "Paris" in tool_calls[0].function.arguments - assert completion.choices[0].finish_reason == "tool_calls" - - @pytest.mark.integration def test_embeddings(client, model_deployer): model_deployer.deploy("embed-model") @@ -1944,64 +1536,6 @@ def test_image_variation(self, client, tmp_path): self._assert_png(self._decode(variation.data[0].b64_json), expect_size=(256, 256)) -@pytest.mark.integration -@pytest.mark.function_gemma -class TestChatTransformersFunctionGemma: - """End-to-end FunctionGemma (Gemma 2) tool calling through transformers. - - Verifies that the `function_gemma` parser correctly intercepts the - `` markers and `` syntax. - """ - - @pytest.fixture(autouse=True, scope="class") - def _deploy(self, model_deployer): - model_deployer.deploy("chat-transformers-function-gemma") - - def test_tool_calling_transformers_function_gemma_loader(self, client): - completion = client.chat.completions.create( - model="chat-transformers-function-gemma", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - ) - tool_calls = completion.choices[0].message.tool_calls - assert tool_calls, f"expected a tool call, got content={completion.choices[0].message.content!r}" - assert tool_calls[0].function.name == "get_weather" - assert "Paris" in tool_calls[0].function.arguments - assert completion.choices[0].finish_reason == "tool_calls" - - def test_tool_calling_streaming_transformers_function_gemma_loader(self, client): - stream = client.chat.completions.create( - model="chat-transformers-function-gemma", - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=[_WEATHER_TOOL], - tool_choice="auto", - max_tokens=128, - stream=True, - ) - - collected = _collect_streaming_tool_call(stream) - - assert collected["tool_calls"], ( - f"expected at least one streamed tool call; got content={collected['content']!r}" - ) - call_0 = collected["tool_calls"][0] - assert call_0["id"], "expected an id on the first tool-call delta" - assert call_0["name"] == "get_weather" - assert collected["name_deltas"] == 1, f"expected one name delta, got {collected['name_deltas']}" - # HF's ``TextIteratorStreamer`` only emits up to the last whitespace - # character (transformers/generation/streamers.py: ``text.rfind(" ")+1``), - # and FunctionGemma's args body contains no internal spaces, so the - # whole body arrives in a single chunk and produces exactly one args - # delta. vLLM / llama_server loaders emit per-token and still satisfy >= 2. - assert collected["args_deltas"] >= 1, f"expected at least one args delta, got {collected['args_deltas']}" - parsed_args = json.loads(call_0["arguments"]) - assert parsed_args.get("city") - assert "Paris" in parsed_args["city"] - assert collected["finish_reason"] == "tool_calls" - - # Responses tools use the *flattened* function shape (name/parameters at the # top level), unlike chat completions which nests them under "function". _WEATHER_TOOL_RESPONSES = { diff --git a/tests/test_mistral_specials_smoketest.py b/tests/test_mistral_specials_smoketest.py index bb26f55..bd5c8ae 100644 --- a/tests/test_mistral_specials_smoketest.py +++ b/tests/test_mistral_specials_smoketest.py @@ -3,8 +3,11 @@ Background ---------- The ``MistralToolCallParser`` declares ``start_marker = "[TOOL_CALLS]"``. -The transformers loader streams generation through ``TextIteratorStreamer`` -with ``skip_special_tokens=True`` ([modelship/infer/transformers/openai/serving_chat.py:221]). +A loader that detokenizes raw model output with ``skip_special_tokens=True`` +would strip that marker before it reaches our parser if the tokenizer +registers it as a special token — this is exactly why +``_resolved_skip_special_tokens`` gets pinned to ``False`` for the mistral +parser (see ``resolve_all_tool_parsers``). If a Mistral tokenizer registers ``[TOOL_CALLS]`` as an *additional special token*, then on a real Mistral model run, the marker would be stripped from @@ -19,8 +22,8 @@ string, decode with each ``skip_special_tokens`` setting, and verify that the stripping behavior is what we feared. This tests the HF contract, not Mistral specifically — it tells us whether *any* tokenizer - that registers the marker as special would lose it on the transformers - loader path. + that registers the marker as special would lose it when a loader + detokenizes with specials skipped. 2. **Real Mistral** (skipped if the tokenizer can't be loaded): load a real Mistral tokenizer and check whether ``[TOOL_CALLS]`` actually @@ -75,13 +78,12 @@ def test_roundtrip_keeps_marker_when_skip_special_tokens_false(self, tokenizer_w def test_roundtrip_strips_marker_when_skip_special_tokens_true(self, tokenizer_with_tool_calls_special): """The hypothesis: when ``[TOOL_CALLS]`` is an additional special token, ``skip_special_tokens=True`` removes it from the decoded - text — which is exactly what the transformers loader does today. + text. - If this assertion holds, the just-merged Mistral parser cannot - activate on the transformers loader for any tokenizer that - registers the marker as special. The fix is loader-side - (per-parser flag flip + noise stripper, deferred from the - ``llama3_json`` PR). + If this assertion holds, the Mistral parser cannot activate on a + loader that detokenizes with specials skipped, for any tokenizer + that registers the marker as special. The fix is loader-side + (per-parser flag flip + noise stripper). """ text = _sample_tool_call_text() ids = tokenizer_with_tool_calls_special.encode(text, add_special_tokens=False) @@ -89,7 +91,7 @@ def test_roundtrip_strips_marker_when_skip_special_tokens_true(self, tokenizer_w assert "[TOOL_CALLS]" not in decoded, ( "expected `[TOOL_CALLS]` to be stripped by skip_special_tokens=True; " f"got decoded={decoded!r}. If this assertion fails, the hypothesis is wrong " - "and Mistral on the transformers loader is fine as-is." + "and Mistral is fine as-is without the skip-specials pin." ) diff --git a/tests/test_parser_resolution.py b/tests/test_parser_resolution.py index 8696254..225af2e 100644 --- a/tests/test_parser_resolution.py +++ b/tests/test_parser_resolution.py @@ -15,7 +15,6 @@ ModelshipConfig, ModelshipModelConfig, ModelUsecase, - TransformersConfig, VllmEngineConfig, ) from modelship.openai.parsers.reasoning.utils import classify_template as classify_reasoning @@ -116,14 +115,6 @@ def test_vllm_explicit_stored(self): resolve_all_tool_parsers(ModelshipConfig(models=[cfg])) assert cfg._resolved_tool_call_parser == "hermes" - def test_transformers_explicit_stored(self): - cfg = _make_cfg( - loader=ModelLoader.transformers, - transformers_config=TransformersConfig(tool_call_parser="hermes"), - ) - resolve_all_tool_parsers(ModelshipConfig(models=[cfg])) - assert cfg._resolved_tool_call_parser == "hermes" - def test_unknown_explicit_raises(self): cfg = _make_cfg(vllm_engine_kwargs=VllmEngineConfig(tool_call_parser="not-a-real-parser")) with pytest.raises(ValueError, match="not-a-real-parser"): @@ -138,11 +129,11 @@ def test_vllm_opt_out_leaves_none(self): class TestResolveSkipSpecialTokens: """``_resolved_skip_special_tokens`` is pinned by the parser's flag. - The transformers loader reads this at startup to decide whether to - flip ``TextIteratorStreamer(skip_special_tokens=False)``. ``None`` - means "loader keeps its own default (True)"; ``False`` means "the - parser's marker is registered as a special token and would be - stripped — keep specials in the stream and noise-strip the rest." + Loaders that detokenize raw model output read this at startup to + decide whether to flip ``skip_special_tokens=False``. ``None`` means + "loader keeps its own default (True)"; ``False`` means "the parser's + marker is registered as a special token and would be stripped — keep + specials in the stream and noise-strip the rest." """ def test_hermes_leaves_skip_specials_default(self): @@ -158,9 +149,6 @@ def test_llama3_json_leaves_skip_specials_default(self): def test_mistral_pins_skip_specials_false(self): # Mistral parser's marker is a special added token; loader must # keep specials so the parser sees `[TOOL_CALLS]`. - cfg = _make_cfg( - loader=ModelLoader.transformers, - transformers_config=TransformersConfig(tool_call_parser="mistral"), - ) + cfg = _make_cfg(vllm_engine_kwargs=VllmEngineConfig(tool_call_parser="mistral")) resolve_all_tool_parsers(ModelshipConfig(models=[cfg])) assert cfg._resolved_skip_special_tokens is False diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 21c763e..f0dbc5f 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -102,7 +102,7 @@ def test_matching_value_no_warning(self): class TestRunPreflightDispatch: def test_returns_empty_for_unregistered_loader(self): cfg = _make_config() - cfg.loader = ModelLoader.transformers + cfg.loader = ModelLoader.diffusers result = run_preflight(cfg, HardwareProfile()) assert result == {} diff --git a/tests/test_tool_calling.py b/tests/test_tool_calling.py index 54febca..730252b 100644 --- a/tests/test_tool_calling.py +++ b/tests/test_tool_calling.py @@ -543,8 +543,9 @@ def test_llama3_json_marker_is_regular_text(self): assert Llama3JsonToolCallParser().markers_are_specials is False def test_mistral_marker_is_a_special_token(self): - # Drives the transformers loader to flip ``skip_special_tokens=False`` - # at startup so ``[TOOL_CALLS]`` survives detokenization. + # Drives loaders that detokenize raw output to flip + # ``skip_special_tokens=False`` at startup so ``[TOOL_CALLS]`` + # survives detokenization. assert MistralToolCallParser().markers_are_specials is True diff --git a/tests/test_transformers_capabilities.py b/tests/test_transformers_capabilities.py deleted file mode 100644 index cc98263..0000000 --- a/tests/test_transformers_capabilities.py +++ /dev/null @@ -1,35 +0,0 @@ -from types import SimpleNamespace - -from modelship.infer.transformers.capabilities import TransformersCapabilities - - -def test_text_generation_pipeline_is_text_only(): - pipe = SimpleNamespace(task="text-generation") - caps = TransformersCapabilities.detect(pipe) # type: ignore[arg-type] - assert caps.supports_image is False - assert caps.supports_audio is False - - -def test_image_text_to_text_pipeline_supports_image(): - pipe = SimpleNamespace(task="image-text-to-text") - caps = TransformersCapabilities.detect(pipe) # type: ignore[arg-type] - assert caps.supports_image is True - assert caps.supports_audio is False - - -def test_visual_question_answering_supports_image(): - pipe = SimpleNamespace(task="visual-question-answering") - caps = TransformersCapabilities.detect(pipe) # type: ignore[arg-type] - assert caps.supports_image is True - - -def test_unknown_task_defaults_to_text_only(): - pipe = SimpleNamespace(task="some-future-task") - caps = TransformersCapabilities.detect(pipe) # type: ignore[arg-type] - assert caps.supports_image is False - - -def test_pipeline_without_task_attribute_is_text_only(): - pipe = SimpleNamespace() - caps = TransformersCapabilities.detect(pipe) # type: ignore[arg-type] - assert caps.supports_image is False diff --git a/tests/test_transformers_chat_reasoning.py b/tests/test_transformers_chat_reasoning.py deleted file mode 100644 index 47c6e8d..0000000 --- a/tests/test_transformers_chat_reasoning.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Tests for the Transformers chat path's reasoning + tool-call composition. - -These tests bypass the real HF pipeline by injecting a callable that returns -a canned generation, so they run offline and do not touch any model weights. -The shared parser/streamer logic is exercised in ``test_reasoning.py`` and -``test_tool_calling.py``; this file focuses on the wiring between -``transformers/openai/serving_chat.py`` and the unified streamer. -""" - -from __future__ import annotations - -import json -from typing import Any - -import pytest - -from modelship.infer.infer_config import RawRequestProxy, TransformersConfig -from modelship.infer.transformers.capabilities import TransformersCapabilities -from modelship.infer.transformers.openai.serving_chat import OpenAIServingChat -from modelship.openai.protocol import ChatCompletionRequest, ChatCompletionResponse - - -class _FakeTokenizer: - def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: - return [0] * len(text.split()) - - def apply_chat_template(self, messages: list[dict], **kwargs: Any) -> Any: - prompt = "\n".join(f"{m['role']}: {m.get('content', '')}" for m in messages) - if "tools" in kwargs: - prompt = f"[TOOLS:{len(kwargs['tools'])}]\n" + prompt - if kwargs.get("tokenize"): - return [0] * len(prompt.split()) - return prompt - - -class _FakePipeline: - def __init__(self, generated_text: str): - self.tokenizer = _FakeTokenizer() - self.task = "text-generation" - self.generated_text = generated_text - - def __call__(self, inputs: Any, **kwargs: Any) -> list[dict]: - return [{"generated_text": self.generated_text}] - - -def _make_serving( - generated: str, - *, - tool_call_parser: str | None = None, - reasoning_parser: str | None = "deepseek_r1", -) -> OpenAIServingChat: - pipe = _FakePipeline(generated) - return OpenAIServingChat( - pipeline=pipe, # type: ignore[arg-type] - model_name="test-model", - config=TransformersConfig(), - capabilities=TransformersCapabilities(supports_image=False, supports_audio=False), - tool_call_parser=tool_call_parser, - reasoning_parser=reasoning_parser, - ) - - -def _raw_request() -> RawRequestProxy: - return RawRequestProxy(None, {}) - - -@pytest.mark.asyncio -async def test_reasoning_only_response_splits_thinking_and_content(): - raw = "let me thinkThe answer is 42." - serving = _make_serving(raw) - req = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}], stream=False) - resp = await serving.create_chat_completion(req, _raw_request()) - - assert isinstance(resp, ChatCompletionResponse) - msg = resp.choices[0].message - assert msg.reasoning == "let me think" - assert msg.content == "The answer is 42." - assert msg.tool_calls == [] - assert resp.choices[0].finish_reason == "stop" - - -@pytest.mark.asyncio -async def test_reasoning_with_no_markers_falls_back_to_content(): - # Reasoning parser is configured, but the model emitted no - # block — content should pass through intact and reasoning is None. - serving = _make_serving("just a regular reply") - req = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}], stream=False) - resp = await serving.create_chat_completion(req, _raw_request()) - - assert isinstance(resp, ChatCompletionResponse) - msg = resp.choices[0].message - assert msg.reasoning is None - assert msg.content == "just a regular reply" - - -@pytest.mark.asyncio -async def test_tool_call_marker_inside_thinking_is_not_parsed_as_tool_call(): - # The single-pass streamer routes the tool marker inside to the - # reasoning view; it must NOT show up as a finalized tool call. - raw = ( - 'I would call {"name": "x", "arguments": {}} here' - "Sure, I'll call it for real now." - ) - serving = _make_serving(raw, tool_call_parser="hermes", reasoning_parser="deepseek_r1") - req = ChatCompletionRequest( - messages=[{"role": "user", "content": "ping"}], - tools=[{"type": "function", "function": {"name": "x"}}], - stream=False, - ) - resp = await serving.create_chat_completion(req, _raw_request()) - - assert isinstance(resp, ChatCompletionResponse) - msg = resp.choices[0].message - assert msg.tool_calls == [] - assert msg.reasoning is not None - assert "" in msg.reasoning # the marker text lives inside reasoning - assert msg.content == "Sure, I'll call it for real now." - assert resp.choices[0].finish_reason == "stop" - - -@pytest.mark.asyncio -async def test_thinking_then_real_tool_call_after_close_marker(): - # Reasoning closes before a real tool call is emitted; both should land. - raw = ( - "I should call get_weather for Paris." - '{"name": "get_weather", "arguments": {"city": "Paris"}}' - ) - serving = _make_serving(raw, tool_call_parser="hermes", reasoning_parser="deepseek_r1") - req = ChatCompletionRequest( - messages=[{"role": "user", "content": "weather paris?"}], - tools=[{"type": "function", "function": {"name": "get_weather"}}], - stream=False, - ) - resp = await serving.create_chat_completion(req, _raw_request()) - - assert isinstance(resp, ChatCompletionResponse) - msg = resp.choices[0].message - assert msg.reasoning == "I should call get_weather for Paris." - assert len(msg.tool_calls) == 1 - assert msg.tool_calls[0].function.name == "get_weather" - assert json.loads(msg.tool_calls[0].function.arguments) == {"city": "Paris"} - assert msg.content is None - assert resp.choices[0].finish_reason == "tool_calls" - - -@pytest.mark.asyncio -async def test_unknown_reasoning_parser_at_init_raises(): - pipe = _FakePipeline("anything") - with pytest.raises(ValueError, match="reasoning"): - OpenAIServingChat( - pipeline=pipe, # type: ignore[arg-type] - model_name="test-model", - config=TransformersConfig(), - capabilities=TransformersCapabilities(supports_image=False, supports_audio=False), - reasoning_parser="not-a-real-parser", - ) - - -@pytest.mark.asyncio -async def test_no_parsers_configured_passes_text_through_unchanged(): - # Both parsers None: the chat path must not extract anything, even if the - # model happens to emit marker-shaped tokens. - raw = "shouldn't be splitplain" - pipe = _FakePipeline(raw) - serving = OpenAIServingChat( - pipeline=pipe, # type: ignore[arg-type] - model_name="test-model", - config=TransformersConfig(), - capabilities=TransformersCapabilities(supports_image=False, supports_audio=False), - ) - req = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}], stream=False) - resp = await serving.create_chat_completion(req, _raw_request()) - - assert isinstance(resp, ChatCompletionResponse) - msg = resp.choices[0].message - assert msg.reasoning is None - assert msg.content == raw - assert msg.tool_calls == [] diff --git a/tests/test_transformers_chat_tools.py b/tests/test_transformers_chat_tools.py deleted file mode 100644 index a077e8c..0000000 --- a/tests/test_transformers_chat_tools.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Tests for the Transformers chat path's tool-call handling. - -These tests bypass the real HF pipeline by injecting a callable that returns -a canned generation, so they run offline and do not touch any model weights. -""" - -from __future__ import annotations - -import json -from typing import Any - -import pytest - -from modelship.infer.infer_config import RawRequestProxy, TransformersConfig -from modelship.infer.transformers.capabilities import TransformersCapabilities -from modelship.infer.transformers.openai.serving_chat import OpenAIServingChat -from modelship.openai.protocol import ChatCompletionRequest, ChatCompletionResponse - - -class _FakeTokenizer: - def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: - return [0] * len(text.split()) - - def apply_chat_template(self, messages: list[dict], **kwargs: Any) -> Any: - prompt = "\n".join(f"{m['role']}: {m.get('content', '')}" for m in messages) - if kwargs.get("tools"): - prompt = f"[TOOLS:{len(kwargs['tools'])}]\n" + prompt - if kwargs.get("enable_thinking") is False: # template branching on a chat_template_kwarg - prompt = "[NOTHINK]\n" + prompt - if kwargs.get("tokenize"): - return [0] * len(prompt.split()) - return prompt - - -class _FakePipeline: - """Stand-in for ``transformers.Pipeline`` that records calls and replays canned output.""" - - def __init__(self, generated_text: str): - self.tokenizer = _FakeTokenizer() - self.task = "text-generation" - self.generated_text = generated_text - self.last_input: Any = None - self.last_kwargs: dict[str, Any] = {} - - def __call__(self, inputs: Any, **kwargs: Any) -> list[dict]: - self.last_input = inputs - self.last_kwargs = kwargs - # Streaming path runs us in a thread with a streamer; signal end so the - # consuming async iterator terminates instead of blocking. - streamer = kwargs.get("streamer") - if streamer is not None: - streamer.end() - return [{"generated_text": self.generated_text}] - - -def _make_serving( - generated: str, - tool_call_parser: str | None = "hermes", - chat_template_kwargs: dict[str, Any] | None = None, -) -> tuple[OpenAIServingChat, _FakePipeline]: - pipe = _FakePipeline(generated) - serving = OpenAIServingChat( - pipeline=pipe, # type: ignore[arg-type] - model_name="test-model", - config=TransformersConfig(), - capabilities=TransformersCapabilities(supports_image=False, supports_audio=False), - tool_call_parser=tool_call_parser, - chat_template_kwargs=chat_template_kwargs, - ) - return serving, pipe - - -def _raw_request() -> RawRequestProxy: - return RawRequestProxy(None, {}) - - -@pytest.mark.asyncio -async def test_response_without_tools_carries_content_only(): - serving, _ = _make_serving("hello there") - req = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}], stream=False) - resp = await serving.create_chat_completion(req, _raw_request()) - - assert isinstance(resp, ChatCompletionResponse) - msg = resp.choices[0].message - assert msg.content == "hello there" - assert msg.tool_calls == [] - assert resp.choices[0].finish_reason == "stop" - - -@pytest.mark.asyncio -async def test_tools_only_render_when_requested(): - serving, pipe = _make_serving("hello") - req = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}], stream=False) - await serving.create_chat_completion(req, _raw_request()) - - # Without `tools` in the request, the pipeline receives the message list - # directly — no pre-rendered prompt. - assert isinstance(pipe.last_input, list) - - -@pytest.mark.asyncio -async def test_tools_in_request_pre_renders_prompt_and_parses_tool_call(): - raw = '{"name": "get_weather", "arguments": {"city": "Paris"}}' - serving, pipe = _make_serving(raw) - req = ChatCompletionRequest( - messages=[{"role": "user", "content": "weather in paris?"}], - tools=[ - { - "type": "function", - "function": {"name": "get_weather", "parameters": {"type": "object"}}, - } - ], - tool_choice="auto", - stream=False, - ) - resp = await serving.create_chat_completion(req, _raw_request()) - - assert isinstance(resp, ChatCompletionResponse) - # Pre-rendered prompt is a string carrying the tool marker injected by our fake template. - assert isinstance(pipe.last_input, str) - assert pipe.last_input.startswith("[TOOLS:1]") - - msg = resp.choices[0].message - assert msg.content is None - assert len(msg.tool_calls) == 1 - assert msg.tool_calls[0].function.name == "get_weather" - assert json.loads(msg.tool_calls[0].function.arguments) == {"city": "Paris"} - assert resp.choices[0].finish_reason == "tool_calls" - - -@pytest.mark.asyncio -async def test_tool_choice_none_skips_tool_rendering(): - serving, pipe = _make_serving("regular reply") - req = ChatCompletionRequest( - messages=[{"role": "user", "content": "hi"}], - tools=[{"type": "function", "function": {"name": "noop"}}], - tool_choice="none", - stream=False, - ) - resp = await serving.create_chat_completion(req, _raw_request()) - - # tool_choice="none" — pipeline should receive messages, not a rendered prompt. - assert isinstance(pipe.last_input, list) - assert isinstance(resp, ChatCompletionResponse) - assert resp.choices[0].message.content == "regular reply" - assert resp.choices[0].message.tool_calls == [] - assert resp.choices[0].finish_reason == "stop" - - -@pytest.mark.asyncio -async def test_tool_call_with_trailing_text_preserves_content(): - raw = 'Calling now.\n{"name": "ping", "arguments": {}}' - serving, _ = _make_serving(raw) - req = ChatCompletionRequest( - messages=[{"role": "user", "content": "ping?"}], - tools=[{"type": "function", "function": {"name": "ping"}}], - stream=False, - ) - resp = await serving.create_chat_completion(req, _raw_request()) - - assert isinstance(resp, ChatCompletionResponse) - msg = resp.choices[0].message - assert msg.content == "Calling now.\n" - assert len(msg.tool_calls) == 1 - - -@pytest.mark.asyncio -async def test_chat_template_kwargs_forwarded_on_no_tools_path(): - # The no-tools path renders the prompt ourselves when kwargs are set, so the - # pipeline receives a string carrying the template's branched-on marker. - serving, pipe = _make_serving("hi back", tool_call_parser=None, chat_template_kwargs={"enable_thinking": False}) - req = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}], stream=False) - await serving.create_chat_completion(req, _raw_request()) - assert isinstance(pipe.last_input, str) - assert pipe.last_input.startswith("[NOTHINK]") - - -@pytest.mark.asyncio -async def test_no_chat_template_kwargs_leaves_no_tools_path_on_message_list(): - # Without kwargs (or tools) the pipeline still gets the raw message list. - serving, pipe = _make_serving("hi back", tool_call_parser=None) - req = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}], stream=False) - await serving.create_chat_completion(req, _raw_request()) - assert isinstance(pipe.last_input, list) - - -@pytest.mark.asyncio -async def test_chat_template_kwargs_forwarded_on_streaming_no_tools_path(): - # The streaming path must pre-render like the non-streaming one so the kwargs - # reach apply_chat_template instead of being dropped by the pipeline. - serving, pipe = _make_serving("hi back", tool_call_parser=None, chat_template_kwargs={"enable_thinking": False}) - req = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}], stream=True) - gen = await serving.create_chat_completion(req, _raw_request()) - async for _ in gen: # type: ignore[union-attr] - pass - assert isinstance(pipe.last_input, str) - assert pipe.last_input.startswith("[NOTHINK]") - - -@pytest.mark.asyncio -async def test_reserved_chat_template_kwargs_are_dropped(): - # `tokenize` would corrupt the token counter and `tools` collide with our - # own argument — both must be stripped, leaving only the safe key. - serving, _ = _make_serving( - "hi back", - tool_call_parser=None, - chat_template_kwargs={ - "enable_thinking": False, - "tokenize": False, - "tools": [], - "messages": [], - "conversation": [], - }, - ) - assert serving.chat_template_kwargs == {"enable_thinking": False} - - -@pytest.mark.asyncio -async def test_unknown_parser_at_init_raises(): - pipe = _FakePipeline("anything") - with pytest.raises(ValueError, match="unknown tool_call_parser"): - OpenAIServingChat( - pipeline=pipe, # type: ignore[arg-type] - model_name="test-model", - config=TransformersConfig(), - capabilities=TransformersCapabilities(supports_image=False, supports_audio=False), - tool_call_parser="not-a-real-parser", - ) diff --git a/tests/test_vllm_vision.py b/tests/test_vllm_vision.py index e570d54..5f5c9f7 100644 --- a/tests/test_vllm_vision.py +++ b/tests/test_vllm_vision.py @@ -91,7 +91,7 @@ def _make_infer(*, supports_image: bool) -> VllmInfer: @pytest.mark.asyncio async def test_image_part_rejected_on_text_only_model_with_400(): """A text-only model receiving image_url returns a 400 BadRequest with - our error envelope — same shape transformers and llama.cpp produce.""" + our error envelope — same shape llama.cpp produces.""" infer = _make_infer(supports_image=False) request = ChatCompletionRequest( model="llm", diff --git a/uv.lock b/uv.lock index 230d547..f1912f4 100644 --- a/uv.lock +++ b/uv.lock @@ -32,31 +32,12 @@ conflicts = [[ [manifest] members = [ - "bark", "kokoroonnx", "modelship", "orpheus", "whispercpp", ] -[[package]] -name = "accelerate" -version = "1.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "huggingface-hub" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "psutil" }, - { name = "pyyaml" }, - { name = "safetensors" }, - { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ca/14/787e5498cd062640f0f3d92ef4ae4063174f76f9afd29d13fc52a319daae/accelerate-1.13.0.tar.gz", hash = "sha256:d631b4e0f5b3de4aff2d7e9e6857d164810dfc3237d54d017f075122d057b236", size = 402835, upload-time = "2026-03-04T19:34:12.359Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/46/02ac5e262d4af18054b3e922b2baedbb2a03289ee792162de60a865defc5/accelerate-1.13.0-py3-none-any.whl", hash = "sha256:cf1a3efb96c18f7b152eb0fa7490f3710b19c3f395699358f08decca2b8b62e0", size = 383744, upload-time = "2026-03-04T19:34:10.313Z" }, -] - [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -262,11 +243,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, ] -[[package]] -name = "bark" -version = "0.1.0" -source = { editable = "plugins/bark" } - [[package]] name = "bitsandbytes" version = "0.49.2" @@ -1668,12 +1644,8 @@ dependencies = [ ] [package.optional-dependencies] -bark = [ - { name = "bark" }, -] cpu = [ { name = "onnxruntime" }, - { name = "sentence-transformers" }, { name = "stable-diffusion-cpp-python" }, { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, @@ -1691,13 +1663,11 @@ dev = [ { name = "ruff" }, ] gpu = [ - { name = "accelerate" }, { name = "bitsandbytes" }, { name = "diffusers" }, { name = "flashinfer-python" }, { name = "nvidia-ml-py" }, { name = "onnxruntime-gpu" }, - { name = "sentence-transformers" }, { name = "stable-diffusion-cpp-python" }, { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, { name = "torchvision", version = "0.26.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, @@ -1720,10 +1690,8 @@ whispercpp = [ [package.metadata] requires-dist = [ - { name = "accelerate", marker = "extra == 'gpu'", specifier = ">=1.6.0" }, { name = "argparse", specifier = ">=1.4.0" }, { name = "asyncio", specifier = ">=4.0.0" }, - { name = "bark", marker = "extra == 'bark'", editable = "plugins/bark" }, { name = "bitsandbytes", marker = "extra == 'gpu'", specifier = ">=0.49.0" }, { name = "diffusers", marker = "extra == 'gpu'", specifier = ">=0.31.0" }, { name = "fakeredis", marker = "extra == 'dev'", specifier = ">=2.36.2" }, @@ -1754,8 +1722,6 @@ requires-dist = [ { name = "requests", specifier = ">=2.32.5" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.11.0" }, { name = "scipy", specifier = ">=1.16.1" }, - { name = "sentence-transformers", marker = "extra == 'cpu'", specifier = ">=3.0.0" }, - { name = "sentence-transformers", marker = "extra == 'gpu'", specifier = ">=3.0.0" }, { name = "soundfile", specifier = ">=0.13.0" }, { name = "stable-diffusion-cpp-python", marker = "extra == 'cpu'", specifier = ">=0.4.7" }, { name = "stable-diffusion-cpp-python", marker = "extra == 'gpu'", specifier = ">=0.4.7" }, @@ -1771,7 +1737,7 @@ requires-dist = [ { name = "vllm", extras = ["audio"], marker = "extra == 'gpu'", specifier = "==0.24.0" }, { name = "whispercpp", marker = "extra == 'whispercpp'", editable = "plugins/whispercpp" }, ] -provides-extras = ["gpu", "cpu", "kokoroonnx", "bark", "orpheus", "whispercpp", "otel", "dev"] +provides-extras = ["gpu", "cpu", "kokoroonnx", "orpheus", "whispercpp", "otel", "dev"] [[package]] name = "mpmath" @@ -3487,27 +3453,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/18/cb614939ccd46d336013cab705f1e11540ec9c68b08ecbb854ab893fc480/segments-2.3.0-py2.py3-none-any.whl", hash = "sha256:30a5656787071430cd22422e04713b2a9beabe1a97d2ebf37f716a56f90577a3", size = 15705, upload-time = "2025-02-20T07:55:39.755Z" }, ] -[[package]] -name = "sentence-transformers" -version = "5.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "huggingface-hub" }, - { name = "numpy" }, - { name = "scikit-learn" }, - { name = "scipy" }, - { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu')" }, - { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and platform_machine != 's390x' and platform_machine != 'x86_64' and extra == 'extra-9-modelship-cpu' and extra == 'extra-9-modelship-gpu') or (platform_machine == 'aarch64' and extra == 'extra-9-modelship-cpu') or (platform_machine == 's390x' and extra == 'extra-9-modelship-cpu') or (platform_machine == 'x86_64' and extra == 'extra-9-modelship-cpu') or (sys_platform != 'darwin' and extra == 'extra-9-modelship-cpu')" }, - { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "extra == 'extra-9-modelship-gpu'" }, - { name = "tqdm" }, - { name = "transformers" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/b0/fcc810aa1747e96eb2e342e16173dbebbff85b8ccd079f35b32874c5f6ce/sentence_transformers-5.4.0.tar.gz", hash = "sha256:ef0c129d653f736df5fd74d46af11a2234328a32cddb1d91a1975c5f6cccc194", size = 428330, upload-time = "2026-04-09T13:34:12.642Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/d4/58f3e1308c8d15cae83407bd651542b62155a2b3248e0290a724e80fd124/sentence_transformers-5.4.0-py3-none-any.whl", hash = "sha256:4e5ac9a19244153f3f4074d898e664990408d7b077fb1d1d0dc9cb8a9316feac", size = 570836, upload-time = "2026-04-09T13:34:11.298Z" }, -] - [[package]] name = "sentencepiece" version = "0.2.1" From 4fbbd9a6c53753433eae385333a72f5992843588 Mon Sep 17 00:00:00 2001 From: Alex M <459478+alez007@users.noreply.github.com> Date: Sat, 4 Jul 2026 21:29:19 +0000 Subject: [PATCH 35/35] refactor: delete dead raw-text parser engine from openai/parsers/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit modelship/openai/parsers/ mixed a large dead streaming/parsing engine (only ever driven by the now-removed transformers and llama_cpp loaders) with a small amount of still-live vLLM auto-detection plumbing. Delete the dead half (ChatOutputStreamer, per-family tool-call/reasoning parsers, the Phase A responses adapter's chat-completions round trip) and extract the live half (chat-template classification, reasoning-suppression probing) into two small flat modules. Also drop _resolved_skip_special_tokens, a write-only leftover from the same removed loaders, and fix resolve_all_tool_parsers to validate explicit tool_call_parser names against vLLM's own ToolParserManager instead of a hand-maintained registry — this also fixes a real mismatch where FunctionGemma auto-detection produced "function_gemma" instead of vLLM's registered "functiongemma", which broke the documented gemma-on-CPU example at preflight. --- modelship/deploy/config.py | 57 +- modelship/infer/infer_config.py | 9 - modelship/openai/chat_utils.py | 1 - modelship/openai/parsers/__init__.py | 25 +- modelship/openai/parsers/output.py | 402 --------- .../{reasoning/utils.py => reasoning.py} | 48 +- .../openai/parsers/reasoning/__init__.py | 29 - .../parsers/reasoning/parsers/__init__.py | 5 - .../openai/parsers/reasoning/parsers/base.py | 34 - .../parsers/reasoning/parsers/deepseek.py | 16 - .../openai/parsers/reasoning/parsers/gemma.py | 19 - .../openai/parsers/reasoning/registry.py | 39 - modelship/openai/parsers/streaming.py | 275 ------- .../utils.py => tool_calling.py} | 28 +- .../openai/parsers/tool_calling/__init__.py | 30 - .../openai/parsers/tool_calling/input.py | 68 -- .../parsers/tool_calling/parsers/__init__.py | 19 - .../parsers/tool_calling/parsers/base.py | 106 --- .../parsers/tool_calling/parsers/gemma.py | 363 --------- .../parsers/tool_calling/parsers/hermes.py | 46 -- .../tool_calling/parsers/llama3_json.py | 109 --- .../parsers/tool_calling/parsers/mistral.py | 109 --- .../tool_calling/parsers/qwen3_coder.py | 125 --- .../openai/parsers/tool_calling/registry.py | 47 -- .../openai/protocol/responses/__init__.py | 32 +- .../openai/protocol/responses/adapter.py | 90 +-- .../openai/protocol/responses/streaming.py | 65 +- tests/test_gemma4_parsers.py | 197 ----- tests/test_integration.py | 12 +- tests/test_mistral_specials_smoketest.py | 154 ---- tests/test_parser_resolution.py | 74 +- tests/test_qwen3_coder_parser.py | 178 ---- tests/test_reasoning.py | 280 ------- tests/test_reasoning_detection.py | 29 +- tests/test_responses_adapter.py | 115 +-- tests/test_responses_streaming.py | 64 +- tests/test_tool_calling.py | 760 ------------------ 37 files changed, 168 insertions(+), 3891 deletions(-) delete mode 100644 modelship/openai/parsers/output.py rename modelship/openai/parsers/{reasoning/utils.py => reasoning.py} (60%) delete mode 100644 modelship/openai/parsers/reasoning/__init__.py delete mode 100644 modelship/openai/parsers/reasoning/parsers/__init__.py delete mode 100644 modelship/openai/parsers/reasoning/parsers/base.py delete mode 100644 modelship/openai/parsers/reasoning/parsers/deepseek.py delete mode 100644 modelship/openai/parsers/reasoning/parsers/gemma.py delete mode 100644 modelship/openai/parsers/reasoning/registry.py delete mode 100644 modelship/openai/parsers/streaming.py rename modelship/openai/parsers/{tool_calling/utils.py => tool_calling.py} (69%) delete mode 100644 modelship/openai/parsers/tool_calling/__init__.py delete mode 100644 modelship/openai/parsers/tool_calling/input.py delete mode 100644 modelship/openai/parsers/tool_calling/parsers/__init__.py delete mode 100644 modelship/openai/parsers/tool_calling/parsers/base.py delete mode 100644 modelship/openai/parsers/tool_calling/parsers/gemma.py delete mode 100644 modelship/openai/parsers/tool_calling/parsers/hermes.py delete mode 100644 modelship/openai/parsers/tool_calling/parsers/llama3_json.py delete mode 100644 modelship/openai/parsers/tool_calling/parsers/mistral.py delete mode 100644 modelship/openai/parsers/tool_calling/parsers/qwen3_coder.py delete mode 100644 modelship/openai/parsers/tool_calling/registry.py delete mode 100644 tests/test_gemma4_parsers.py delete mode 100644 tests/test_mistral_specials_smoketest.py delete mode 100644 tests/test_qwen3_coder_parser.py delete mode 100644 tests/test_reasoning.py delete mode 100644 tests/test_tool_calling.py diff --git a/modelship/deploy/config.py b/modelship/deploy/config.py index b5e2505..658cabd 100644 --- a/modelship/deploy/config.py +++ b/modelship/deploy/config.py @@ -8,10 +8,8 @@ from modelship.infer.infer_config import ModelLoader, ModelshipConfig, ModelUsecase from modelship.infer.model_resolver import resolve_model_source from modelship.logging import get_logger -from modelship.openai.parsers.reasoning.registry import get_parser as get_reasoning_parser -from modelship.openai.parsers.reasoning.utils import classify_template as classify_reasoning_template -from modelship.openai.parsers.tool_calling.registry import available_parsers, get_parser -from modelship.openai.parsers.tool_calling.utils import classify_template +from modelship.openai.parsers.reasoning import classify_template as classify_reasoning_template +from modelship.openai.parsers.tool_calling import classify_template from modelship.openai.parsers.utils import read_chat_template logger = get_logger("startup") @@ -171,24 +169,28 @@ def resolve_all_tool_parsers(yml_conf: ModelshipConfig) -> None: truth and never re-implements the precedence. Auto-detection runs for vllm. llama_server does its own tool-call - detection internally, unrelated to this registry; diffusers has no chat - path; custom is plugin-managed. + detection internally; diffusers has no chat path; custom is plugin-managed. Behavior per model: - Loader-specific opt-out (see `_is_explicit_tool_opt_out`): leaves `_resolved_tool_call_parser` as None. - - Explicit parser name configured: validated against the registry, - stored on `_resolved_tool_call_parser`. Raises if unknown. + - Explicit parser name configured: validated against vLLM's own + `ToolParserManager`, stored on `_resolved_tool_call_parser`. Raises if + unknown. - Auto-detected, registered: stored on `_resolved_tool_call_parser`. - Auto-detected as `unknown` / known-but-unregistered: warn, leave None. - Not detected: leave None (no template tool-call affordance). """ - registered = set(available_parsers()) - for cfg in yml_conf.models: - if cfg.loader != ModelLoader.vllm: - continue - if cfg.usecase != ModelUsecase.generate: - continue + vllm_generate_cfgs = [ + cfg for cfg in yml_conf.models if cfg.loader == ModelLoader.vllm and cfg.usecase == ModelUsecase.generate + ] + if not vllm_generate_cfgs: + return + + from vllm.tool_parsers import ToolParserManager + + registered = set(ToolParserManager.list_registered()) + for cfg in vllm_generate_cfgs: if _is_explicit_tool_opt_out(cfg): logger.info("Tool-call resolution skipped for '%s' (explicit opt-out).", cfg.name) continue @@ -202,7 +204,6 @@ def resolve_all_tool_parsers(yml_conf: ModelshipConfig) -> None: f"which is not registered. Available: {sorted(registered) or '(none)'}." ) cfg._resolved_tool_call_parser = explicit - _merge_skip_specials(cfg, explicit, is_reasoning=False) logger.info("Using explicit tool_call_parser=%r for '%s'", explicit, cfg.name) continue @@ -230,33 +231,9 @@ def resolve_all_tool_parsers(yml_conf: ModelshipConfig) -> None: ) continue cfg._resolved_tool_call_parser = detected - _merge_skip_specials(cfg, detected, is_reasoning=False) logger.info("Auto-detected tool_call_parser=%r for '%s'", detected, cfg.name) -def _skip_specials_for(parser_name: str, is_reasoning: bool = False) -> bool | None: - """Resolve the ``skip_special_tokens`` setting a loader should use. - - Returns ``False`` when the parser declares ``markers_are_specials`` - (its marker is registered as a special token and would be stripped by - the loader's default detokenization) — the loader must keep specials - in the stream and noise-strip the rest itself. Returns ``None`` - otherwise so the loader keeps its own default (``True``). - """ - get_fn = get_reasoning_parser if is_reasoning else get_parser - return False if get_fn(parser_name).markers_are_specials else None - - -def _merge_skip_specials(cfg, parser_name: str, is_reasoning: bool = False) -> None: - """Update ``_resolved_skip_special_tokens`` if the parser requires it. - - Once set to ``False`` (keep specials), it is never reset to ``None``. - """ - if cfg._resolved_skip_special_tokens is False: - return - cfg._resolved_skip_special_tokens = _skip_specials_for(parser_name, is_reasoning=is_reasoning) - - def _is_explicit_reasoning_opt_out(cfg) -> bool: """Loader-specific explicit "no reasoning auto-detection" signal. @@ -295,7 +272,6 @@ def resolve_all_reasoning_parsers(yml_conf: ModelshipConfig) -> None: if explicit is not None: cfg._resolved_reasoning_parser = explicit - _merge_skip_specials(cfg, explicit, is_reasoning=True) logger.info("Using explicit reasoning_parser=%r for '%s'", explicit, cfg.name) continue @@ -312,5 +288,4 @@ def resolve_all_reasoning_parsers(yml_conf: ModelshipConfig) -> None: if detected is None: continue cfg._resolved_reasoning_parser = detected - _merge_skip_specials(cfg, detected, is_reasoning=True) logger.info("Auto-detected reasoning_parser=%r for '%s'", detected, cfg.name) diff --git a/modelship/infer/infer_config.py b/modelship/infer/infer_config.py index b0b06a0..3dee1f9 100644 --- a/modelship/infer/infer_config.py +++ b/modelship/infer/infer_config.py @@ -215,15 +215,6 @@ class ModelshipModelConfig(BaseModel): _resolved_tool_call_parser: str | None = PrivateAttr(default=None) _resolved_reasoning_parser: str | None = PrivateAttr(default=None) _resolved_chat_template: str | None = PrivateAttr(default=None) - # Pinned at startup from the resolved tool-call parser's - # ``markers_are_specials`` flag. Loaders that detokenize raw model - # output consult this to decide whether to flip - # ``skip_special_tokens=False`` — required for - # parsers like Mistral whose ``[TOOL_CALLS]`` marker is registered as a - # special token in the tokenizer and would otherwise be stripped before - # the parser sees it. ``None`` means the loader should keep its own - # default. - _resolved_skip_special_tokens: bool | None = PrivateAttr(default=None) @model_validator(mode="before") @classmethod diff --git a/modelship/openai/chat_utils.py b/modelship/openai/chat_utils.py index efa8633..c37e616 100644 --- a/modelship/openai/chat_utils.py +++ b/modelship/openai/chat_utils.py @@ -250,7 +250,6 @@ def build_from_parsed( elif isinstance(finish_reasons, str): fr = finish_reasons else: - # Fallback derivation logic similar to finish_reason_for fr = "tool_calls" if parsed.has_tool_calls else "stop" choice_logprobs = None diff --git a/modelship/openai/parsers/__init__.py b/modelship/openai/parsers/__init__.py index bac8f65..d9e38ac 100644 --- a/modelship/openai/parsers/__init__.py +++ b/modelship/openai/parsers/__init__.py @@ -1,21 +1,8 @@ -"""Cross-loader parser toolkit: reasoning + tool calls. +"""Chat-template parser-name detection, shared by driver preflight and the vllm loader. -Loaders import streaming helpers and the unified streamer/result -classes from this package; the per-family parsers live under the -``reasoning`` and ``tool_calling`` subpackages. +``tool_calling.py`` / ``reasoning.py`` classify a model's chat template into the +vLLM-native tool-call / reasoning parser name it needs; ``utils.py`` reads chat +templates off disk and renders minimal generation prompts. All three are consumed +directly by module path (``modelship.openai.parsers.{tool_calling,reasoning,utils}``) +rather than re-exported here. """ - -from modelship.openai.parsers.output import ChatOutputStreamer, ParsedChatOutput -from modelship.openai.parsers.streaming import ( - build_chat_completion_response, - finish_reason_for, - stream_chat_completion, -) - -__all__ = [ - "ChatOutputStreamer", - "ParsedChatOutput", - "build_chat_completion_response", - "finish_reason_for", - "stream_chat_completion", -] diff --git a/modelship/openai/parsers/output.py b/modelship/openai/parsers/output.py deleted file mode 100644 index d709cf7..0000000 --- a/modelship/openai/parsers/output.py +++ /dev/null @@ -1,402 +0,0 @@ -"""Single-pass chat-output streamer that handles reasoning + tool calls. - -This is the loader-shared driver consumed by every chat serving path -that emits raw text (transformers, llama.cpp, plugin-wrapped engines). -It walks the cumulative model output once and routes each region to -its appropriate stream: - -1. **Reasoning regions** — text inside the optional reasoning parser's - marker pair. Surfaced via ``DeltaMessage.reasoning``. -2. **Tool-call regions** — text inside the tool-call parser's marker - pair, with name/arguments extraction. Surfaced via - ``DeltaMessage.tool_calls``. -3. **Content regions** — everything else. Surfaced via - ``DeltaMessage.content``. - -The single-pass design matters: a ```` marker emitted inside -```` is part of the model's reasoning, not a real tool call. -Two chained streamers would parse it as a real call; one streamer that -knows about both marker kinds correctly routes it to the reasoning -view. Both the streaming and non-streaming paths run the same streamer -so behavior cannot drift between them. -""" - -from __future__ import annotations - -import re -from dataclasses import dataclass - -from modelship.openai.parsers.reasoning.parsers import ReasoningParser -from modelship.openai.parsers.tool_calling.parsers import ToolCallParser -from modelship.openai.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - FunctionCall, - ToolCall, - random_uuid, -) - - -@dataclass(frozen=True) -class ParsedChatOutput: - """Aggregate result of parsing a model's full chat-completion text. - - ``content`` carries the residual non-reasoning, non-tool-call text - once both kinds of markers are stripped. It is ``None`` when tool - calls were extracted *and* the residual is empty, matching OpenAI's - behavior of nulling ``content`` alongside ``tool_calls``. - ``reasoning`` is the concatenation of all ``...`` regions - (or whichever marker pair the reasoning parser declared); ``None`` - when reasoning was not enabled or no markers were emitted. - """ - - content: str | None - reasoning: str | None - tool_calls: list[ToolCall] - - @property - def has_tool_calls(self) -> bool: - return bool(self.tool_calls) - - -class ChatOutputStreamer: - """Per-request, stateful chat-output extractor. - - State held per request: - - - ``_sent_content_idx`` / ``_sent_reasoning_idx`` — chars already - shipped from each non-tool-call stream. - - ``_sent_name`` / ``_sent_id`` per tool index — whether the - function name and id deltas have been emitted. - - ``_sent_args`` per tool index — chars of arguments already - shipped (the suffix-diff cursor). - - ``_finalized_indices`` / ``_finalized_calls`` — closed tool-call - blocks accumulated for :attr:`result` and ``finish_reason``. - """ - - def __init__( - self, - tool_call_parser: ToolCallParser | None, - reasoning_parser: ReasoningParser | None = None, - noise_specials: tuple[str, ...] = (), - ): - if tool_call_parser is None and reasoning_parser is None: - raise ValueError("ChatOutputStreamer requires at least one parser (tool-call or reasoning)") - self._tool_parser = tool_call_parser - self._tool_start = tool_call_parser.start_marker if tool_call_parser else "" - self._tool_end = tool_call_parser.end_marker if tool_call_parser else "" - self._tool_consume_start = tool_call_parser.consume_start_marker if tool_call_parser else True - # Loader-supplied list of registered special tokens to silently - # drop before scanning. Used when the loader runs detokenization - # with ``skip_special_tokens=False`` to keep the parser's marker - # visible — every OTHER special (``<|im_end|>``, ``<|eot_id|>``, - # ``[INST]``, etc.) is junk from the client's perspective and - # must not leak into content/reasoning views. Sorted long-first - # so a longer marker is replaced before a substring of it can - # cause a false partial match. - self._noise_specials: tuple[str, ...] = tuple(sorted(noise_specials, key=len, reverse=True)) - if self._noise_specials: - pattern = "|".join(re.escape(s) for s in self._noise_specials) - self._noise_regex = re.compile(pattern) - else: - self._noise_regex = None - - self._reasoning = reasoning_parser - self._reasoning_start = reasoning_parser.start_marker if reasoning_parser else "" - self._reasoning_end = reasoning_parser.end_marker if reasoning_parser else "" - self._sent_content_idx = 0 - self._sent_reasoning_idx = 0 - self._sent_name: list[bool] = [] - self._sent_id: list[str] = [] - self._sent_args: list[str] = [] - self._finalized_indices: set[int] = set() - self._finalized_calls: list[ToolCall] = [] - self._last_raw_text = "" - self._stripped_head = "" - self._unstripped_tail = "" - self._last_text = "" - - def extract_streaming(self, current_text: str) -> DeltaMessage | None: - """Run one diff pass against ``current_text`` and return any new deltas.""" - delta_text = current_text[len(self._last_raw_text) :] - self._last_raw_text = current_text - - current_text = self._strip_noise_delta(delta_text) - self._last_text = current_text - - regions = self._scan(current_text, hold_marker_tail=True) - content_delta = self._diff_content(regions) - reasoning_delta = self._diff_reasoning(regions) - tool_call_deltas = self._emit_new_tool_call_fragments(regions) - if not content_delta and not reasoning_delta and not tool_call_deltas: - return None - return DeltaMessage(content=content_delta, reasoning=reasoning_delta, tool_calls=tool_call_deltas) - - def finalize(self) -> DeltaMessage | None: - """Flush any held-back tails once no more text is coming. - - The mid-stream holdback can leave a tool call's closing marker - unscanned until this final pass — so we re-run tool-call - extraction here too, finalizing any call whose terminator only - now resolves. This is idempotent: ``_finalized_indices`` blocks - re-finalizing and ``_sent_name`` / ``_sent_args`` block - re-emitting deltas for calls already streamed. - """ - regions = self._scan(self._last_text, hold_marker_tail=False) - content_delta = self._diff_content(regions) - reasoning_delta = self._diff_reasoning(regions) - tool_call_deltas = self._emit_new_tool_call_fragments(regions) - if content_delta is None and reasoning_delta is None and not tool_call_deltas: - return None - return DeltaMessage(content=content_delta, reasoning=reasoning_delta, tool_calls=tool_call_deltas) - - @property - def result(self) -> ParsedChatOutput: - """Final view, suitable for the non-streaming response shape.""" - regions = self._scan(self._last_text, hold_marker_tail=False) - content_view = "".join(payload for kind, payload, _ in regions if kind == "content") - reasoning_view = "".join(payload for kind, payload, _ in regions if kind == "reasoning") - content = (content_view if content_view.strip() else None) if self._finalized_calls else (content_view or None) - return ParsedChatOutput( - content=content, - reasoning=reasoning_view or None, - tool_calls=list(self._finalized_calls), - ) - - # ------------------------------------------------------------------ - # Single-pass scanner - # ------------------------------------------------------------------ - - def _strip_noise_delta(self, delta_text: str) -> str: - """Drop loader-supplied noise specials from new text deltas. - - Maintains state to handle noise tokens split across chunks. - """ - if self._noise_regex is None: - self._stripped_head += delta_text - return self._stripped_head - - # 1. Combine the held-back tail from the last chunk with the new chunk - working_text = self._unstripped_tail + delta_text - - # 2. Strip noise from this combined window using regex - working_text = self._noise_regex.sub("", working_text) - - # 3. Determine how much of the working text is "safe" to commit. - # The unsafe portion is the end of the string that might form a prefix - # of any noise token. - safe_idx = len(working_text) - for s in self._noise_specials: - overlap = _safe_overlap_index(working_text, s) - safe_idx = min(safe_idx, overlap) - - # 4. Commit the safe part and hold back the unsafe tail - self._stripped_head += working_text[:safe_idx] - self._unstripped_tail = working_text[safe_idx:] - - # 5. Return the full accumulated string for the scanner - return self._stripped_head + self._unstripped_tail - - def _scan(self, text: str, *, hold_marker_tail: bool) -> list[tuple[str, str, bool]]: - """Walk ``text`` once, returning ordered ``(kind, payload, is_complete)`` regions. - - ``kind`` is one of ``"content"``, ``"reasoning"``, ``"tool_call"``. - ``is_complete`` is meaningful only for ``"tool_call"``; for the - other kinds it is always ``True`` (their boundaries are handled - via the marker-tail holdback in the content/reasoning views). - - ``hold_marker_tail`` controls mid-stream holdback: trailing - bytes that could be the prefix of any expected opening or - closing marker are withheld, so the client never sees a - fragment of a tag forwarded as content/reasoning bytes. At - finalize time we know no more text is coming, so all held - tails are flushed. - """ - regions: list[tuple[str, str, bool]] = [] - pos = 0 - n = len(text) - while pos < n: - tool_start = text.find(self._tool_start, pos) if self._tool_start else -1 - reason_start = text.find(self._reasoning_start, pos) if self._reasoning_start else -1 - next_starts = [s for s in (tool_start, reason_start) if s >= 0] - if not next_starts: - # No more openings ahead; the rest is content. - remainder = text[pos:] - if hold_marker_tail: - safe = self._safe_outside_flush_index(remainder) - regions.append(("content", remainder[:safe], True)) - else: - regions.append(("content", remainder, True)) - break - - next_pos = min(next_starts) - if next_pos > pos: - regions.append(("content", text[pos:next_pos], True)) - - if next_pos == reason_start: - payload_start = next_pos + len(self._reasoning_start) - end = text.find(self._reasoning_end, payload_start) if self._reasoning_end else -1 - if end < 0: - inner = text[payload_start:] - if hold_marker_tail: - safe = _safe_overlap_index(inner, self._reasoning_end) - regions.append(("reasoning", inner[:safe], False)) - else: - regions.append(("reasoning", inner, False)) - break - regions.append(("reasoning", text[payload_start:end], True)) - pos = end + len(self._reasoning_end) - else: - # ``consume_start_marker=False`` keeps the marker bytes at the - # head of the payload — used by JSON-shape parsers whose marker - # (e.g. ``{"name"``) is itself part of the object to parse. - consume = self._tool_consume_start - payload_start = next_pos + (len(self._tool_start) if consume else 0) - end = text.find(self._tool_end, payload_start) if self._tool_end else -1 - if end < 0: - partial = text[payload_start:] - if hold_marker_tail: - # No end marker → region extends to EOS; nothing to hold back. - safe = _safe_overlap_index(partial, self._tool_end) if self._tool_end else len(partial) - regions.append(("tool_call", partial[:safe], False)) - else: - # Finalize: with an end marker we never saw it (unterminated, leave incomplete); - # without an end marker the region is complete by design. - regions.append(("tool_call", partial, not self._tool_end)) - break - regions.append(("tool_call", text[payload_start:end], True)) - pos = end + len(self._tool_end) - return regions - - def _safe_outside_flush_index(self, buf: str) -> int: - """Index up to which ``buf`` (a content tail) can be flushed. - - ``buf`` does not contain a full opening marker; the unsafe tail - is the longest proper-prefix overlap with whichever opening - marker(s) the streamer is watching for. Noise specials count too - — a trailing ``<|eot_i`` could become ``<|eot_id|>`` on the next - pass, and we must not ship the partial as content before the - cumulative buffer disambiguates. - """ - cap = len(buf) - for marker in (self._tool_start, self._reasoning_start, *self._noise_specials): - if not marker: - continue - cap = min(cap, _safe_overlap_index(buf, marker)) - return cap - - # ------------------------------------------------------------------ - # Content + reasoning diff - # ------------------------------------------------------------------ - - def _diff_content(self, regions: list[tuple[str, str, bool]]) -> str | None: - view = "".join(payload for kind, payload, _ in regions if kind == "content") - if len(view) <= self._sent_content_idx: - return None - new = view[self._sent_content_idx :] - self._sent_content_idx = len(view) - return new or None - - def _diff_reasoning(self, regions: list[tuple[str, str, bool]]) -> str | None: - view = "".join(payload for kind, payload, _ in regions if kind == "reasoning") - if len(view) <= self._sent_reasoning_idx: - return None - new = view[self._sent_reasoning_idx :] - self._sent_reasoning_idx = len(view) - return new or None - - # ------------------------------------------------------------------ - # Tool-call fragments - # ------------------------------------------------------------------ - - def _emit_new_tool_call_fragments(self, regions: list[tuple[str, str, bool]]) -> list[DeltaToolCall]: - if self._tool_parser is None: - return [] - deltas: list[DeltaToolCall] = [] - tool_blocks = [(payload, complete) for kind, payload, complete in regions if kind == "tool_call"] - # Flatten across regions and sub-blocks. Most parsers map one region to - # one call (default `split_payload`); families like Mistral wrap N calls - # in a single envelope and split them out here. The flat index is what - # the OpenAI ``tool_calls[i]`` slot uses. - flat_index = 0 - stop = False - for region_payload, region_complete in tool_blocks: - if stop: - break - for sub_payload, sub_complete in self._tool_parser.split_payload(region_payload, region_complete): - i = flat_index - self._ensure_slot(i) - - if not self._sent_name[i]: - name = self._tool_parser.extract_partial_name(sub_payload) - if name is None: - if sub_complete: - # Closed but malformed (no extractable name) — skip, - # but still process later sub-blocks. - flat_index += 1 - continue - # Mid-stream and no name yet; per OpenAI convention we don't - # advance to later blocks until the current one has a name. - stop = True - break - tool_id = f"chatcmpl-tool-{random_uuid()}" - self._sent_name[i] = True - self._sent_id[i] = tool_id - deltas.append( - DeltaToolCall( - index=i, - id=tool_id, - type="function", - function=DeltaFunctionCall(name=name), - ) - ) - - args = self._tool_parser.extract_partial_args(sub_payload, is_complete=sub_complete) - if args is not None and len(args) > len(self._sent_args[i]): - diff = args[len(self._sent_args[i]) :] - self._sent_args[i] = args - deltas.append( - DeltaToolCall( - index=i, - function=DeltaFunctionCall(arguments=diff), - ) - ) - - if sub_complete and i not in self._finalized_indices and self._sent_name[i]: - self._finalized_indices.add(i) - self._finalized_calls.append( - ToolCall( - id=self._sent_id[i], - type="function", - function=FunctionCall( - name=self._tool_parser.extract_partial_name(sub_payload) or "", - arguments=self._sent_args[i], - ), - ) - ) - flat_index += 1 - return deltas - - def _ensure_slot(self, i: int) -> None: - while len(self._sent_name) <= i: - self._sent_name.append(False) - self._sent_id.append("") - self._sent_args.append("") - - -def _safe_overlap_index(buf: str, marker: str) -> int: - """Index up to which ``buf`` can be flushed without risking a split marker. - - ``buf`` is known not to contain the full marker. The unsafe tail is - the longest proper-prefix overlap between ``buf`` and ``marker``: if - the next chunk completes that prefix, we'd have to retract bytes we - already streamed. Holding them back avoids the retraction. - """ - if not marker: - return len(buf) - max_overlap = min(len(buf), len(marker) - 1) - for k in range(max_overlap, 0, -1): - if buf.endswith(marker[:k]): - return len(buf) - k - return len(buf) diff --git a/modelship/openai/parsers/reasoning/utils.py b/modelship/openai/parsers/reasoning.py similarity index 60% rename from modelship/openai/parsers/reasoning/utils.py rename to modelship/openai/parsers/reasoning.py index 5778fb4..6805e57 100644 --- a/modelship/openai/parsers/reasoning/utils.py +++ b/modelship/openai/parsers/reasoning.py @@ -1,26 +1,28 @@ -"""Reasoning parser detection by chat-template inspection.""" +"""Reasoning parser name detection by chat-template inspection. + +Auto-detects which vLLM-native reasoning parser (``vllm.reasoning.ReasoningParserManager``) +a model's chat template calls for, by marker sniffing. +""" from __future__ import annotations from collections.abc import Callable -from pathlib import Path from modelship.logging import get_logger -from modelship.openai.parsers.utils import read_chat_template logger = get_logger("openai.reasoning.detect") - -def detect_reasoning_parser(model_path: str | Path) -> str | None: - """Return the name of the reasoning parser the model needs, or None.""" - template = read_chat_template(model_path) - if template is None: - return None - return classify_template(template) +# (start_marker, end_marker) for the two auto-detectable families, used by +# resolve_active_reasoning_parser to probe whether a deployment's rendered +# prompt shows reasoning pre-suppressed (e.g. Qwen3 enable_thinking=false). +_MARKERS: dict[str, tuple[str, str]] = { + "deepseek_r1": ("", ""), + "gemma4": ("<|channel>thought\n", ""), +} def classify_template(template: str) -> str | None: - """Map a chat-template string to a reasoning parser name based on markers.""" + """Map a chat-template string to a vLLM reasoning parser name based on markers.""" if "<|channel>thought" in template: return "gemma4" if "" in template or "" in template: @@ -57,24 +59,28 @@ def reasoning_active_in_render(rendered_prompt: str, start_marker: str, end_mark def resolve_active_reasoning_parser(candidate: str | None, render_prompt: Callable[[], str]) -> str | None: """Downgrade a candidate reasoning parser to ``None`` if it is suppressed. - ``candidate`` is the parser detected from the template *markers* (capability). - ``render_prompt`` renders a generation prompt through the loader's real render - path, so the deployment's effective ``chat_template_kwargs`` are already applied. - Returns the candidate when reasoning is (or might be) active, ``None`` when the - render shows positive evidence of suppression. Falls back to the candidate if the - render raises — never invents a parser. + ``candidate`` is the parser detected from the template *markers* (capability), + or an explicit user-configured vLLM reasoning parser name. ``render_prompt`` + renders a generation prompt through the loader's real render path, so the + deployment's effective ``chat_template_kwargs`` are already applied. Returns + the candidate when reasoning is (or might be) active, ``None`` when the render + shows positive evidence of suppression. Falls back to the candidate if no known + markers exist for it, or if the render raises — never invents a parser. """ if candidate is None: return None - from modelship.openai.parsers.reasoning.registry import get_parser - + markers = _MARKERS.get(candidate) + if markers is None: + # No known start/end markers for this name (e.g. an explicitly configured + # vLLM reasoning parser modelship doesn't auto-detect) — nothing to probe. + return candidate + start_marker, end_marker = markers try: - parser = get_parser(candidate) rendered = render_prompt() except Exception as exc: logger.warning("reasoning probe failed (%s); keeping reasoning_parser=%r", exc, candidate) return candidate - if reasoning_active_in_render(rendered, parser.start_marker, parser.end_marker): + if reasoning_active_in_render(rendered, start_marker, end_marker): return candidate logger.info("reasoning_parser %r disabled: render shows reasoning suppressed for this deployment", candidate) return None diff --git a/modelship/openai/parsers/reasoning/__init__.py b/modelship/openai/parsers/reasoning/__init__.py deleted file mode 100644 index 2d0d964..0000000 --- a/modelship/openai/parsers/reasoning/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Cross-loader reasoning parser toolkit. - -Loaders without native reasoning support (transformers, llama.cpp) use -the parsers here, dispatched via the unified -:class:`ChatOutputStreamer`, to surface ``...`` blocks -in the protocol-level ``reasoning`` field. vLLM has its own built-in -reasoning parsers and uses only the auto-detected parser name. -""" - -from modelship.openai.parsers.reasoning.parsers import DeepseekR1ReasoningParser, ReasoningParser -from modelship.openai.parsers.reasoning.registry import available_parsers, get_parser, register_parser -from modelship.openai.parsers.reasoning.utils import ( - classify_template, - detect_reasoning_parser, - reasoning_active_in_render, - resolve_active_reasoning_parser, -) - -__all__ = [ - "DeepseekR1ReasoningParser", - "ReasoningParser", - "available_parsers", - "classify_template", - "detect_reasoning_parser", - "get_parser", - "reasoning_active_in_render", - "register_parser", - "resolve_active_reasoning_parser", -] diff --git a/modelship/openai/parsers/reasoning/parsers/__init__.py b/modelship/openai/parsers/reasoning/parsers/__init__.py deleted file mode 100644 index 5eb8bda..0000000 --- a/modelship/openai/parsers/reasoning/parsers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from modelship.openai.parsers.reasoning.parsers.base import ReasoningParser -from modelship.openai.parsers.reasoning.parsers.deepseek import DeepseekR1ReasoningParser -from modelship.openai.parsers.reasoning.parsers.gemma import Gemma4ReasoningParser - -__all__ = ["DeepseekR1ReasoningParser", "Gemma4ReasoningParser", "ReasoningParser"] diff --git a/modelship/openai/parsers/reasoning/parsers/base.py b/modelship/openai/parsers/reasoning/parsers/base.py deleted file mode 100644 index 92c4109..0000000 --- a/modelship/openai/parsers/reasoning/parsers/base.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Base class for model-family-specific reasoning output parsers. - -Reasoning parsers are *marker-based*: each family wraps its -chain-of-thought in a fixed pair of literal strings (```` / -```` for DeepSeek-R1, QwQ, Qwen3, Phi-4-reasoning, ...). -Unlike tool-call parsers, the payload between markers is opaque text — -no per-family extractors are needed, just the marker pair. - -The unified :class:`ChatOutputStreamer` (over in -``modelship.openai.parsers.tool_calling.parsers``) consumes a parser -instance and walks the cumulative model output once, splitting it into -content / reasoning / tool-call regions in a single pass. -""" - -from __future__ import annotations - -from abc import ABC - - -class ReasoningParser(ABC): - """Family-specific marker pair for reasoning extraction. - - Subclasses set ``name``, ``start_marker``, ``end_marker``. They hold - no per-request state — that all lives on :class:`ChatOutputStreamer`. - """ - - name: str - start_marker: str - end_marker: str - # Set True when the parser's marker(s) are registered as *special tokens* - # in the tokenizers of the model families this parser targets. Loaders - # that decode with ``skip_special_tokens=True`` by default would - # otherwise strip the marker before this parser ever sees it. - markers_are_specials: bool = False diff --git a/modelship/openai/parsers/reasoning/parsers/deepseek.py b/modelship/openai/parsers/reasoning/parsers/deepseek.py deleted file mode 100644 index a4af28a..0000000 --- a/modelship/openai/parsers/reasoning/parsers/deepseek.py +++ /dev/null @@ -1,16 +0,0 @@ -"""DeepSeek-R1-style ``...`` reasoning parser. - -Used by DeepSeek-R1 (and distilled variants), QwQ, Qwen3, and -Phi-4-reasoning, all of which wrap their chain-of-thought in the -literal tags ```` / ````. -""" - -from __future__ import annotations - -from modelship.openai.parsers.reasoning.parsers.base import ReasoningParser - - -class DeepseekR1ReasoningParser(ReasoningParser): - name = "deepseek_r1" - start_marker = "" - end_marker = "" diff --git a/modelship/openai/parsers/reasoning/parsers/gemma.py b/modelship/openai/parsers/reasoning/parsers/gemma.py deleted file mode 100644 index bb853e9..0000000 --- a/modelship/openai/parsers/reasoning/parsers/gemma.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Gemma 4 "Thinking Mode" reasoning parser. - -Gemma 4 uses a "channel" mechanism to separate internal reasoning from -the final answer. Reasoning is wrapped in `<|channel>thought\\n...`. -""" - -from __future__ import annotations - -from modelship.openai.parsers.reasoning.parsers.base import ReasoningParser - - -class Gemma4ReasoningParser(ReasoningParser): - name = "gemma4" - # We include 'thought\n' in the start marker so ChatOutputStreamer - # strips it from the reasoning payload, consistent with vLLM's - # implementation. - start_marker = "<|channel>thought\n" - end_marker = "" - markers_are_specials = True diff --git a/modelship/openai/parsers/reasoning/registry.py b/modelship/openai/parsers/reasoning/registry.py deleted file mode 100644 index b325967..0000000 --- a/modelship/openai/parsers/reasoning/registry.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Registry of named reasoning parsers, dispatched by configuration string. - -Mirrors ``modelship.openai.parsers.tool_calling.registry``: loaders look -up a reasoning parser by the name resolved on the driver -(``_resolved_reasoning_parser``) and hand it to the unified -``ChatOutputStreamer``. vLLM bypasses this registry — it has its own -built-in reasoning parsers and consumes only the resolved name. -""" - -from __future__ import annotations - -from modelship.openai.parsers.reasoning.parsers import ( - DeepseekR1ReasoningParser, - Gemma4ReasoningParser, - ReasoningParser, -) - -_PARSERS: dict[str, ReasoningParser] = { - DeepseekR1ReasoningParser.name: DeepseekR1ReasoningParser(), - Gemma4ReasoningParser.name: Gemma4ReasoningParser(), -} - - -def get_parser(name: str) -> ReasoningParser: - """Return the parser registered under ``name`` or raise ``ValueError``.""" - try: - return _PARSERS[name] - except KeyError: - available = ", ".join(sorted(_PARSERS)) or "(none)" - raise ValueError(f"unknown reasoning_parser {name!r}; available: {available}") from None - - -def available_parsers() -> list[str]: - return sorted(_PARSERS) - - -def register_parser(parser: ReasoningParser) -> None: - """Register an additional parser. Intended for tests and plugin code.""" - _PARSERS[parser.name] = parser diff --git a/modelship/openai/parsers/streaming.py b/modelship/openai/parsers/streaming.py deleted file mode 100644 index f57d0bd..0000000 --- a/modelship/openai/parsers/streaming.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Loader-agnostic chat-completion helpers for parser-aware responses. - -Loaders produce text differently — HF ``Pipeline`` for transformers, async -iterators from plugins — but the OpenAI response shapes (streaming and non-streaming) are the -same. This module owns those shapes plus the -:class:`~modelship.openai.parsers.output.ChatOutputStreamer` driving -loop, so the loaders only deal in plain text. - -Two helpers: - -- :func:`build_chat_completion_response` — non-streaming. Loader hands in - the full completion text and token counts; we parse reasoning and - tool calls and pack the OpenAI ``ChatCompletionResponse``. -- :func:`stream_chat_completion` — streaming. Loader hands in an - ``AsyncIterator[str]`` of new text pieces; we emit the SSE byte - stream. - -Both accept ``parser_name`` (tool-call) and ``reasoning_parser_name`` -independently. Either or both may be ``None``. -""" - -from __future__ import annotations - -import asyncio -import json -from collections.abc import AsyncIterator, Callable - -from modelship.logging import TRACE, get_logger -from modelship.openai.parsers.output import ChatOutputStreamer, ParsedChatOutput -from modelship.openai.parsers.reasoning.registry import get_parser as get_reasoning_parser -from modelship.openai.parsers.tool_calling.registry import get_parser as get_tool_call_parser -from modelship.openai.protocol import ( - ChatCompletionResponse, - ChatCompletionResponseChoice, - ChatCompletionResponseStreamChoice, - ChatCompletionStreamResponse, - ChatMessage, - DeltaMessage, - UsageInfo, -) - -logger = get_logger("openai.parsers.streaming") - - -def _log_tool_calls(request_id: str, parsed: ParsedChatOutput, finish_reason: str, completion_tokens: int) -> None: - """Log the structured tool calls handed to the client. - - Loaders already TRACE-log the model's raw text; this logs the *parsed* - OpenAI tool calls so it's visible exactly what a downstream client (e.g. - Home Assistant) receives and tries to execute — the key diagnostic when a - client rejects an otherwise-200 response. - """ - if not parsed.tool_calls or not logger.isEnabledFor(TRACE): - return - summary = "; ".join(f"{tc.function.name}({tc.function.arguments})" for tc in parsed.tool_calls) - logger.log( - TRACE, - "chat %s -> client: %d tool call(s) [finish_reason=%s, completion_tokens=%s]: %s", - request_id, - len(parsed.tool_calls), - finish_reason, - completion_tokens, - summary, - ) - - -def finish_reason_for(parsed: ParsedChatOutput, completion_tokens: int, max_tokens: int | None) -> str: - """Compute the OpenAI ``finish_reason`` for a chat completion.""" - if parsed.has_tool_calls: - return "tool_calls" - if max_tokens is not None and completion_tokens >= max_tokens: - return "length" - return "stop" - - -def build_chat_completion_response( - *, - request_id: str, - model_name: str, - text: str, - parser_name: str | None, - reasoning_parser_name: str | None = None, - noise_specials: tuple[str, ...] = (), - prompt_tokens: int, - completion_tokens: int, - max_tokens: int | None, - created: int, -) -> ChatCompletionResponse: - """Parse ``text`` and pack it into an OpenAI ``ChatCompletionResponse``. - - Non-streaming counterpart of :func:`stream_chat_completion`. When - both parser names are ``None``, ``text`` becomes the message - content as-is with no extraction. ``noise_specials`` is forwarded to - the streamer so loaders that decode with ``skip_special_tokens=False`` - can have unwanted special tokens (``<|eot_id|>``, ``[INST]``, etc.) - silently dropped before parsing. - """ - parsed = parse_chat_completion_text( - text, - parser_name=parser_name, - reasoning_parser_name=reasoning_parser_name, - noise_specials=noise_specials, - ) - finish_reason = finish_reason_for(parsed, completion_tokens, max_tokens) - _log_tool_calls(request_id, parsed, finish_reason, completion_tokens) - return ChatCompletionResponse( - id=request_id, - model=model_name, - choices=[ - ChatCompletionResponseChoice( - index=0, - message=ChatMessage( - role="assistant", - content=parsed.content, - reasoning=parsed.reasoning, - tool_calls=parsed.tool_calls, - ), - finish_reason=finish_reason, - ) - ], - usage=UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ), - created=created, - ) - - -async def stream_chat_completion( - *, - request_id: str, - model_name: str, - text_chunks: AsyncIterator[str], - parser_name: str | None, - reasoning_parser_name: str | None = None, - noise_specials: tuple[str, ...] = (), - count_tokens: Callable[[str], int], - prompt_tokens: int, - max_tokens: int | None, - include_usage: bool, - created: int, -) -> AsyncIterator[str]: - """Drive the OpenAI streaming-response protocol from a stream of text pieces. - - Yields SSE strings ready to be forwarded to the client. When either - ``parser_name`` or ``reasoning_parser_name`` is set, the cumulative - buffer is fed to a :class:`ChatOutputStreamer` so newly-emittable - reasoning, content, and tool-call fragments show up in deltas - without a half-formed marker ever reaching the client. When both - are ``None``, each chunk is forwarded as a content delta unchanged. - - ``count_tokens`` is consulted only at end-of-stream to compute the - completion-token count for ``finish_reason`` and (optionally) - usage; pass ``lambda _: 0`` if the loader doesn't have a tokenizer - handy. - """ - streamer = _make_streamer( - parser_name=parser_name, - reasoning_parser_name=reasoning_parser_name, - noise_specials=noise_specials, - ) - accumulated = "" - - yield _delta_chunk(request_id, model_name, DeltaMessage(role="assistant"), created) - - async for piece in text_chunks: - if not piece: - continue - accumulated += piece - if streamer is None: - yield _delta_chunk(request_id, model_name, DeltaMessage(content=piece), created) - await asyncio.sleep(0) - continue - delta = streamer.extract_streaming(accumulated) - if delta is not None: - yield _delta_chunk(request_id, model_name, delta, created) - await asyncio.sleep(0) - - if streamer is not None: - final = streamer.finalize() - if final is not None: - yield _delta_chunk(request_id, model_name, final, created) - parsed = streamer.result - else: - parsed = ParsedChatOutput(content=accumulated or None, reasoning=None, tool_calls=[]) - - completion_tokens = count_tokens(accumulated) - finish_reason = finish_reason_for(parsed, completion_tokens, max_tokens) - _log_tool_calls(request_id, parsed, finish_reason, completion_tokens) - - yield _encode_chunk( - ChatCompletionStreamResponse( - id=request_id, - model=model_name, - choices=[ - ChatCompletionResponseStreamChoice( - index=0, - delta=DeltaMessage(), - finish_reason=finish_reason, - ) - ], - created=created, - ) - ) - - if include_usage: - yield _encode_chunk( - ChatCompletionStreamResponse( - id=request_id, - model=model_name, - choices=[], - usage=UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ), - created=created, - ) - ) - - yield "data: [DONE]\n\n" - - -def parse_chat_completion_text( - text: str, - *, - parser_name: str | None, - reasoning_parser_name: str | None, - noise_specials: tuple[str, ...] = (), -) -> ParsedChatOutput: - """Run the shared :class:`ChatOutputStreamer` over a full completion text. - - Public so a non-streaming loader that parses text itself can feed the - result into :func:`modelship.openai.chat_utils.build_from_parsed` rather - than going through :func:`build_chat_completion_response`. - """ - streamer = _make_streamer( - parser_name=parser_name, - reasoning_parser_name=reasoning_parser_name, - noise_specials=noise_specials, - ) - if streamer is None: - return ParsedChatOutput(content=text or None, reasoning=None, tool_calls=[]) - streamer.extract_streaming(text) - streamer.finalize() - return streamer.result - - -def _make_streamer( - *, - parser_name: str | None, - reasoning_parser_name: str | None, - noise_specials: tuple[str, ...] = (), -) -> ChatOutputStreamer | None: - if parser_name is None and reasoning_parser_name is None: - return None - tool_parser = get_tool_call_parser(parser_name) if parser_name else None - reasoning_parser = get_reasoning_parser(reasoning_parser_name) if reasoning_parser_name else None - return ChatOutputStreamer(tool_parser, reasoning_parser, noise_specials=noise_specials) - - -def _delta_chunk(request_id: str, model_name: str, delta: DeltaMessage, created: int) -> str: - return _encode_chunk( - ChatCompletionStreamResponse( - id=request_id, - model=model_name, - choices=[ChatCompletionResponseStreamChoice(index=0, delta=delta)], - created=created, - ) - ) - - -def _encode_chunk(chunk: ChatCompletionStreamResponse) -> str: - return f"data: {json.dumps(chunk.model_dump(mode='json'))}\n\n" diff --git a/modelship/openai/parsers/tool_calling/utils.py b/modelship/openai/parsers/tool_calling.py similarity index 69% rename from modelship/openai/parsers/tool_calling/utils.py rename to modelship/openai/parsers/tool_calling.py index 4a6f753..bd6d752 100644 --- a/modelship/openai/parsers/tool_calling/utils.py +++ b/modelship/openai/parsers/tool_calling.py @@ -1,17 +1,19 @@ -"""Tool-call parser detection by chat-template inspection.""" +"""Tool-call parser name detection by chat-template inspection. -from __future__ import annotations - -from pathlib import Path +Auto-detects which vLLM-native tool-call parser (``vllm.tool_parsers.ToolParserManager``) +a model's chat template calls for, by marker sniffing. Names returned here must match +vLLM's own registered parser names exactly — ``deploy/config.py`` validates them against +vLLM's registry directly rather than a modelship-side one. +""" -from modelship.openai.parsers.utils import read_chat_template +from __future__ import annotations -def detect_tool_parser(model_path: str | Path) -> str | None: - """Return the name of the tool-call parser required by the model or None. +def classify_template(template: str) -> str | None: + """Map a chat-template string to a vLLM tool-call parser name based on markers. - Returns "gemma4" if `<|tool_call>` markers are found. - - Returns "function_gemma" if `` markers are found. + - Returns "functiongemma" if `` markers are found. - Returns "qwen3_coder" if `` markers are found. - Returns "mistral" if `[TOOL_CALLS]` markers are found. @@ -19,14 +21,6 @@ def detect_tool_parser(model_path: str | Path) -> str | None: - Returns "unknown" if tool logic is found but format markers are not recognized. - Returns ``None`` if no chat template / no tool logic is detected. """ - template = read_chat_template(model_path) - if template is None: - return None - return classify_template(template) - - -def classify_template(template: str) -> str | None: - """Map a chat-template string to a parser name based on tool-call markers.""" if "tools" not in template and "tool_calls" not in template and "function" not in template: return None @@ -34,7 +28,7 @@ def classify_template(template: str) -> str | None: return "gemma4" if "" in template: - return "function_gemma" + return "functiongemma" # Qwen3-Coder shares the ```` envelope with Hermes but the # body is XML rather than JSON. The `` list[dict[str, Any]] | None: - """Apply OpenAI ``tool_choice`` semantics to the request's ``tools`` list. - - Returns the list of tools to render into the prompt, or ``None`` when - the request should be served without any tool-calling affordance. - - - ``tool_choice == "none"`` — suppress tools entirely. - - ``tool_choice == "auto"`` / ``"required"`` (or unset) — pass all tools - through. Whether ``"required"`` is enforced is the loader's call (see - :func:`request_forces_tool_call`), not this helper's. - - ``tool_choice == {"type": "function", "function": {"name": "X"}}`` — - filter ``tools`` to that single function. - """ - if not tools: - return None - if tool_choice in (None, "auto", "required"): - return tools - if tool_choice == "none": - return None - if isinstance(tool_choice, dict): - fn = tool_choice.get("function") or {} - name = fn.get("name") if isinstance(fn, dict) else None - if isinstance(name, str) and name: - filtered = [t for t in tools if (t.get("function") or {}).get("name") == name] - if not filtered: - logger.warning("tool_choice names function %r which is not in the request's tools list", name) - return tools - return filtered - logger.warning("unrecognized tool_choice value %r; falling back to 'auto' semantics", tool_choice) - return tools - - -def request_forces_tool_call(tool_choice: str | dict[str, Any] | None) -> bool: - """Whether ``tool_choice`` mandates a tool call (vs. allowing free text). - - True for ``"required"`` and the ``{"type": "function", "function": - {"name": "X"}}`` named-function form. Loaders use this to pick the - tool-only grammar root. - """ - if tool_choice == "required": - return True - if isinstance(tool_choice, dict): - fn = tool_choice.get("function") or {} - name = fn.get("name") if isinstance(fn, dict) else None - return isinstance(name, str) and bool(name) - return False diff --git a/modelship/openai/parsers/tool_calling/parsers/__init__.py b/modelship/openai/parsers/tool_calling/parsers/__init__.py deleted file mode 100644 index f6f444d..0000000 --- a/modelship/openai/parsers/tool_calling/parsers/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from modelship.openai.parsers.tool_calling.parsers.base import ToolCallParser -from modelship.openai.parsers.tool_calling.parsers.gemma import ( - FunctionGemmaToolCallParser, - Gemma4ToolCallParser, -) -from modelship.openai.parsers.tool_calling.parsers.hermes import HermesToolCallParser -from modelship.openai.parsers.tool_calling.parsers.llama3_json import Llama3JsonToolCallParser -from modelship.openai.parsers.tool_calling.parsers.mistral import MistralToolCallParser -from modelship.openai.parsers.tool_calling.parsers.qwen3_coder import Qwen3CoderToolCallParser - -__all__ = [ - "FunctionGemmaToolCallParser", - "Gemma4ToolCallParser", - "HermesToolCallParser", - "Llama3JsonToolCallParser", - "MistralToolCallParser", - "Qwen3CoderToolCallParser", - "ToolCallParser", -] diff --git a/modelship/openai/parsers/tool_calling/parsers/base.py b/modelship/openai/parsers/tool_calling/parsers/base.py deleted file mode 100644 index e98c854..0000000 --- a/modelship/openai/parsers/tool_calling/parsers/base.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Base class for model-family-specific tool-call output parsers. - -Parsers in this codebase are *marker-based*: each family wraps tool-call -JSON in a fixed pair of literal strings (```` / ```` -for Hermes, ``[TOOL_CALLS]`` / closing token for Mistral, ...). A subclass -declares the marker pair plus two small extractors that pick a function -name and an arguments substring out of the (possibly partial) JSON between -the markers. - -The cumulative-text streaming and non-streaming paths both run through -:class:`~modelship.openai.parsers.output.ChatOutputStreamer`, which -consumes a parser instance plus an optional reasoning parser and walks -the model output once. -""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from modelship.openai.parsers.output import ParsedChatOutput - - -class ToolCallParser(ABC): - """Family-specific knobs the streamer needs to drive its diff loop. - - Subclasses set ``start_marker`` / ``end_marker`` and implement the two - extractors. They never touch streaming state — that lives on - :class:`~modelship.openai.parsers.output.ChatOutputStreamer`, which is - instantiated once per request. - """ - - name: str - start_marker: str - end_marker: str - # When False, the streamer locates ``start_marker`` but does NOT skip past - # it: the marker bytes stay at the head of the payload fed to the - # extractors. Used by parsers (e.g. ``llama3_json``) whose marker is part - # of the JSON they need to parse — ``{"name"`` cannot be consumed without - # destroying the object's structure. Defaults to True to preserve the - # Hermes/Mistral semantics where the marker is an envelope tag stripped - # before the body. - consume_start_marker: bool = True - # Set True when the parser's marker(s) are registered as *special tokens* - # in the tokenizers of the model families this parser targets — the way - # ``[TOOL_CALLS]`` is in Mistral v3+ tokenizers - # (``added_tokens_decoder`` with ``special=True``). Loaders that decode - # with ``skip_special_tokens=True`` by default would otherwise strip the - # marker before this parser ever sees it, leaving the parser permanently - # idle. Loaders that detokenize raw output read this flag at startup, - # flip ``skip_special_tokens`` to ``False`` when set, and noise-strip - # every OTHER registered special from chunks themselves. Hermes and - # llama3_json keep the default — their markers are regular text. - markers_are_specials: bool = False - # Delimiter token wrapping string values in custom-syntax (non-JSON) call - # bodies — ```` for FunctionGemma, ``<|"|>`` for Gemma 4. ``None`` - # for JSON-family parsers (Hermes/Mistral) whose bodies need no such token. - # The GBNF tool-grammar emitter reads it to wrap string-valued arguments. - string_delim: str | None = None - - @abstractmethod - def extract_partial_name(self, partial_payload: str) -> str | None: - """Return the function name if a complete quoted name is visible yet, else ``None``. - - Called every delta until it yields a non-``None`` result; once a name - has been emitted to the client, the streamer stops asking and starts - forwarding arguments bytes for that tool. - """ - - @abstractmethod - def extract_partial_args(self, partial_payload: str, is_complete: bool = False) -> str | None: - """Return the arguments substring as the client should see it so far. - - The streamer takes a length-diff of successive returns and forwards - only the new bytes. Implementations must withhold any trailing bytes - that could plausibly be the envelope closer landing ahead of the - family's end marker — otherwise those bytes leak into the args - stream and the client receives malformed JSON. - - ``is_complete`` is True when the streamer has confirmed the tool call - is fully terminated (or at the end of generation). This allows parsers - that construct JSON dynamically (e.g., from custom syntax) to flush - withheld structural bytes. - """ - - def split_payload(self, payload: str, is_complete: bool) -> list[tuple[str, bool]]: - """Split a region payload into ``(sub_payload, is_complete)`` per-call entries. - - Default: one call per region (Hermes-style). Families like Mistral - whose envelope wraps a JSON array of calls override this to yield - one entry per array element. Each entry is then fed independently to - :meth:`extract_partial_name` / :meth:`extract_partial_args`, and each - becomes its own OpenAI ``tool_calls[i]`` slot. - """ - return [(payload, is_complete)] - - def parse(self, text: str) -> ParsedChatOutput: - # Local import: ``output`` imports ``ToolCallParser`` for typing, - # so importing it at module top would create a cycle. - from modelship.openai.parsers.output import ChatOutputStreamer - - streamer = ChatOutputStreamer(self) - streamer.extract_streaming(text) - streamer.finalize() - return streamer.result diff --git a/modelship/openai/parsers/tool_calling/parsers/gemma.py b/modelship/openai/parsers/tool_calling/parsers/gemma.py deleted file mode 100644 index e109991..0000000 --- a/modelship/openai/parsers/tool_calling/parsers/gemma.py +++ /dev/null @@ -1,363 +0,0 @@ -"""Tool call parsers for Google Gemma models. - -Gemma 4 uses a custom serialization format (not JSON) for tool calls: -`<|tool_call>call:func_name{key:<|"|>value<|"|>,num:42}` - -FunctionGemma (Gemma 2/3 fine-tune) uses: -`call:func_name{key:value}` - -The two formats share the same envelope shape and only differ in the string -delimiter, so both parsers reuse the same custom-syntax parser parameterized -on ``string_delim``. -""" - -from __future__ import annotations - -import json -import re - -from modelship.logging import get_logger -from modelship.openai.parsers.tool_calling.parsers.base import ToolCallParser - -logger = get_logger("openai.parsers.tool_calling.gemma") - -STRING_DELIM = '<|"|>' -ESCAPE_DELIM = "" - -# Trailing characters withheld from streaming JSON output so a closing -# brace / quote / structural byte never lands ahead of more args bytes. -# The streamer takes a length-diff of successive returns, so anything -# emitted here must be a monotonic prefix of the final clean output. -_STREAM_STRIP_TAIL = ("}", "]", '"', ",", " ", ":", "<", "|", "\\", ">") - -# Separator between the literal ``call`` keyword and the function name. -# The spec is ``call:name`` but smaller FunctionGemma checkpoints -# (e.g. 270m-it) sometimes emit ``call name`` — accept whitespace in -# place of (or alongside) the colon so those outputs still parse. -_CALL_SEP = re.compile(r"call[\s:]+") - - -class Gemma4ToolCallParser(ToolCallParser): - name = "gemma4" - start_marker = "<|tool_call>" - end_marker = "" - markers_are_specials = True - # String delimiter used inside the call body. Subclasses override - # this when the family uses a different delimiter token (e.g. - # FunctionGemma's ````). - string_delim: str = STRING_DELIM - - def extract_partial_name(self, partial_payload: str) -> str | None: - """Extract function name from 'call:func_name{...' (or 'call func_name{...').""" - match = _CALL_SEP.match(partial_payload) - if match is None: - return None - if "{" not in partial_payload: - return None - name_part = partial_payload[match.end() : partial_payload.find("{")] - return name_part.strip() or None - - def extract_partial_args(self, partial_payload: str, is_complete: bool = False) -> str | None: - """Parse the custom syntax into a dict and dump it to JSON.""" - if "{" not in partial_payload: - return None - - raw_args = partial_payload[partial_payload.find("{") + 1 :] - # Strip trailing '}' if present (it's the structural end of the call) - if raw_args.endswith("}"): - raw_args = raw_args[:-1] - - try: - # ``partial=not is_complete`` tells the parser to withhold - # ambiguous trailing bytes (unterminated string values, trailing - # bare values) so partial output stays a monotonic prefix of the - # eventual final value. - args_dict = _parse_args(raw_args, partial=not is_complete, string_delim=self.string_delim) - if not args_dict: - return "{}" if is_complete else "" - - full_json = json.dumps(args_dict, ensure_ascii=False) - - if is_complete: - return full_json - - # The streamer takes a suffix-diff. To prevent it from emitting - # the closing braces of the JSON object prematurely (which might - # move if more keys arrive), we withhold trailing JSON structural - # characters during streaming. - safe_json = full_json - while safe_json and safe_json[-1] in _STREAM_STRIP_TAIL: - safe_json = safe_json[:-1] - - return safe_json - except (RecursionError, ValueError, TypeError, IndexError) as e: - logger.debug("Failed to parse %s args %r: %s", self.name, raw_args, e) - return "" - - def split_payload(self, payload: str, is_complete: bool) -> list[tuple[str, bool]]: - """Multiple calls can be concatenated: `call:a{...}call:b{...}`. - - Accepts either ``call:name`` or ``call name`` (whitespace) — same - relaxation as :meth:`extract_partial_name`. - """ - calls: list[tuple[str, bool]] = [] - pos = 0 - n = len(payload) - - while pos < n: - match = _CALL_SEP.search(payload, pos) - if match is None: - break - start = match.start() - - # Find the next call: a real new call is preceded by a closing - # brace from the previous one, so skip over any ``call`` tokens - # nested inside string values or object bodies. - next_match = _CALL_SEP.search(payload, match.end()) - while next_match is not None: - preceding = payload[start : next_match.start()].rstrip() - if preceding.endswith("}"): - break - next_match = _CALL_SEP.search(payload, next_match.end()) - - if next_match is None: - # This is the last call in the payload. - calls.append((payload[start:], is_complete)) - break - calls.append((payload[start : next_match.start()], True)) # Inner calls are complete - pos = next_match.start() - - return calls or [(payload, is_complete)] - - -class FunctionGemmaToolCallParser(Gemma4ToolCallParser): - """Parses FunctionGemma output which uses `` instead of `<|"|>`. - - The envelope and string-delim tokens (````, - ````, ````) are registered as special tokens - on the FunctionGemma tokenizer, so ``markers_are_specials = True`` so - loaders that detokenize raw output keep them visible to the parser. - """ - - name = "function_gemma" - start_marker = "" - end_marker = "" - markers_are_specials = True - string_delim = ESCAPE_DELIM - - -def _parse_value(value_str: str) -> object: - value_str = value_str.strip() - if not value_str: - return value_str - if value_str == "true": - return True - if value_str == "false": - return False - if value_str.lower() in ("null", "none", "nil"): - return None - try: - if "." in value_str: - return float(value_str) - return int(value_str) - except ValueError: - pass - return value_str - - -def _safe_overlap(buf: str, marker: str) -> int: - """Index up to which ``buf`` can be flushed without splitting ``marker``. - - Returns the position past the last byte safe to commit; trailing bytes - that could be a proper prefix of ``marker`` are excluded so a partial - parse of an unterminated string value never includes transient bytes - that will disappear once the delim arrives. This keeps partial output - a monotonic prefix of the eventual final output, which the streamer's - suffix-diff relies on. - """ - if not marker: - return len(buf) - max_overlap = min(len(buf), len(marker) - 1) - for k in range(max_overlap, 0, -1): - if buf.endswith(marker[:k]): - return len(buf) - k - return len(buf) - - -def _parse_args(args_str: str, *, partial: bool = False, string_delim: str = STRING_DELIM) -> dict: - if not args_str or not args_str.strip(): - return {} - - result: dict = {} - i = 0 - n = len(args_str) - delim_len = len(string_delim) - - while i < n: - # Skip whitespace and commas - while i < n and args_str[i] in (" ", ",", "\n", "\t"): - i += 1 - if i >= n: - break - - # Parse key - key_start = i - while i < n and args_str[i] != ":": - i += 1 - if i >= n: - break - key = args_str[key_start:i].strip() - i += 1 # skip ':' - - if i >= n: - if not partial: - result[key] = "" - break - - while i < n and args_str[i] in (" ", "\n", "\t"): - i += 1 - if i >= n: - if not partial: - result[key] = "" - break - - # String value: ... - if args_str[i:].startswith(string_delim): - i += delim_len - val_start = i - end_pos = args_str.find(string_delim, i) - if end_pos == -1: - val = args_str[val_start:] - if partial: - # The closing delim hasn't arrived yet — trim any trailing - # bytes that could be a proper prefix of the delim so the - # partial value is a prefix of the eventual final value. - val = val[: _safe_overlap(val, string_delim)] - result[key] = val - break - result[key] = args_str[val_start:end_pos] - i = end_pos + delim_len - # Nested object - elif args_str[i] == "{": - depth = 1 - obj_start = i + 1 - i += 1 - while i < n and depth > 0: - if args_str[i:].startswith(string_delim): - i += delim_len - nd = args_str.find(string_delim, i) - i = n if nd == -1 else nd + delim_len - continue - if args_str[i] == "{": - depth += 1 - elif args_str[i] == "}": - depth -= 1 - i += 1 - if depth > 0: - result[key] = _parse_args(args_str[obj_start:i], partial=True, string_delim=string_delim) - else: - result[key] = _parse_args(args_str[obj_start : i - 1], string_delim=string_delim) - # Array - elif args_str[i] == "[": - depth = 1 - arr_start = i + 1 - i += 1 - while i < n and depth > 0: - if args_str[i:].startswith(string_delim): - i += delim_len - nd = args_str.find(string_delim, i) - i = n if nd == -1 else nd + delim_len - continue - if args_str[i] == "[": - depth += 1 - elif args_str[i] == "]": - depth -= 1 - i += 1 - if depth > 0: - result[key] = _parse_array(args_str[arr_start:i], partial=True, string_delim=string_delim) - else: - result[key] = _parse_array(args_str[arr_start : i - 1], string_delim=string_delim) - # Bare value - else: - val_start = i - while i < n and args_str[i] not in (",", "}", "]"): - i += 1 - if partial and i >= n: - break - result[key] = _parse_value(args_str[val_start:i]) - - return result - - -def _parse_array(arr_str: str, *, partial: bool = False, string_delim: str = STRING_DELIM) -> list: - items: list = [] - i = 0 - n = len(arr_str) - delim_len = len(string_delim) - while i < n: - while i < n and arr_str[i] in (" ", ",", "\n", "\t"): - i += 1 - if i >= n: - break - # String value - if arr_str[i:].startswith(string_delim): - i += delim_len - val_start = i - end_pos = arr_str.find(string_delim, i) - if end_pos == -1: - val = arr_str[val_start:] - if partial: - val = val[: _safe_overlap(val, string_delim)] - items.append(val) - break - items.append(arr_str[val_start:end_pos]) - i = end_pos + delim_len - # Nested object - elif arr_str[i] == "{": - depth = 1 - obj_start = i + 1 - i += 1 - while i < n and depth > 0: - if arr_str[i:].startswith(string_delim): - i += delim_len - nd = arr_str.find(string_delim, i) - i = n if nd == -1 else nd + delim_len - continue - if arr_str[i] == "{": - depth += 1 - elif arr_str[i] == "}": - depth -= 1 - i += 1 - if depth > 0: - items.append(_parse_args(arr_str[obj_start:i], partial=True, string_delim=string_delim)) - else: - items.append(_parse_args(arr_str[obj_start : i - 1], string_delim=string_delim)) - # Nested array - elif arr_str[i] == "[": - depth = 1 - inner_start = i + 1 - i += 1 - while i < n and depth > 0: - if arr_str[i:].startswith(string_delim): - i += delim_len - nd = arr_str.find(string_delim, i) - i = n if nd == -1 else nd + delim_len - continue - if arr_str[i] == "[": - depth += 1 - elif arr_str[i] == "]": - depth -= 1 - i += 1 - if depth > 0: - items.append(_parse_array(arr_str[inner_start:i], partial=True, string_delim=string_delim)) - else: - items.append(_parse_array(arr_str[inner_start : i - 1], string_delim=string_delim)) - # Bare value - else: - val_start = i - while i < n and arr_str[i] not in (",", "]"): - i += 1 - if partial and i >= n: - break - items.append(_parse_value(arr_str[val_start:i])) - return items diff --git a/modelship/openai/parsers/tool_calling/parsers/hermes.py b/modelship/openai/parsers/tool_calling/parsers/hermes.py deleted file mode 100644 index a8e418b..0000000 --- a/modelship/openai/parsers/tool_calling/parsers/hermes.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Hermes-style ``{json}`` parser. - -Used by Hermes-2-Pro, Qwen2.5-Instruct, and a large family of NousResearch / -community fine-tunes whose chat templates wrap each tool call in the literal -tags ```` / ```` around a JSON object of the shape -``{"name": "...", "arguments": {...}}``. -""" - -from __future__ import annotations - -import re - -from modelship.openai.parsers.tool_calling.parsers.base import ToolCallParser - - -class HermesToolCallParser(ToolCallParser): - name = "hermes" - start_marker = "" - end_marker = "" - - _NAME_RE = re.compile(r'"name"\s*:\s*"([^"]+)"') - _ARGS_RE = re.compile(r'"arguments"\s*:\s*') - - def extract_partial_name(self, partial_payload: str) -> str | None: - m = self._NAME_RE.search(partial_payload) - return m.group(1) if m else None - - def extract_partial_args(self, partial_payload: str, is_complete: bool = False) -> str | None: - m = self._ARGS_RE.search(partial_payload) - if m is None: - return None - args = partial_payload[m.end() :].rstrip() - if args.endswith("}"): - # The block envelope is `{"name":"x","arguments":}`. The - # closing brace of the envelope arrives in the byte stream before - # `` does, so we cannot tell whether any given - # trailing `}` belongs to the args object or to the envelope. - # Withholding one trailing `}` keeps the args stream well-formed: - # if the model goes on to emit more args bytes, the held brace is - # recovered on the next pass; if instead it goes on to emit - # ``, the held brace was the envelope closer and - # discarding it was correct. At ``is_complete=True`` the trailing - # `}` is always the envelope closer (the args object's own closer - # is preceded by it), so the strip still applies. - args = args[:-1].rstrip() - return args or None diff --git a/modelship/openai/parsers/tool_calling/parsers/llama3_json.py b/modelship/openai/parsers/tool_calling/parsers/llama3_json.py deleted file mode 100644 index e1c8565..0000000 --- a/modelship/openai/parsers/tool_calling/parsers/llama3_json.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Llama-3.1/3.2 JSON tool-call parser. - -Used by Llama-3.1 / Llama-3.2 Instruct models whose chat template renders a -custom JSON tool call as a bare object — ``{"name": "fn", "parameters": -{...}}`` followed by ``<|eot_id|>`` (the official Meta / vLLM -``tool_chat_template_llama3.1_json.jinja`` shape). A leading -``<|python_tag|>`` is *not* part of the JSON variant; it is reserved for -built-in code-interpreter calls and would be stripped by HF's default -``skip_special_tokens=True`` anyway. The parser starts at the literal -substring ``{"name"`` so it sees the JSON regardless of whether the tag -prefix was emitted upstream. - -Differences from Hermes / Mistral: - -- The marker ``{"name"`` is *part of* the JSON object, not an envelope - around it, so the parser sets :attr:`consume_start_marker` to False so - the streamer keeps the marker bytes at the head of the payload. -- The arguments field name is ``parameters`` per Meta's spec; some - community fine-tunes emit ``arguments`` instead. The parser accepts - either and the streamer forwards the field bytes verbatim — clients - see the JSON the model produced. -- Multi-call output uses ``; `` (semicolon-space) between top-level - objects, matching vLLM's convention. -""" - -from __future__ import annotations - -import re - -from modelship.openai.parsers.tool_calling.parsers.base import ToolCallParser - - -class Llama3JsonToolCallParser(ToolCallParser): - name = "llama3_json" - start_marker = '{"name"' - end_marker = "" - consume_start_marker = False - - _NAME_RE = re.compile(r'"name"\s*:\s*"([^"]+)"') - _ARGS_RE = re.compile(r'"(?:parameters|arguments)"\s*:\s*') - - def split_payload(self, payload: str, is_complete: bool) -> list[tuple[str, bool]]: - """Split ``{...}; {...}`` into per-call ``({...}, complete)`` entries. - - Walks the payload once tracking string state and brace depth. Each - balanced top-level ``{...}`` becomes one complete sub-block; a - trailing partially-open object becomes one incomplete sub-block so - the streamer can begin emitting its name/args as bytes arrive. The - ``; `` separator between objects is skipped along with any other - whitespace at depth 0. - - ``is_complete`` is the region's completion flag (True at finalize). - It only affects whether a trailing partial object is reported as - complete — closed objects are always complete on their own merits. - """ - results: list[tuple[str, bool]] = [] - depth = 0 - in_str = False - escape = False - start: int | None = None - for i, c in enumerate(payload): - if escape: - escape = False - continue - if in_str: - if c == "\\": - escape = True - elif c == '"': - in_str = False - continue - if c == '"': - in_str = True - elif c == "{": - if depth == 0: - start = i - depth += 1 - elif c == "}": - depth -= 1 - if depth == 0 and start is not None: - results.append((payload[start : i + 1], True)) - start = None - - if start is not None and depth > 0: - # Trailing partial object — only complete if the region is - # complete (finalize) AND we somehow ended balanced, which the - # closed-on-its-own-merits branch above would have caught. - results.append((payload[start:], is_complete and depth == 0)) - return results - - def extract_partial_name(self, partial_payload: str) -> str | None: - m = self._NAME_RE.search(partial_payload) - return m.group(1) if m else None - - def extract_partial_args(self, partial_payload: str, is_complete: bool = False) -> str | None: - m = self._ARGS_RE.search(partial_payload) - if m is None: - return None - args = partial_payload[m.end() :].rstrip() - if args.endswith("}"): - # Mirror Hermes's trailing-`}` strip: the per-call envelope is - # ``{"name": "x", "parameters": }``. The closing brace of - # the envelope arrives in the byte stream alongside (or before) - # the args object's closer, so always strip one trailing `}` — - # at ``is_complete=True`` it is the envelope closer, and - # mid-stream we withhold the ambiguous brace so the streamed - # args view never contains it. If more args bytes follow, the - # held brace is recovered on the next pass. - args = args[:-1].rstrip() - return args or None diff --git a/modelship/openai/parsers/tool_calling/parsers/mistral.py b/modelship/openai/parsers/tool_calling/parsers/mistral.py deleted file mode 100644 index a24473b..0000000 --- a/modelship/openai/parsers/tool_calling/parsers/mistral.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Mistral-style ``[TOOL_CALLS][{...}, {...}]`` parser. - -Used by Mistral 7B Instruct v0.3+, Mistral Small/Large, and downstream -fine-tunes whose chat templates emit a single ``[TOOL_CALLS]`` envelope -followed by a JSON array of one or more ``{"name": "...", "arguments": -{...}}`` objects, with no trailing closing marker — the array runs to -end-of-stream. - -The per-call JSON shape is identical to Hermes; the differences are -that all calls share one envelope (we split the array into per-call -sub-blocks via :meth:`split_payload`) and that the region is -EOS-bounded (the streamer marks it complete at finalize time when -``end_marker`` is empty). -""" - -from __future__ import annotations - -import re - -from modelship.openai.parsers.tool_calling.parsers.base import ToolCallParser - - -class MistralToolCallParser(ToolCallParser): - name = "mistral" - start_marker = "[TOOL_CALLS]" - end_marker = "" - # ``[TOOL_CALLS]`` is registered in Mistral v3+ tokenizers' - # ``added_tokens_decoder`` with ``special=True`` — alongside ``[INST]``, - # ``[/INST]``, ``[AVAILABLE_TOOLS]``, etc. ``skip_special_tokens=True`` - # strips it on the way out of detokenization, so loaders must opt in to - # keep it. See ``tests/test_mistral_specials_smoketest.py`` for the - # round-trip evidence. - markers_are_specials = True - - _NAME_RE = re.compile(r'"name"\s*:\s*"([^"]+)"') - _ARGS_RE = re.compile(r'"arguments"\s*:\s*') - - def split_payload(self, payload: str, is_complete: bool) -> list[tuple[str, bool]]: - """Split ``[{...}, {...}]`` into per-call ``({...}, complete)`` entries. - - Walks the payload once tracking string state and brace depth. Each - balanced top-level ``{...}`` becomes one complete sub-block; a - trailing partially-open object becomes one incomplete sub-block (so - the streamer can start emitting its name/args as bytes arrive). - - ``is_complete`` is the region's completion flag (True at finalize). - It only affects whether a trailing partial object is reported as - complete — closed objects are always complete on their own merits. - """ - s = payload.lstrip() - # Skip the leading `[` of the JSON array if present. Some templates - # render `[TOOL_CALLS] [{...}]` with whitespace; lstrip handled that. - if s.startswith("["): - s = s[1:] - - results: list[tuple[str, bool]] = [] - depth = 0 - in_str = False - escape = False - start: int | None = None - for i, c in enumerate(s): - if escape: - escape = False - continue - if in_str: - if c == "\\": - escape = True - elif c == '"': - in_str = False - continue - if c == '"': - in_str = True - elif c == "{": - if depth == 0: - start = i - depth += 1 - elif c == "}": - depth -= 1 - if depth == 0 and start is not None: - results.append((s[start : i + 1], True)) - start = None - - if start is not None and depth > 0: - # Partial trailing object. Mark it complete only if the whole - # region is complete AND somehow we landed inside an object — in - # practice the model never closes the array without closing the - # object, so a complete region means a clean split above. - results.append((s[start:], is_complete and depth == 0)) - return results - - def extract_partial_name(self, partial_payload: str) -> str | None: - m = self._NAME_RE.search(partial_payload) - return m.group(1) if m else None - - def extract_partial_args(self, partial_payload: str, is_complete: bool = False) -> str | None: - m = self._ARGS_RE.search(partial_payload) - if m is None: - return None - args = partial_payload[m.end() :].rstrip() - if args.endswith("}"): - # Mirror Hermes: the per-call envelope is - # `{"name":"x","arguments":}` and the closing brace of - # the envelope arrives in the byte stream alongside (or before) - # the args object's closer. Always strip one trailing `}` — at - # ``is_complete=True`` it is the envelope closer, and mid-stream - # we withhold the ambiguous brace so the streamed args view - # never contains it. - args = args[:-1].rstrip() - return args or None diff --git a/modelship/openai/parsers/tool_calling/parsers/qwen3_coder.py b/modelship/openai/parsers/tool_calling/parsers/qwen3_coder.py deleted file mode 100644 index 004a4aa..0000000 --- a/modelship/openai/parsers/tool_calling/parsers/qwen3_coder.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Qwen3-Coder XML-style tool call parser. - -Qwen3-Coder reuses the same ```` / ```` envelope as -Hermes but the body is an XML-ish nested structure instead of a JSON -object: - - - - - value_a - - - 42 - - - - -Disambiguation from Hermes happens upstream in -:func:`modelship.openai.parsers.tool_calling.utils.classify_template`: a -chat template that mentions ```` marker falls through -to Hermes. - -Each ``...`` envelope holds exactly one -```` block; multiple tool calls arrive as multiple -back-to-back envelopes, which the streamer already treats as independent -regions, so no :meth:`split_payload` override is needed. - -Parameter values are kept as strings — Qwen3-Coder treats them as -textual blobs without explicit type markers. Callers that need typed -arguments (numbers, booleans, nested JSON) can JSON-decode the value in -their tool handler. Streaming-safe coercion would require schema-aware -disambiguation we don't have at this layer. -""" - -from __future__ import annotations - -import json -import re - -from modelship.openai.parsers.tool_calling.parsers.base import ToolCallParser - -_FUNC_OPEN_RE = re.compile(r"]+)>") -_PARAM_OPEN_RE = re.compile(r"]+)>") -_PARAM_CLOSE = "" -_FUNC_CLOSE = "" - -# Structural suffix that ``json.dumps`` always emits for a non-empty dict -# whose last value is a string: the closing quote of that value plus the -# closing brace of the object. Stripping exactly these two bytes exposes -# the in-flight value as a monotonic prefix without touching content -# inside the value — a character-wise loop would eat trailing ``,``, -# ``}``, or ``"`` that are actually part of the parameter value. -_STREAM_STRIP_TAIL = '"}' - - -class Qwen3CoderToolCallParser(ToolCallParser): - name = "qwen3_coder" - start_marker = "" - end_marker = "" - - def extract_partial_name(self, partial_payload: str) -> str | None: - m = _FUNC_OPEN_RE.search(partial_payload) - return m.group(1).strip() if m else None - - def extract_partial_args(self, partial_payload: str, is_complete: bool = False) -> str | None: - fm = _FUNC_OPEN_RE.search(partial_payload) - if fm is None: - return None - - body = partial_payload[fm.end() :] - end_func = body.find(_FUNC_CLOSE) - if end_func != -1: - body = body[:end_func] - - args: dict[str, str] = {} - pos = 0 - while True: - pm = _PARAM_OPEN_RE.search(body, pos) - if pm is None: - break - key = pm.group(1).strip() - val_start = pm.end() - close = body.find(_PARAM_CLOSE, val_start) - if close == -1: - # In-flight parameter value. Withhold any trailing bytes - # that could be a proper prefix of ```` or of - # the next `` int: - """Index up to which ``buf`` can be flushed without splitting ``marker``. - - Returns the position past the last byte safe to commit; trailing bytes - that could be a proper prefix of ``marker`` are excluded so partial - output stays a monotonic prefix of the eventual final value. - """ - if not marker: - return len(buf) - max_overlap = min(len(buf), len(marker) - 1) - for k in range(max_overlap, 0, -1): - if buf.endswith(marker[:k]): - return len(buf) - k - return len(buf) diff --git a/modelship/openai/parsers/tool_calling/registry.py b/modelship/openai/parsers/tool_calling/registry.py deleted file mode 100644 index a02cfe0..0000000 --- a/modelship/openai/parsers/tool_calling/registry.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Registry of named tool-call parsers, dispatched by configuration string. - -The registry is the single seam between a loader and the per-family parsers. -Loaders that emit raw text (Transformers, plugin-wrapped engines) look up a -parser by the name configured on the deployment and feed it the model's -output. Loaders with native tool-call support (vLLM, llama.cpp via a -function-calling chat handler) bypass the registry entirely. -""" - -from __future__ import annotations - -from modelship.openai.parsers.tool_calling.parsers import ( - FunctionGemmaToolCallParser, - Gemma4ToolCallParser, - HermesToolCallParser, - Llama3JsonToolCallParser, - MistralToolCallParser, - Qwen3CoderToolCallParser, - ToolCallParser, -) - -_PARSERS: dict[str, ToolCallParser] = { - FunctionGemmaToolCallParser.name: FunctionGemmaToolCallParser(), - Gemma4ToolCallParser.name: Gemma4ToolCallParser(), - HermesToolCallParser.name: HermesToolCallParser(), - Llama3JsonToolCallParser.name: Llama3JsonToolCallParser(), - MistralToolCallParser.name: MistralToolCallParser(), - Qwen3CoderToolCallParser.name: Qwen3CoderToolCallParser(), -} - - -def get_parser(name: str) -> ToolCallParser: - """Return the parser registered under ``name`` or raise ``ValueError``.""" - try: - return _PARSERS[name] - except KeyError: - available = ", ".join(sorted(_PARSERS)) or "(none)" - raise ValueError(f"unknown tool_call_parser {name!r}; available: {available}") from None - - -def available_parsers() -> list[str]: - return sorted(_PARSERS) - - -def register_parser(parser: ToolCallParser) -> None: - """Register an additional parser. Intended for tests and plugin code.""" - _PARSERS[parser.name] = parser diff --git a/modelship/openai/protocol/responses/__init__.py b/modelship/openai/protocol/responses/__init__.py index 74fdb3a..a74f337 100644 --- a/modelship/openai/protocol/responses/__init__.py +++ b/modelship/openai/protocol/responses/__init__.py @@ -1,23 +1,23 @@ -"""Responses API (``/v1/responses``) schemas and the stateless chat adapter. +"""Responses API (``/v1/responses``) schemas and the request-side chat adapter. -Phase A implements ``/v1/responses`` as a stateless adapter over the existing -chat-completion pipeline rather than a new inference path: the request is -translated to a :class:`ChatCompletionRequest`, run through the unchanged -``handle.generate`` deployment method, and the chat result is translated back -into a Responses :class:`ResponseObject`. +Each loader implements its own ``create_response`` natively (shaping straight +from its parsed chat output — see ``chat_utils.build_responses_items_from_parsed``), +non-streaming and streaming alike; there is no generic response-side fallback. +The one thing every loader shares is the request-side translation: an incoming +``ResponsesRequest`` is turned into a ``ChatCompletionRequest`` via +``responses_request_to_chat`` before the loader re-derives its own chat request +internally. Thin re-exporter over the leaf submodules (mirrors the parent ``protocol`` package): :mod:`.schemas` holds the pydantic models, :mod:`.adapter` the -non-streaming translation, :mod:`.streaming` the streaming translator. The -submodules import cleanly — ``adapter`` pulls in ``schemas``, ``streaming`` -pulls in ``adapter`` + ``schemas``, and none reach back into the top-level -``protocol`` package — so no import cycle exists here. +request-side translation plus the shared response-envelope helpers, :mod:`.streaming` +the streaming translator. The submodules import cleanly — ``adapter`` pulls in +``schemas``, ``streaming`` pulls in ``adapter`` + ``schemas``, and none reach back +into the top-level ``protocol`` package — so no import cycle exists here. """ from modelship.openai.protocol.responses.adapter import ( UnsupportedResponsesFeatureError, - chat_response_to_responses, - responses_from_chat, responses_request_to_chat, ) from modelship.openai.protocol.responses.schemas import ( @@ -35,10 +35,7 @@ ResponsesRequest, ResponseUsage, ) -from modelship.openai.protocol.responses.streaming import ( - ResponsesStreamTranslator, - responses_stream_from_chat, -) +from modelship.openai.protocol.responses.streaming import ResponsesStreamTranslator __all__ = [ "ResponseFunctionToolCall", @@ -56,8 +53,5 @@ "ResponsesRequest", "ResponsesStreamTranslator", "UnsupportedResponsesFeatureError", - "chat_response_to_responses", - "responses_from_chat", "responses_request_to_chat", - "responses_stream_from_chat", ] diff --git a/modelship/openai/protocol/responses/adapter.py b/modelship/openai/protocol/responses/adapter.py index 98e4207..345f350 100644 --- a/modelship/openai/protocol/responses/adapter.py +++ b/modelship/openai/protocol/responses/adapter.py @@ -1,21 +1,19 @@ -"""Stateless translation between the Responses API and chat completions. +"""Translation between the Responses API and chat completions. + +:func:`responses_request_to_chat` — ``ResponsesRequest`` → ``ChatCompletionRequest``. +Translates the structurally different bits (``input``/``instructions`` → +``messages``, flattened tools → nested, ``text.format`` → ``response_format``, +``max_output_tokens`` → ``max_completion_tokens``) and rejects features not yet +supported (``previous_response_id``, ``background``, hosted built-in tools) with +an explicit error rather than dropping them silently. Every loader's +``create_response`` calls into this for the request side; ``vllm``/``llama_server`` +then shape the response natively from their own parsed output rather than going +back through a chat-completion object (see ``chat_utils.build_responses_items_from_parsed``) +— ``build_response_object``/``_usage_from_chat``/``_status_for`` below are the shared +envelope helpers both that native path and the streaming translator build on. -Two directions: - -- :func:`responses_request_to_chat` — ``ResponsesRequest`` → ``ChatCompletionRequest``. - Translates the structurally different bits (``input``/``instructions`` → - ``messages``, flattened tools → nested, ``text.format`` → ``response_format``, - ``max_output_tokens`` → ``max_completion_tokens``) and rejects features Phase A - cannot honor (``previous_response_id``, ``background``, hosted built-in tools) - with an explicit error rather than dropping them silently. -- :func:`chat_response_to_responses` — ``ChatCompletionResponse`` → ``ResponseObject``. - Maps the parsed message into Responses ``output[]`` items (reasoning / message / - function_call) and remaps token usage. - -Phase A is text + reasoning + (client-driven) tool calling, non-streaming. ``store`` is accepted but never persisted — the response echoes ``store=False``. -Image/audio input parts are reduced to their text for now (vision over -``/v1/responses`` is out of Phase A scope). +Image/audio input parts are reduced to their text for now. Imports come from the sibling ``schemas`` submodule and the ``chat`` submodule, never from the top-level ``modelship.openai.protocol`` package — that package @@ -24,19 +22,13 @@ from __future__ import annotations -from collections.abc import AsyncIterator from typing import Any, Literal -from modelship.openai.protocol.chat import ChatCompletionRequest, ChatCompletionResponse +from modelship.openai.protocol.chat import ChatCompletionRequest from modelship.openai.protocol.responses.schemas import ( - ResponseFunctionToolCall, ResponseInputTokensDetails, ResponseObject, - ResponseOutputMessage, - ResponseOutputText, ResponseOutputTokensDetails, - ResponseReasoningItem, - ResponseReasoningSummary, ResponsesRequest, ResponseUsage, ) @@ -44,7 +36,7 @@ class UnsupportedResponsesFeatureError(ValueError): - """A Responses request used a feature Phase A does not implement. + """A Responses request used a feature that is not supported. Subclasses ``ValueError`` so ``create_error_response`` maps it to an OpenAI-style 400 ``invalid_request_error``. @@ -228,57 +220,11 @@ def _response_format_from_text(text: dict[str, Any] | None) -> dict[str, Any] | # --------------------------------------------------------------------------- -# Response: chat → Responses +# Response envelope: shared by every loader's native create_response and by +# the streaming translator. # --------------------------------------------------------------------------- -def chat_response_to_responses(chat: ChatCompletionResponse, request: ResponsesRequest) -> ResponseObject: - """Translate a non-streaming ``ChatCompletionResponse`` into a ``ResponseObject``.""" - choice = chat.choices[0] - message = choice.message - - output: list[Any] = [] - # OpenAI emits reasoning first, then the assistant message / tool calls. - if message.reasoning: - output.append(ResponseReasoningItem(summary=[ResponseReasoningSummary(text=message.reasoning)])) - if message.content: - output.append(ResponseOutputMessage(content=[ResponseOutputText(text=message.content)])) - for call in message.tool_calls: - output.append( - ResponseFunctionToolCall( - call_id=call.id, - name=call.function.name, - arguments=call.function.arguments, - ) - ) - - status, incomplete = _status_for(choice.finish_reason) - - return build_response_object( - request, - status=status, - output=output, - usage=_usage_from_chat(chat.usage), - incomplete=incomplete, - model=chat.model, - ) - - -async def responses_from_chat(response_gen: AsyncIterator[Any], request: ResponsesRequest) -> AsyncIterator[Any]: - """Adapt a non-streaming chat response generator into Responses output. - - Lets ``/v1/responses`` reuse ``_handle_response`` unchanged: chat response - objects are translated to ``ResponseObject`` while errors and Ray exceptions - pass through to the same handling as every other endpoint. The streaming - counterpart is ``responses_stream_from_chat`` in :mod:`.streaming`. - """ - async for item in response_gen: - if isinstance(item, ChatCompletionResponse): - yield chat_response_to_responses(item, request) - else: - yield item - - def build_response_object( request: ResponsesRequest, *, diff --git a/modelship/openai/protocol/responses/streaming.py b/modelship/openai/protocol/responses/streaming.py index 1cb6a69..efa3d35 100644 --- a/modelship/openai/protocol/responses/streaming.py +++ b/modelship/openai/protocol/responses/streaming.py @@ -1,12 +1,12 @@ -"""Streaming translation: chat-completion SSE → Responses event stream. +"""Streaming translation: typed chat chunks → Responses event stream. -The chat loaders all emit Server-Sent Events as ``data: {chunk}\\n\\n`` strings -(``ChatCompletionStreamResponse`` payloads) terminated by ``data: [DONE]`` — this -is the one wire shape vLLM and the ``ChatOutputStreamer`` loaders (transformers, -plugins) share. The Responses API instead wants a *semantic* event -protocol: named events (``response.created``, ``response.output_text.delta``, …) -each carrying a monotonically increasing ``sequence_number`` plus the relevant -output index, wrapping each output item in explicit added/done brackets. +``vllm``/``llama_server`` feed :class:`ResponsesStreamTranslator` directly with +typed ``ChatCompletionStreamResponse`` chunks (the same DTO their chat streaming +path produces) — no SSE text round-trip. The Responses API wants a *semantic* +event protocol instead: named events (``response.created``, +``response.output_text.delta``, …) each carrying a monotonically increasing +``sequence_number`` plus the relevant output index, wrapping each output item +in explicit added/done brackets. :class:`ResponsesStreamTranslator` consumes the parsed chat chunks and brackets them. Design choices that matter: @@ -32,7 +32,7 @@ from __future__ import annotations import json -from collections.abc import AsyncIterator, Iterator +from collections.abc import Iterator from typing import Any from modelship.openai.protocol.base import random_uuid @@ -57,23 +57,6 @@ def _sse(event_type: str, payload: dict[str, Any]) -> str: return f"event: {event_type}\ndata: {json.dumps({'type': event_type, **payload})}\n\n" -def _parse_chat_sse(raw: str) -> Iterator[ChatCompletionStreamResponse]: - """Parse chat ``data: {...}`` SSE messages into stream chunks. - - A single ``raw`` string may bundle several SSE messages (e.g. from - buffering or batching). Yields a chunk for every ``data:`` line, skipping - the ``[DONE]`` sentinel and any non-data lines. - """ - for line in raw.splitlines(): - line = line.strip() - if not line.startswith("data:"): - continue - payload = line[len("data:") :].strip() - if not payload or payload == "[DONE]": - continue - yield ChatCompletionStreamResponse.model_validate_json(payload) - - class ResponsesStreamTranslator: """Stateful translator from chat stream chunks to Responses SSE events. @@ -342,33 +325,3 @@ def _close_tools(self) -> Iterator[str]: "response.output_item.done", {"output_index": oi, "item": item.model_dump(mode="json")}, ) - - -async def responses_stream_from_chat(response_gen: AsyncIterator[Any], request: ResponsesRequest) -> AsyncIterator[Any]: - """Adapt a streaming chat response generator into Responses SSE events. - - Lets ``/v1/responses`` reuse ``_handle_response`` unchanged. The first item - decides the path: an early ``ErrorResponse`` (loader rejected before - streaming) is passed through so the gateway renders the proper HTTP error - instead of a ``200`` event stream; chat SSE strings start the event stream. - """ - translator = ResponsesStreamTranslator(request) - started = False - async for item in response_gen: - if not isinstance(item, str): - if not started: - # Pre-stream error/object — let _handle_response handle it. - yield item - return - # A non-string mid-stream is unexpected; skip rather than corrupt the stream. - continue - if not started: - started = True - for event in translator.start(): - yield event - for chunk in _parse_chat_sse(item): - for event in translator.process(chunk): - yield event - if started: - for event in translator.finish(): - yield event diff --git a/tests/test_gemma4_parsers.py b/tests/test_gemma4_parsers.py deleted file mode 100644 index 10d2a5b..0000000 --- a/tests/test_gemma4_parsers.py +++ /dev/null @@ -1,197 +0,0 @@ -import json - -from modelship.openai.parsers.output import ChatOutputStreamer -from modelship.openai.parsers.reasoning.parsers.gemma import Gemma4ReasoningParser -from modelship.openai.parsers.tool_calling.parsers.gemma import ( - FunctionGemmaToolCallParser, - Gemma4ToolCallParser, -) - - -class TestGemma4ReasoningParser: - def test_extracts_reasoning_and_strips_label(self): - parser = Gemma4ReasoningParser() - streamer = ChatOutputStreamer(None, parser) - - # Simulate model output: <|channel>thought\nI am reasoningFinal answer - text = "<|channel>thought\nI am reasoningFinal answer" - streamer.extract_streaming(text) - streamer.finalize() - - result = streamer.result - assert result.reasoning == "I am reasoning" - assert result.content == "Final answer" - - -class TestGemma4ToolCallParser: - def test_single_tool_call(self): - parser = Gemma4ToolCallParser() - # Simulate: <|tool_call>call:get_weather{location:<|"|>Tokyo<|"|>} - text = '<|tool_call>call:get_weather{location:<|"|>Tokyo<|"|>}' - result = parser.parse(text) - - assert len(result.tool_calls) == 1 - assert result.tool_calls[0].function.name == "get_weather" - assert json.loads(result.tool_calls[0].function.arguments) == {"location": "Tokyo"} - - def test_multiple_concatenated_tool_calls(self): - parser = Gemma4ToolCallParser() - # Simulate: <|tool_call>call:a{x:1}call:b{y:2} - text = '<|tool_call>call:get_weather{location:<|"|>Tokyo<|"|>}call:get_time{timezone:<|"|>JST<|"|>}' - result = parser.parse(text) - - assert len(result.tool_calls) == 2 - assert result.tool_calls[0].function.name == "get_weather" - assert result.tool_calls[1].function.name == "get_time" - assert json.loads(result.tool_calls[0].function.arguments) == {"location": "Tokyo"} - assert json.loads(result.tool_calls[1].function.arguments) == {"timezone": "JST"} - - def test_streaming_tool_call(self): - parser = Gemma4ToolCallParser() - streamer = ChatOutputStreamer(parser) - - # Chunk 1: Start and name - d1 = streamer.extract_streaming("<|tool_call>call:get_weather{loc") - assert d1.tool_calls[0].function.name == "get_weather" - - # Chunk 2: Partial arguments - d2 = streamer.extract_streaming('<|tool_call>call:get_weather{location:<|"|>Tok') - # "location": "Tok" -> length diff - assert "location" in d2.tool_calls[0].function.arguments - - # Chunk 3: End of call - d3 = streamer.extract_streaming('<|tool_call>call:get_weather{location:<|"|>Tokyo<|"|>}') - assert "yo" in d3.tool_calls[0].function.arguments - - streamer.finalize() - result = streamer.result - assert len(result.tool_calls) == 1 - assert json.loads(result.tool_calls[0].function.arguments) == {"location": "Tokyo"} - - -class TestFunctionGemmaToolCallParser: - def test_single_tool_call(self): - parser = FunctionGemmaToolCallParser() - # Simulate: call:get_weather{location:Tokyo} - text = "call:get_weather{location:Tokyo}" - result = parser.parse(text) - - assert len(result.tool_calls) == 1 - assert result.tool_calls[0].function.name == "get_weather" - assert json.loads(result.tool_calls[0].function.arguments) == {"location": "Tokyo"} - - def test_streaming_tool_call(self): - parser = FunctionGemmaToolCallParser() - streamer = ChatOutputStreamer(parser) - - # Chunk 1: Start and name - d1 = streamer.extract_streaming("call:get_weather{loc") - assert d1.tool_calls[0].function.name == "get_weather" - - # Chunk 2: Partial arguments - d2 = streamer.extract_streaming("call:get_weather{location:Tok") - # "location": "Tok" -> length diff - assert "location" in d2.tool_calls[0].function.arguments - - # Chunk 3: End of call - d3 = streamer.extract_streaming( - "call:get_weather{location:Tokyo}" - ) - assert "yo" in d3.tool_calls[0].function.arguments - - streamer.finalize() - result = streamer.result - assert len(result.tool_calls) == 1 - assert json.loads(result.tool_calls[0].function.arguments) == {"location": "Tokyo"} - - def test_markers_are_specials_is_true(self): - # FunctionGemma's envelope and string-delim tokens are registered - # specials on the tokenizer; loaders that detokenize raw output - # key off this flag to switch ``skip_special_tokens=False`` and - # keep the markers visible to the parser. - assert FunctionGemmaToolCallParser.markers_are_specials is True - - def test_accepts_space_separator_instead_of_colon(self): - # Regression: the 270m-it checkpoint sometimes emits ``call `` - # (whitespace) instead of ``call:``. Both shapes must parse. - parser = FunctionGemmaToolCallParser() - text = "call get_weather{city:Paris}" - result = parser.parse(text) - - assert len(result.tool_calls) == 1 - assert result.tool_calls[0].function.name == "get_weather" - assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Paris"} - - -class TestGemmaNestedStructures: - """Custom-syntax parser handles nested objects and arrays correctly.""" - - def test_array_of_objects(self): - parser = Gemma4ToolCallParser() - text = "<|tool_call>call:f{items:[{a:1},{a:2}]}" - result = parser.parse(text) - assert json.loads(result.tool_calls[0].function.arguments) == {"items": [{"a": 1}, {"a": 2}]} - - def test_array_of_mixed_types(self): - parser = Gemma4ToolCallParser() - text = '<|tool_call>call:f{items:[<|"|>x<|"|>,{n:1},<|"|>y<|"|>]}' - result = parser.parse(text) - assert json.loads(result.tool_calls[0].function.arguments) == {"items": ["x", {"n": 1}, "y"]} - - def test_nested_arrays(self): - parser = Gemma4ToolCallParser() - text = "<|tool_call>call:f{grid:[[1,2],[3,4]]}" - result = parser.parse(text) - assert json.loads(result.tool_calls[0].function.arguments) == {"grid": [[1, 2], [3, 4]]} - - def test_function_gemma_array_of_objects(self): - parser = FunctionGemmaToolCallParser() - text = "call:f{items:[{a:1},{a:2}]}" - result = parser.parse(text) - assert json.loads(result.tool_calls[0].function.arguments) == {"items": [{"a": 1}, {"a": 2}]} - - -class TestGemmaStreamingMonotonicity: - """Partial string parsing must yield monotonic prefixes of the final value. - - Regression: when a chunk ends mid-way through the closing string delim - (e.g. ````), the partial parse used - to include those bytes in the string value, producing a stripped JSON - longer than the eventual clean output — and the streamer's - length-greater diff machinery could not retract the transient bytes, - leaving the client stuck with a malformed value. - """ - - def test_function_gemma_chunk_mid_closing_delim(self): - parser = FunctionGemmaToolCallParser() - streamer = ChatOutputStreamer(parser) - chunks = [ - "call:f{location:Tokyocall:f{location:Tokyo}", - ] - seen = "" - for c in chunks: - d = streamer.extract_streaming(c) - if d and d.tool_calls: - for t in d.tool_calls: - if t.function and t.function.arguments: - seen += t.function.arguments - streamer.finalize() - assert json.loads(seen) == {"location": "Tokyo"} - - def test_gemma4_chunk_mid_closing_delim(self): - parser = Gemma4ToolCallParser() - streamer = ChatOutputStreamer(parser) - chunks = [ - '<|tool_call>call:f{x:<|"|>val<', - '<|tool_call>call:f{x:<|"|>val<|"|>}', - ] - seen = "" - for c in chunks: - d = streamer.extract_streaming(c) - if d and d.tool_calls: - for t in d.tool_calls: - if t.function and t.function.arguments: - seen += t.function.arguments - streamer.finalize() - assert json.loads(seen) == {"x": "val"} diff --git a/tests/test_integration.py b/tests/test_integration.py index 1a2b67d..694f546 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -91,10 +91,9 @@ "name": "chat-llama-server", # Qwen3-0.6B GGUF through the llama_server loader: a llama-server # subprocess doing its own chat templating, tool-call, and reasoning - # parsing (`--jinja --reasoning-format auto`) instead of modelship's - # ChatOutputStreamer. `parallel: 4` exercises the loader's headline - # capability: true multi-slot concurrency instead of a single - # asyncio.Lock serializing every request. n_ctx is per-slot (the + # parsing (`--jinja --reasoning-format auto`). `parallel: 4` exercises + # the loader's headline capability: true multi-slot concurrency + # instead of a single asyncio.Lock serializing every request. n_ctx is per-slot (the # loader launches with `-c n_ctx*parallel`), bumped for reasoning # headroom. "model": "lmstudio-community/Qwen3-0.6B-GGUF:*Q4_K_M.gguf", @@ -785,8 +784,7 @@ class TestChatLlamaServer: """End-to-end chat, tool calling, reasoning, and concurrency through the `llama_server` loader (a `llama-server` subprocess proxied over its native OpenAI-compatible HTTP API). Reasoning and tool-call parsing is - llama-server's own (`--jinja --reasoning-format auto`), not modelship's - `ChatOutputStreamer`. + llama-server's own (`--jinja --reasoning-format auto`). """ @pytest.fixture(autouse=True, scope="class") @@ -935,7 +933,7 @@ def test_tool_markers_inside_reasoning_not_double_counted_llama_server(self, cli """Verifies llama-server's own parser doesn't double-count a `...` illustration quoted inside `` reasoning as a second, real call — a bug pattern plausible for any - single-pass parser, not just modelship's `ChatOutputStreamer`. + single-pass parser. Coaxes the model into illustrating tool-call syntax inside its reasoning before making one actual call, and asserts exactly one real diff --git a/tests/test_mistral_specials_smoketest.py b/tests/test_mistral_specials_smoketest.py deleted file mode 100644 index bd5c8ae..0000000 --- a/tests/test_mistral_specials_smoketest.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Smoke test for the hypothesis that ``[TOOL_CALLS]`` is stripped before our parser sees it. - -Background ----------- -The ``MistralToolCallParser`` declares ``start_marker = "[TOOL_CALLS]"``. -A loader that detokenizes raw model output with ``skip_special_tokens=True`` -would strip that marker before it reaches our parser if the tokenizer -registers it as a special token — this is exactly why -``_resolved_skip_special_tokens`` gets pinned to ``False`` for the mistral -parser (see ``resolve_all_tool_parsers``). - -If a Mistral tokenizer registers ``[TOOL_CALLS]`` as an *additional special -token*, then on a real Mistral model run, the marker would be stripped from -the streamed text before reaching our parser — and our parser would never -enter tool-call mode. Streamer-level unit tests don't catch this because -they feed strings directly, bypassing the tokenizer round-trip. - -This file confirms or rejects the hypothesis in two parts: - -1. **Synthetic** (always runs): build a tokenizer with ``[TOOL_CALLS]`` - registered as an additional special token, encode a sample tool-call - string, decode with each ``skip_special_tokens`` setting, and verify - that the stripping behavior is what we feared. This tests the HF - contract, not Mistral specifically — it tells us whether *any* tokenizer - that registers the marker as special would lose it when a loader - detokenizes with specials skipped. - -2. **Real Mistral** (skipped if the tokenizer can't be loaded): load a - real Mistral tokenizer and check whether ``[TOOL_CALLS]`` actually - appears in its specials list. Skipped on missing HF auth or no - network, so this part is opt-in. -""" - -from __future__ import annotations - -import os - -import pytest - - -def _sample_tool_call_text() -> str: - return '[TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}}]' - - -class TestSyntheticAdditionalSpecialTokenStripping: - """Pure HF behavior check — no network, no auth. - - Take any small public tokenizer, add ``[TOOL_CALLS]`` as an additional - special token, and confirm that ``skip_special_tokens=True`` strips it - out of the decoded text. If this passes, the same will happen on any - real Mistral tokenizer that registers ``[TOOL_CALLS]`` as special — - which is the second test in this file. - """ - - @pytest.fixture(scope="class") - def tokenizer_with_tool_calls_special(self): - from transformers import AutoTokenizer - - # Qwen2.5-0.5B's tokenizer is already used by the project's - # integration suite (no auth, small download). We only need its - # vocabulary; we then register `[TOOL_CALLS]` as a special token - # on top of it to mirror the Mistral configuration. - try: - tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B-Instruct") - except Exception as e: - pytest.skip(f"could not load the host tokenizer for the synthetic test: {e}") - tokenizer.add_special_tokens({"additional_special_tokens": ["[TOOL_CALLS]"]}) - return tokenizer - - def test_roundtrip_keeps_marker_when_skip_special_tokens_false(self, tokenizer_with_tool_calls_special): - text = _sample_tool_call_text() - ids = tokenizer_with_tool_calls_special.encode(text, add_special_tokens=False) - decoded = tokenizer_with_tool_calls_special.decode(ids, skip_special_tokens=False) - assert "[TOOL_CALLS]" in decoded, ( - "with skip_special_tokens=False the marker MUST round-trip — if this fails, the test setup is wrong" - ) - - def test_roundtrip_strips_marker_when_skip_special_tokens_true(self, tokenizer_with_tool_calls_special): - """The hypothesis: when ``[TOOL_CALLS]`` is an additional special - token, ``skip_special_tokens=True`` removes it from the decoded - text. - - If this assertion holds, the Mistral parser cannot activate on a - loader that detokenizes with specials skipped, for any tokenizer - that registers the marker as special. The fix is loader-side - (per-parser flag flip + noise stripper). - """ - text = _sample_tool_call_text() - ids = tokenizer_with_tool_calls_special.encode(text, add_special_tokens=False) - decoded = tokenizer_with_tool_calls_special.decode(ids, skip_special_tokens=True) - assert "[TOOL_CALLS]" not in decoded, ( - "expected `[TOOL_CALLS]` to be stripped by skip_special_tokens=True; " - f"got decoded={decoded!r}. If this assertion fails, the hypothesis is wrong " - "and Mistral is fine as-is without the skip-specials pin." - ) - - -class TestRealMistralTokenizer: - """Verify the second half of the hypothesis on a real Mistral tokenizer. - - Skipped unless a Mistral repo is reachable (typically requires - ``HF_TOKEN`` since ``mistralai/*`` repos are gated). - """ - - @pytest.fixture(scope="class") - def mistral_tokenizer(self): - from transformers import AutoTokenizer - - # Try the canonical v0.3 first; fall back to other v3+ Mistral - # repos that ship the same `[TOOL_CALLS]` marker if v0.3 is - # unreachable. - candidates = [ - "mistralai/Mistral-7B-Instruct-v0.3", - "mistralai/Mistral-Small-Instruct-2409", - "mistralai/Mistral-Nemo-Instruct-2407", - ] - last_err: Exception | None = None - for repo in candidates: - try: - return AutoTokenizer.from_pretrained(repo) - except Exception as e: - last_err = e - token_present = "set" if os.environ.get("HF_TOKEN") else "unset" - pytest.skip( - f"no Mistral tokenizer reachable from {candidates!r} (HF_TOKEN {token_present}); last error: {last_err!r}" - ) - - def test_tool_calls_is_a_special_added_token(self, mistral_tokenizer): - """Confirm ``[TOOL_CALLS]`` is registered as a special added token. - - Mistral v3+ tokenizers register tool-protocol markers in - ``added_tokens_decoder`` with ``special=True``, NOT in - ``all_special_tokens`` (which only carries the bos/eos/unk core). - ``skip_special_tokens=True`` strips both groups. Checking - ``all_special_tokens`` alone misses this — the right attribute is - ``added_tokens_decoder``. - """ - special_added: dict[int, str] = { - tid: tok.content for tid, tok in mistral_tokenizer.added_tokens_decoder.items() if tok.special - } - assert "[TOOL_CALLS]" in special_added.values(), ( - "hypothesis disproved: `[TOOL_CALLS]` is NOT a special added token on this Mistral tokenizer " - f"(special_added={special_added!r}); the marker would survive skip_special_tokens=True and " - "the parser is fine." - ) - - def test_real_tokenizer_strips_marker_with_skip_special_tokens_true(self, mistral_tokenizer): - """End-to-end confirmation on a real Mistral tokenizer.""" - text = _sample_tool_call_text() - ids = mistral_tokenizer.encode(text, add_special_tokens=False) - decoded = mistral_tokenizer.decode(ids, skip_special_tokens=True) - assert "[TOOL_CALLS]" not in decoded, ( - f"expected `[TOOL_CALLS]` stripped on the real Mistral tokenizer; got decoded={decoded!r}." - ) diff --git a/tests/test_parser_resolution.py b/tests/test_parser_resolution.py index 225af2e..2f4e242 100644 --- a/tests/test_parser_resolution.py +++ b/tests/test_parser_resolution.py @@ -17,7 +17,8 @@ ModelUsecase, VllmEngineConfig, ) -from modelship.openai.parsers.reasoning.utils import classify_template as classify_reasoning +from modelship.openai.parsers.reasoning import classify_template as classify_reasoning +from modelship.openai.parsers.tool_calling import classify_template as classify_tool_calling def _make_cfg(**overrides) -> ModelshipModelConfig: @@ -31,6 +32,49 @@ def _make_cfg(**overrides) -> ModelshipModelConfig: return ModelshipModelConfig(**base) +class TestClassifyToolTemplate: + """``classify_template`` must return names that match vLLM's own + ``ToolParserManager`` registry exactly — ``resolve_all_tool_parsers`` + validates auto-detected names against it directly.""" + + def test_no_tool_markers_returns_none(self): + assert classify_tool_calling("plain template with no markers") is None + + def test_gemma4_marker(self): + assert classify_tool_calling("{% if tools %}<|tool_call>{% endif %}") == "gemma4" + + def test_function_gemma_marker_matches_vllm_name(self): + # Regression: vLLM registers this parser as "functiongemma" (no + # underscore); modelship's own class used to be named "function_gemma" + # and this detector returned that mismatched name, which would fail + # validation against vLLM's real registry (or, worse, be handed + # straight to vLLM's OpenAIServingRender and fail there instead). + assert classify_tool_calling("{% if tools %}{% endif %}") == "functiongemma" + + def test_qwen3_coder_function_marker_routes_ahead_of_hermes(self): + # The chat template mentions tools (gating clause) and contains + # ``{% endif %}") == "llama3_json" + + def test_unrecognized_markers_returns_unknown(self): + assert classify_tool_calling("{% if tools %}some tool syntax{% endif %}") == "unknown" + + class TestClassifyReasoningTemplate: def test_open_think_tag(self): assert classify_reasoning("......") == "deepseek_r1" @@ -124,31 +168,3 @@ def test_vllm_opt_out_leaves_none(self): cfg = _make_cfg(vllm_engine_kwargs=VllmEngineConfig(enable_auto_tool_choice=False, tool_call_parser="hermes")) resolve_all_tool_parsers(ModelshipConfig(models=[cfg])) assert cfg._resolved_tool_call_parser is None - - -class TestResolveSkipSpecialTokens: - """``_resolved_skip_special_tokens`` is pinned by the parser's flag. - - Loaders that detokenize raw model output read this at startup to - decide whether to flip ``skip_special_tokens=False``. ``None`` means - "loader keeps its own default (True)"; ``False`` means "the parser's - marker is registered as a special token and would be stripped — keep - specials in the stream and noise-strip the rest." - """ - - def test_hermes_leaves_skip_specials_default(self): - cfg = _make_cfg(vllm_engine_kwargs=VllmEngineConfig(tool_call_parser="hermes")) - resolve_all_tool_parsers(ModelshipConfig(models=[cfg])) - assert cfg._resolved_skip_special_tokens is None - - def test_llama3_json_leaves_skip_specials_default(self): - cfg = _make_cfg(vllm_engine_kwargs=VllmEngineConfig(tool_call_parser="llama3_json")) - resolve_all_tool_parsers(ModelshipConfig(models=[cfg])) - assert cfg._resolved_skip_special_tokens is None - - def test_mistral_pins_skip_specials_false(self): - # Mistral parser's marker is a special added token; loader must - # keep specials so the parser sees `[TOOL_CALLS]`. - cfg = _make_cfg(vllm_engine_kwargs=VllmEngineConfig(tool_call_parser="mistral")) - resolve_all_tool_parsers(ModelshipConfig(models=[cfg])) - assert cfg._resolved_skip_special_tokens is False diff --git a/tests/test_qwen3_coder_parser.py b/tests/test_qwen3_coder_parser.py deleted file mode 100644 index 6996d8f..0000000 --- a/tests/test_qwen3_coder_parser.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Qwen3-Coder XML-style tool-call parser tests. - -Covers parsing of the ``…`` -envelope, multi-call back-to-back streams, streaming monotonicity at -marker boundaries, and the ``classify_template`` disambiguation that -keeps Qwen3-Coder templates from falling through to the Hermes parser. -""" - -from __future__ import annotations - -import json - -from modelship.openai.parsers.output import ChatOutputStreamer -from modelship.openai.parsers.tool_calling.parsers.qwen3_coder import Qwen3CoderToolCallParser -from modelship.openai.parsers.tool_calling.utils import classify_template - - -class TestQwen3CoderClassify: - """``classify_template`` must route Qwen3-Coder templates here, not to Hermes.""" - - def test_function_marker_routes_to_qwen3_coder(self): - # The chat template mentions tools (gating clause) and contains - # ```` envelope but no - # ``\n\n\nTokyo\n\n\n" - ) - result = Qwen3CoderToolCallParser().parse(text) - assert len(result.tool_calls) == 1 - assert result.tool_calls[0].function.name == "get_weather" - assert json.loads(result.tool_calls[0].function.arguments) == {"location": "Tokyo"} - - def test_single_tool_call_multi_params(self): - text = ( - "\n" - "\n" - "\nrust async runtimes\n\n" - "\n5\n\n" - "\n" - "" - ) - result = Qwen3CoderToolCallParser().parse(text) - assert result.tool_calls[0].function.name == "search" - # Values are kept as strings — see parser module docstring on why. - assert json.loads(result.tool_calls[0].function.arguments) == { - "query": "rust async runtimes", - "top_k": "5", - } - - def test_inline_values_without_newlines(self): - text = "example.com" - result = Qwen3CoderToolCallParser().parse(text) - assert json.loads(result.tool_calls[0].function.arguments) == {"host": "example.com"} - - def test_two_back_to_back_tool_calls(self): - # Each call is its own ``…`` envelope; the - # streamer treats them as independent regions. - text = ( - "\n\n\n1\n\n\n" - "\n\n\n2\n\n\n" - ) - result = Qwen3CoderToolCallParser().parse(text) - assert len(result.tool_calls) == 2 - assert result.tool_calls[0].function.name == "a" - assert result.tool_calls[1].function.name == "b" - assert json.loads(result.tool_calls[0].function.arguments) == {"x": "1"} - assert json.loads(result.tool_calls[1].function.arguments) == {"y": "2"} - - def test_no_tool_calls_returns_text_unchanged(self): - result = Qwen3CoderToolCallParser().parse("just a regular response") - assert result.tool_calls == [] - assert result.content == "just a regular response" - assert result.has_tool_calls is False - - def test_value_preserves_internal_whitespace(self): - # Leading/trailing whitespace stripped, internal preserved. - text = "\ndef f():\n return 1\n" - result = Qwen3CoderToolCallParser().parse(text) - assert json.loads(result.tool_calls[0].function.arguments) == { - "code": "def f():\n return 1", - } - - -class TestQwen3CoderStreaming: - def test_streaming_basic(self): - parser = Qwen3CoderToolCallParser() - streamer = ChatOutputStreamer(parser) - - d1 = streamer.extract_streaming("\n\n\n\n\nTok") - - # Final close. - streamer.extract_streaming( - "\n\n\nTokyo\n\n\n" - ) - streamer.finalize() - - result = streamer.result - assert len(result.tool_calls) == 1 - assert json.loads(result.tool_calls[0].function.arguments) == {"location": "Tokyo"} - - def test_streaming_monotonic_concat_matches_final(self): - # The concatenation of all streamed arg deltas must equal the - # final arguments JSON (the streamer's prefix-extension contract). - parser = Qwen3CoderToolCallParser() - streamer = ChatOutputStreamer(parser) - chunks = [ - "\n\n\nls", - "\n\n\nls -la", - "\n\n\nls -la\n\n" - "\n/tmp\n\n\n", - ] - seen = "" - for c in chunks: - d = streamer.extract_streaming(c) - if d and d.tool_calls: - for t in d.tool_calls: - if t.function and t.function.arguments: - seen += t.function.arguments - streamer.finalize() - assert json.loads(seen) == {"cmd": "ls -la", "cwd": "/tmp"} - - def test_streaming_chunk_mid_parameter_close(self): - # Regression: a chunk that ends mid-```` (e.g. - # ``\n\n\nval\n\n\nval\n\n", - ] - seen = "" - for c in chunks: - d = streamer.extract_streaming(c) - if d and d.tool_calls: - for t in d.tool_calls: - if t.function and t.function.arguments: - seen += t.function.arguments - streamer.finalize() - assert json.loads(seen) == {"x": "val"} - - def test_streaming_chunk_mid_next_parameter_opener(self): - # Regression: between parameters, a chunk ending in ``\n\n\nfirst\n\n\n\n\nfirst\n\n" - "\nsecond\n\n\n", - ] - seen = "" - for c in chunks: - d = streamer.extract_streaming(c) - if d and d.tool_calls: - for t in d.tool_calls: - if t.function and t.function.arguments: - seen += t.function.arguments - streamer.finalize() - assert json.loads(seen) == {"a": "first", "b": "second"} diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py deleted file mode 100644 index 956acc2..0000000 --- a/tests/test_reasoning.py +++ /dev/null @@ -1,280 +0,0 @@ -"""Tests for the cross-loader reasoning parser toolkit. - -Covers: -- The reasoning parser registry and the DeepSeek-R1 marker pair. -- ``ChatOutputStreamer`` running with only a reasoning parser. -- Composition: reasoning + tool-call parsers together, including the - case where a tool-call marker appears *inside* a reasoning region - (must not be parsed as a real tool call). -""" - -from __future__ import annotations - -import json - -import pytest - -from modelship.openai.parsers.output import ChatOutputStreamer -from modelship.openai.parsers.reasoning import ( - DeepseekR1ReasoningParser, - available_parsers, - get_parser, - register_parser, -) -from modelship.openai.parsers.reasoning.parsers import ReasoningParser -from modelship.openai.parsers.tool_calling.parsers import HermesToolCallParser - - -class TestReasoningRegistry: - def test_default_registry_includes_deepseek_r1(self): - assert "deepseek_r1" in available_parsers() - - def test_get_parser_returns_singleton(self): - a = get_parser("deepseek_r1") - b = get_parser("deepseek_r1") - assert a is b - - def test_unknown_parser_raises_with_available_list(self): - with pytest.raises(ValueError, match="deepseek_r1"): - get_parser("does-not-exist") - - def test_register_parser_makes_it_findable(self): - class Stub(ReasoningParser): - name = "stub-reasoning" - start_marker = "<>" - end_marker = "<>" - - register_parser(Stub()) - try: - assert get_parser("stub-reasoning").name == "stub-reasoning" - finally: - from modelship.openai.parsers.reasoning import registry - - registry._PARSERS.pop("stub-reasoning", None) - - -class TestDeepseekR1Markers: - def test_marker_pair(self): - p = DeepseekR1ReasoningParser() - assert p.start_marker == "" - assert p.end_marker == "" - - -class TestReasoningOnlyStreamer: - """``ChatOutputStreamer`` driven by a reasoning parser alone (no tools).""" - - def _feed(self, chunks: list[str]) -> tuple[ChatOutputStreamer, list]: - streamer = ChatOutputStreamer(None, DeepseekR1ReasoningParser()) - deltas = [] - cumulative = "" - for chunk in chunks: - cumulative += chunk - d = streamer.extract_streaming(cumulative) - if d is not None: - deltas.append(d) - final = streamer.finalize() - if final is not None: - deltas.append(final) - return streamer, deltas - - def test_at_least_one_parser_required(self): - with pytest.raises(ValueError, match="at least one parser"): - ChatOutputStreamer(None, None) - - def test_pure_content_streams_immediately(self): - _, deltas = self._feed(["Hello", " ", "world"]) - assert "".join(d.content or "" for d in deltas) == "Hello world" - assert all((d.reasoning or "") == "" for d in deltas) - - def test_reasoning_block_routed_to_reasoning_field(self): - text = "thinking stepfinal answer" - _, deltas = self._feed([text]) - reasoning = "".join(d.reasoning or "" for d in deltas) - content = "".join(d.content or "" for d in deltas) - assert reasoning == "thinking step" - assert content == "final answer" - - def test_reasoning_streams_incrementally(self): - # Drip-feed a `` block one char at a time. We want at least - # several reasoning deltas, not one big shot at finalize. - text = "abc" - _, deltas = self._feed(list(text)) - reasoning_deltas = [d for d in deltas if d.reasoning] - assert len(reasoning_deltas) >= 2 - assert "".join(d.reasoning or "" for d in deltas) == "abc" - - def test_holds_back_marker_prefix_until_disambiguated(self): - # ``; the streamer must - # not ship those as content mid-stream. - streamer = ChatOutputStreamer(None, DeepseekR1ReasoningParser()) - d = streamer.extract_streaming("hello x") - assert d is None or (d.reasoning or "") == "x" - - def test_held_tail_flushes_when_disambiguated_as_content(self): - # ``; the held tail should flush. - streamer = ChatOutputStreamer(None, DeepseekR1ReasoningParser()) - streamer.extract_streaming("hi `; mid-stream we must not ship it as reasoning bytes - # because the next chunk might prove it was the closer. - streamer = ChatOutputStreamer(None, DeepseekR1ReasoningParser()) - d = streamer.extract_streaming("thinking...` but never closes. Finalize should not - # raise; the open block becomes reasoning content (best-effort - # rendering of what the model produced). - text = "incomplete" - streamer, _ = self._feed([text]) - assert streamer.result.reasoning == "incomplete" - assert streamer.result.content is None - - def test_no_reasoning_no_content_returns_no_delta(self): - streamer = ChatOutputStreamer(None, DeepseekR1ReasoningParser()) - # An empty cumulative text produces nothing. - assert streamer.extract_streaming("") is None - - def test_chunk_boundary_in_open_marker(self): - # The opening `` is split exactly at ``. - streamer = ChatOutputStreamer(None, DeepseekR1ReasoningParser()) - d1 = streamer.extract_streaming("hi r") - d3 = streamer.extract_streaming("hi rok") - # Combine all yielded deltas + finalize. - deltas = [d for d in (d1, d2, d3) if d is not None] - final = streamer.finalize() - if final is not None: - deltas.append(final) - assert "".join(d.reasoning or "" for d in deltas) == "r" - assert "".join(d.content or "" for d in deltas) == "hi ok" - - -class TestComposition: - """ChatOutputStreamer with both reasoning + tool-call parsers active. - - The single-pass design must route tool-call markers that appear - inside a reasoning region to the reasoning view, NOT parse them as - real tool calls. - """ - - def _feed(self, chunks: list[str]) -> tuple[ChatOutputStreamer, list]: - streamer = ChatOutputStreamer(HermesToolCallParser(), DeepseekR1ReasoningParser()) - deltas = [] - cumulative = "" - for chunk in chunks: - cumulative += chunk - d = streamer.extract_streaming(cumulative) - if d is not None: - deltas.append(d) - final = streamer.finalize() - if final is not None: - deltas.append(final) - return streamer, deltas - - def test_reasoning_then_tool_call_then_residual(self): - text = ( - "I should call get_weather" - '{"name": "get_weather", "arguments": {"city": "Paris"}}' - "ok" - ) - streamer, deltas = self._feed([text]) - # Reasoning surfaced. - assert "".join(d.reasoning or "" for d in deltas) == "I should call get_weather" - # Real tool call extracted. - assert len(streamer.result.tool_calls) == 1 - assert streamer.result.tool_calls[0].function.name == "get_weather" - # `ok` ends up in content view. - assert (streamer.result.content or "") == "ok" - - def test_tool_marker_inside_reasoning_is_not_a_real_tool_call(self): - # The model is "thinking out loud" about tool calls inside its - # `` block. Those markers must NOT be parsed as real - # tool calls — they're part of the reasoning text. - text = 'I might need {"name":"x","arguments":{}} lateractual answer' - streamer, _ = self._feed([text]) - assert streamer.result.tool_calls == [] - # Reasoning includes the literal markers because the streamer - # surfaces the reasoning region's bytes verbatim. - assert (streamer.result.reasoning or "").startswith("I might need ") - assert streamer.result.content == "actual answer" - - def test_real_tool_call_after_reasoning_streams_correctly(self): - # Char-by-char to confirm both streams progress incrementally. - text = 'plan{"name": "ping", "arguments": {"x": 1}}' - _, deltas = self._feed(list(text)) - - reasoning = "".join(d.reasoning or "" for d in deltas) - assert reasoning == "plan" - - tool_deltas = [tc for d in deltas for tc in d.tool_calls] - names = [tc.function.name for tc in tool_deltas if tc.function and tc.function.name] - assert names == ["ping"] - joined_args = "".join( - tc.function.arguments or "" for tc in tool_deltas if tc.function and tc.function.arguments - ) - assert json.loads(joined_args) == {"x": 1} - - def test_chunk_boundary_at_marker_seam(self): - # The chunk lands exactly at `......` - # — the streamer must hold `r{"name": "p", "arguments": {}}', - ] - streamer, deltas = self._feed(chunks) - # `hello ` lands as content, never `hello only when enable_thinking is false. @@ -67,18 +66,16 @@ def boom(): assert resolve_active_reasoning_parser("deepseek_r1", boom) == "deepseek_r1" + def test_unknown_candidate_skips_probe(self): + # An explicitly configured vLLM reasoning parser modelship doesn't + # auto-detect (no known start/end markers) — nothing to probe against, + # so the candidate is trusted as-is and the render is never called. + called = False -class TestRequestForcesToolCall: - def test_required(self): - assert request_forces_tool_call("required") is True - - def test_named_function(self): - assert request_forces_tool_call({"type": "function", "function": {"name": "f"}}) is True - - def test_named_function_without_name(self): - assert request_forces_tool_call({"type": "function", "function": {}}) is False + def render(): + nonlocal called + called = True + return "" - def test_auto_and_none(self): - assert request_forces_tool_call("auto") is False - assert request_forces_tool_call("none") is False - assert request_forces_tool_call(None) is False + assert resolve_active_reasoning_parser("qwen3", render) == "qwen3" + assert called is False diff --git a/tests/test_responses_adapter.py b/tests/test_responses_adapter.py index 9c04921..bd12b36 100644 --- a/tests/test_responses_adapter.py +++ b/tests/test_responses_adapter.py @@ -1,21 +1,10 @@ -"""Tests for the stateless Responses <-> chat-completions adapter (Phase A).""" +"""Tests for the Responses -> chat-completions request-side adapter.""" import pytest -from modelship.openai.protocol import ( - ChatCompletionResponse, - ChatCompletionResponseChoice, - ChatMessage, - CompletionTokenUsageInfo, - FunctionCall, - PromptTokenUsageInfo, - ResponsesRequest, - ToolCall, - UsageInfo, -) +from modelship.openai.protocol import ResponsesRequest from modelship.openai.protocol.responses import ( UnsupportedResponsesFeatureError, - chat_response_to_responses, responses_request_to_chat, ) @@ -153,103 +142,3 @@ def test_store_true_is_accepted(self): # store defaults to true on OpenAI; we accept-but-don't-persist rather than reject. chat = responses_request_to_chat(_req(store=True)) assert chat.messages == [{"role": "user", "content": "hello"}] - - -def _chat_response(*, content=None, reasoning=None, tool_calls=None, finish_reason="stop") -> ChatCompletionResponse: - return ChatCompletionResponse( - model="m", - choices=[ - ChatCompletionResponseChoice( - index=0, - message=ChatMessage( - role="assistant", - content=content, - reasoning=reasoning, - tool_calls=tool_calls or [], - ), - finish_reason=finish_reason, - ) - ], - usage=UsageInfo(prompt_tokens=5, completion_tokens=7, total_tokens=12), - ) - - -class TestResponseTranslation: - def test_text_becomes_output_message(self): - resp = chat_response_to_responses(_chat_response(content="hi"), _req()) - assert resp.object == "response" - assert resp.status == "completed" - assert len(resp.output) == 1 - msg = resp.output[0] - assert msg.type == "message" - assert msg.content[0].type == "output_text" - assert msg.content[0].text == "hi" - - def test_reasoning_becomes_reasoning_item_first(self): - resp = chat_response_to_responses(_chat_response(content="answer", reasoning="because"), _req()) - assert resp.output[0].type == "reasoning" - assert resp.output[0].summary[0].text == "because" - assert resp.output[1].type == "message" - - def test_tool_calls_become_function_call_items(self): - tc = ToolCall(id="call_9", function=FunctionCall(name="f", arguments="{}")) - resp = chat_response_to_responses(_chat_response(tool_calls=[tc]), _req()) - fc = [o for o in resp.output if o.type == "function_call"] - assert len(fc) == 1 - assert fc[0].call_id == "call_9" - assert fc[0].name == "f" - - def test_usage_remapped(self): - resp = chat_response_to_responses(_chat_response(content="x"), _req()) - assert resp.usage.input_tokens == 5 - assert resp.usage.output_tokens == 7 - assert resp.usage.total_tokens == 12 - - def test_token_details_propagated_when_present(self): - # vLLM (prefix caching / reasoning models) reports these; they must - # reach the client, not be discarded. - chat = ChatCompletionResponse( - model="m", - choices=[ - ChatCompletionResponseChoice( - index=0, - message=ChatMessage(role="assistant", content="x"), - finish_reason="stop", - ) - ], - usage=UsageInfo( - prompt_tokens=5, - completion_tokens=7, - total_tokens=12, - prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=3), - completion_tokens_details=CompletionTokenUsageInfo(reasoning_tokens=4), - ), - ) - resp = chat_response_to_responses(chat, _req()) - assert resp.usage.input_tokens_details.cached_tokens == 3 - assert resp.usage.output_tokens_details.reasoning_tokens == 4 - - def test_token_details_default_zero_when_absent(self): - resp = chat_response_to_responses(_chat_response(content="x"), _req()) - assert resp.usage.input_tokens_details.cached_tokens == 0 - assert resp.usage.output_tokens_details.reasoning_tokens == 0 - - def test_length_finish_reason_is_incomplete(self): - resp = chat_response_to_responses(_chat_response(content="x", finish_reason="length"), _req()) - assert resp.status == "incomplete" - assert resp.incomplete_details == {"reason": "max_output_tokens"} - - def test_content_filter_finish_reason_is_incomplete(self): - resp = chat_response_to_responses(_chat_response(content="x", finish_reason="content_filter"), _req()) - assert resp.status == "incomplete" - assert resp.incomplete_details == {"reason": "content_filter"} - - def test_store_always_false_and_settings_echoed(self): - resp = chat_response_to_responses( - _chat_response(content="x"), - _req(store=True, instructions="sys", max_output_tokens=64, metadata={"k": "v"}), - ) - assert resp.store is False - assert resp.instructions == "sys" - assert resp.max_output_tokens == 64 - assert resp.metadata == {"k": "v"} diff --git a/tests/test_responses_streaming.py b/tests/test_responses_streaming.py index 2a17b0d..1982be3 100644 --- a/tests/test_responses_streaming.py +++ b/tests/test_responses_streaming.py @@ -1,14 +1,12 @@ -"""Tests for the streaming Responses <-> chat-completions translator (Phase A2). +"""Tests for the streaming Responses <-> chat-completions translator. -The translator consumes the chat SSE chunk stream every loader emits and emits +The translator consumes typed chat stream chunks every loader emits and emits the Responses event protocol. These tests drive it directly with synthesized chat chunks (no Ray, no loader) and assert the event sequence/shape. """ import json -import pytest - from modelship.openai.protocol import ( ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, @@ -18,11 +16,7 @@ ResponsesRequest, UsageInfo, ) -from modelship.openai.protocol.responses.streaming import ( - ResponsesStreamTranslator, - _parse_chat_sse, - responses_stream_from_chat, -) +from modelship.openai.protocol.responses.streaming import ResponsesStreamTranslator def _req(**overrides) -> ResponsesRequest: @@ -73,30 +67,6 @@ def _types(events: list[dict]) -> list[str]: return [e["type"] for e in events] -class TestSseParsing: - def test_parses_data_chunk(self): - chunk = _chunk(DeltaMessage(content="hi")) - raw = f"data: {json.dumps(chunk.model_dump(mode='json'))}\n\n" - parsed = list(_parse_chat_sse(raw)) - assert len(parsed) == 1 - assert parsed[0].choices[0].delta.content == "hi" - - def test_done_sentinel_yields_nothing(self): - assert list(_parse_chat_sse("data: [DONE]\n\n")) == [] - - def test_non_data_line_yields_nothing(self): - assert list(_parse_chat_sse(": keep-alive\n\n")) == [] - - def test_bundled_messages_all_parsed(self): - # A single raw string may bundle several SSE messages; every data line - # must be parsed (not just the first), [DONE] skipped. - a = json.dumps(_chunk(DeltaMessage(content="a")).model_dump(mode="json")) - b = json.dumps(_chunk(DeltaMessage(content="b")).model_dump(mode="json")) - raw = f"data: {a}\n\ndata: {b}\n\ndata: [DONE]\n\n" - parsed = list(_parse_chat_sse(raw)) - assert [c.choices[0].delta.content for c in parsed] == ["a", "b"] - - class TestTextStream: def test_envelope_and_text_event_sequence(self): translator = ResponsesStreamTranslator(_req()) @@ -297,31 +267,3 @@ def test_fail_mid_tool_call_closes_it_with_partial_arguments(self): assert failed["output"][0]["type"] == "function_call" assert failed["output"][0]["arguments"] == '{"lo' assert failed["output"][0]["status"] == "completed" - - -class TestStreamWrapper: - @pytest.mark.asyncio - async def test_wraps_chat_sse_strings_into_events(self): - async def gen(): - yield f"data: {json.dumps(_chunk(DeltaMessage(content='hi')).model_dump(mode='json'))}\n\n" - yield f"data: {json.dumps(_chunk(finish_reason='stop').model_dump(mode='json'))}\n\n" - yield "data: [DONE]\n\n" - - out = [e async for e in responses_stream_from_chat(gen(), _req())] - assert all(isinstance(e, str) for e in out) - assert out[0].startswith("event: response.created") - assert "event: response.completed" in out[-1] - - @pytest.mark.asyncio - async def test_pre_stream_error_object_passes_through(self): - from modelship.openai.protocol import create_error_response - - err = create_error_response("nope") - - async def gen(): - yield err - - out = [e async for e in responses_stream_from_chat(gen(), _req())] - # Passed straight through (not turned into a 200 event stream) so - # _handle_response can render the proper HTTP error. - assert out == [err] diff --git a/tests/test_tool_calling.py b/tests/test_tool_calling.py deleted file mode 100644 index 730252b..0000000 --- a/tests/test_tool_calling.py +++ /dev/null @@ -1,760 +0,0 @@ -"""Tests for the cross-loader tool-calling toolkit.""" - -from __future__ import annotations - -import json -from typing import ClassVar - -import pytest - -from modelship.openai.parsers.output import ChatOutputStreamer -from modelship.openai.parsers.streaming import parse_chat_completion_text -from modelship.openai.parsers.tool_calling import ( - available_parsers, - get_parser, - register_parser, - resolve_tools_for_request, -) -from modelship.openai.parsers.tool_calling.parsers import ( - HermesToolCallParser, - Llama3JsonToolCallParser, - MistralToolCallParser, - ToolCallParser, -) - - -class TestRegistry: - def test_default_registry_includes_hermes(self): - assert "hermes" in available_parsers() - - def test_default_registry_includes_mistral(self): - assert "mistral" in available_parsers() - - def test_default_registry_includes_llama3_json(self): - assert "llama3_json" in available_parsers() - - def test_get_parser_returns_singleton(self): - a = get_parser("hermes") - b = get_parser("hermes") - assert a is b - - def test_unknown_parser_raises_with_available_list(self): - with pytest.raises(ValueError, match="hermes"): - get_parser("does-not-exist") - - def test_register_parser_makes_it_findable(self): - class Stub(ToolCallParser): - name = "stub-test-parser" - start_marker = "<<" - end_marker = ">>" - - def extract_partial_name(self, partial_payload: str) -> str | None: - return None - - def extract_partial_args(self, partial_payload: str) -> str | None: - return None - - register_parser(Stub()) - try: - assert get_parser("stub-test-parser").name == "stub-test-parser" - finally: - # Clean up so other tests don't see the stub. - from modelship.openai.parsers.tool_calling import registry - - registry._PARSERS.pop("stub-test-parser", None) - - -class TestHermesParser: - parser = HermesToolCallParser() - - def test_no_tool_calls_returns_text_unchanged(self): - result = self.parser.parse("just a regular response") - assert result.tool_calls == [] - assert result.content == "just a regular response" - assert result.has_tool_calls is False - - def test_single_tool_call(self): - text = '{"name": "get_weather", "arguments": {"city": "Paris"}}' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert result.tool_calls[0].function.name == "get_weather" - assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Paris"} - assert result.content is None - - def test_multiple_tool_calls(self): - text = ( - '{"name": "a", "arguments": {"x": 1}}' - '{"name": "b", "arguments": {"y": 2}}' - ) - result = self.parser.parse(text) - assert [tc.function.name for tc in result.tool_calls] == ["a", "b"] - - def test_tool_call_with_residual_text(self): - # Trailing whitespace in the preamble is preserved — only an - # all-whitespace residual gets nulled out alongside tool_calls. - text = 'Sure, calling that.\n{"name": "ping", "arguments": {}}' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert result.content == "Sure, calling that.\n" - - def test_whitespace_only_preamble_nulls_content(self): - text = ' \n{"name": "ping", "arguments": {}}' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert result.content is None - - def test_string_arguments_forwarded_verbatim(self): - # vLLM-style: the streamer forwards the raw bytes of the arguments - # value as the model emitted them, including any surrounding quotes - # if the model wrapped its arguments in a JSON string literal. The - # OpenAI streaming contract treats `arguments` as an opaque string - # the client concatenates and parses. - text = '{"name": "x", "arguments": "{\\"a\\": 1}"}' - result = self.parser.parse(text) - assert result.tool_calls[0].function.arguments == '"{\\"a\\": 1}"' - - def test_object_arguments_passed_through(self): - text = '{"name": "x", "arguments": {"a": 1, "b": [2, 3]}}' - result = self.parser.parse(text) - assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1, "b": [2, 3]} - - def test_block_without_extractable_name_is_dropped(self): - # When the block contains nothing the name regex can hook onto we - # silently drop it — there is nothing to tell the client about. - text = "{not valid json}" - result = self.parser.parse(text) - assert result.tool_calls == [] - - def test_missing_name_drops_call(self): - text = '{"arguments": {}}' - result = self.parser.parse(text) - assert result.tool_calls == [] - - def test_empty_name_drops_call(self): - text = '{"name": "", "arguments": {}}' - result = self.parser.parse(text) - assert result.tool_calls == [] - - def test_each_tool_call_gets_unique_id(self): - text = ( - '{"name": "a", "arguments": {}}{"name": "b", "arguments": {}}' - ) - result = self.parser.parse(text) - assert result.tool_calls[0].id != result.tool_calls[1].id - - -class TestChatOutputStreamer: - """Drive the streamer one chunk at a time and verify the deltas. - - The cumulative-text protocol matches what serving_chat does in production: - every fed string is the *full* generated text so far, not just the latest - delta. - """ - - def _feed(self, chunks: list[str]) -> tuple[ChatOutputStreamer, list]: - streamer = ChatOutputStreamer(HermesToolCallParser()) - deltas = [] - cumulative = "" - for chunk in chunks: - cumulative += chunk - d = streamer.extract_streaming(cumulative) - if d is not None: - deltas.append(d) - final = streamer.finalize() - if final is not None: - deltas.append(final) - return streamer, deltas - - def test_pure_content_streams_immediately(self): - _, deltas = self._feed(["Hello", " ", "world"]) - assert "".join(d.content or "" for d in deltas) == "Hello world" - assert all(not d.tool_calls for d in deltas) - - def test_holds_back_marker_prefix_in_content_until_finalize(self): - # `<` could be the first char of ``; the streamer must not - # ship it mid-stream. Once finalize() runs (no more text coming) the - # held tail is safe to flush as content. - streamer = ChatOutputStreamer(HermesToolCallParser()) - mid = streamer.extract_streaming("before <") - assert mid is not None and mid.content == "before " - final = streamer.finalize() - assert final is not None and final.content == "<" - - def test_held_tail_flushes_when_disambiguated(self): - # `{"name": "get_weather", "arguments": {"city": "Paris"}}') - _, deltas = self._feed(chunks) - - tool_deltas = [tc for d in deltas for tc in d.tool_calls] - # First tool delta: name + id, no arguments. - assert tool_deltas[0].function is not None - assert tool_deltas[0].function.name == "get_weather" - assert tool_deltas[0].id is not None - assert tool_deltas[0].function.arguments is None - # Subsequent deltas carry arguments fragments only (no name). - arg_deltas = tool_deltas[1:] - assert all(d.function and d.function.name is None for d in arg_deltas) - # Concatenated arguments form valid JSON. - joined_args = "".join(d.function.arguments or "" for d in arg_deltas) - assert json.loads(joined_args) == {"city": "Paris"} - - def test_arguments_stream_incrementally(self): - # Feed the args char-by-char; each char-after-name should generate - # an args delta of length 1 (or close to it). - prefix = '{"name": "ping", "arguments": ' - suffix = '{"x": 42}}' - _, deltas = self._feed([prefix, *list(suffix)]) - - arg_deltas = [tc for d in deltas for tc in d.tool_calls if tc.function and tc.function.arguments is not None] - assert len(arg_deltas) >= 3 # incremental, not one big shot - joined = "".join(d.function.arguments or "" for d in arg_deltas) - assert json.loads(joined) == {"x": 42} - - def test_multiple_tool_calls_get_distinct_indices(self): - text = ( - '{"name": "a", "arguments": {"x": 1}}' - '{"name": "b", "arguments": {"y": 2}}' - ) - _, deltas = self._feed([text]) - - tool_deltas = [tc for d in deltas for tc in d.tool_calls] - indices = {d.index for d in tool_deltas} - assert indices == {0, 1} - names = [d.function.name for d in tool_deltas if d.function and d.function.name] - assert names == ["a", "b"] - - def test_content_after_tool_call_resumes_streaming(self): - text = '{"name": "p", "arguments": {}} ok' - _, deltas = self._feed([text]) - joined_content = "".join(d.content or "" for d in deltas) - assert "ok" in joined_content - - def test_partial_name_held_until_closing_quote(self): - # While the model is mid-name (``"name": "get_wea`` so far), the - # streamer must NOT send a partial name — wait for the closing quote. - streamer = ChatOutputStreamer(HermesToolCallParser()) - partial = '{"name": "get_wea' - d = streamer.extract_streaming(partial) - # No name yet → no tool delta. - assert d is None or all(not tc.function or not tc.function.name for tc in d.tool_calls) - - def test_unterminated_block_is_finalized_without_crash(self): - # Model stops mid tool-call. The streamer should not raise on finalize - # and should not claim a finalized ToolCall (the call wasn't closed). - text = '{"name": "incomplete", "arguments": {"a": 1' - streamer, _ = self._feed([text]) - assert streamer.result.tool_calls == [] - - def test_skipped_malformed_block_does_not_block_subsequent_finalization(self): - # If the first tool-call block is malformed (missing name), it should be - # skipped, but the SECOND block (if valid) should still be finalized. - text = ( - '{"arguments": {"skipped": true}}' - '{"name": "valid", "arguments": {"x": 1}}' - ) - streamer, _ = self._feed([text]) - assert [tc.function.name for tc in streamer.result.tool_calls] == ["valid"] - - -class TestParseFullFinalization: - """Non-streaming one-shot parsing via ``parse_chat_completion_text``. - - Tool-call finalization happens on ``finalize()``, so a single - ``extract_streaming`` + ``finalize`` pass over the full text must - yield the same result the incremental streaming drive does. - """ - - def test_single_complete_call_one_shot(self): - # Closing marker at the very end of the text: the one-shot pass - # must still finalize the call (this is the reported llama_cpp bug). - - text = '\n{"name": "HassTurnOff", "arguments": {"area": "Living Room"}}\n' - parsed = parse_chat_completion_text(text, parser_name="hermes", reasoning_parser_name=None) - assert parsed.content is None - assert len(parsed.tool_calls) == 1 - assert parsed.tool_calls[0].function.name == "HassTurnOff" - assert parsed.tool_calls[0].function.arguments == '{"area": "Living Room"}' - - def test_one_shot_matches_incremental_drive(self): - text = '\n{"name": "HassTurnOff", "arguments": {"area": "Living Room"}}\n' - one_shot = parse_chat_completion_text(text, parser_name="hermes", reasoning_parser_name=None) - - streamer = ChatOutputStreamer(HermesToolCallParser()) - cumulative = "" - for ch in text: - cumulative += ch - streamer.extract_streaming(cumulative) - streamer.finalize() - incremental = streamer.result - - assert one_shot.content == incremental.content - assert [tc.function.name for tc in one_shot.tool_calls] == [tc.function.name for tc in incremental.tool_calls] - assert [tc.function.arguments for tc in one_shot.tool_calls] == [ - tc.function.arguments for tc in incremental.tool_calls - ] - - def test_content_only_text_unchanged(self): - text = "Just a plain answer, no tools here." - parsed = parse_chat_completion_text(text, parser_name="hermes", reasoning_parser_name=None) - assert parsed.content == text - assert parsed.tool_calls == [] - - def test_two_calls_one_shot_both_finalized(self): - text = ( - '\n{"name": "a", "arguments": {"x": 1}}\n' - '\n{"name": "b", "arguments": {"y": 2}}\n' - ) - parsed = parse_chat_completion_text(text, parser_name="hermes", reasoning_parser_name=None) - assert [tc.function.name for tc in parsed.tool_calls] == ["a", "b"] - assert parsed.tool_calls[0].id != parsed.tool_calls[1].id - - def test_finalize_does_not_double_emit_after_streaming(self): - # The whole call is streamed and finalized during extract_streaming; - # finalize() re-runs tool extraction but must stay idempotent — no - # duplicate name/args deltas and no duplicate finalized call. - text = '{"name": "get_weather", "arguments": {"city": "Paris"}}' - streamer = ChatOutputStreamer(HermesToolCallParser()) - cumulative = "" - stream_deltas = [] - for ch in text: - cumulative += ch - d = streamer.extract_streaming(cumulative) - if d is not None: - stream_deltas.append(d) - - final = streamer.finalize() - # Nothing new to emit on finalize — the call already streamed fully. - assert final is None or not final.tool_calls - - name_deltas = [tc for d in stream_deltas for tc in d.tool_calls if tc.function and tc.function.name] - assert [d.function.name for d in name_deltas] == ["get_weather"] - assert len(streamer.result.tool_calls) == 1 - - -class TestResolveToolsForRequest: - tools: ClassVar = [ - {"type": "function", "function": {"name": "alpha"}}, - {"type": "function", "function": {"name": "beta"}}, - ] - - def test_no_tools_returns_none(self): - assert resolve_tools_for_request(None, "auto") is None - assert resolve_tools_for_request([], "auto") is None - - def test_auto_passes_through(self): - assert resolve_tools_for_request(self.tools, "auto") == self.tools - - def test_unset_tool_choice_passes_through(self): - assert resolve_tools_for_request(self.tools, None) == self.tools - - def test_none_suppresses_tools(self): - assert resolve_tools_for_request(self.tools, "none") is None - - def test_required_passes_through(self): - # We cannot strictly enforce a tool call without constrained decoding, - # so "required" downgrades to "auto" semantics (with a logged warning). - assert resolve_tools_for_request(self.tools, "required") == self.tools - - def test_specific_function_filters_to_that_tool(self): - result = resolve_tools_for_request(self.tools, {"type": "function", "function": {"name": "beta"}}) - assert result is not None - assert len(result) == 1 - assert result[0]["function"]["name"] == "beta" - - def test_unknown_function_falls_back_to_all(self): - # If the named function isn't in the tools list, fall back to passing - # them all through rather than emitting an empty list. - result = resolve_tools_for_request(self.tools, {"type": "function", "function": {"name": "missing"}}) - assert result == self.tools - - def test_unrecognized_choice_falls_back_to_all(self): - result = resolve_tools_for_request(self.tools, "weird-mode") - assert result == self.tools - - -class TestMistralParser: - parser = MistralToolCallParser() - - def test_no_tool_calls_returns_text_unchanged(self): - result = self.parser.parse("just a regular response") - assert result.tool_calls == [] - assert result.content == "just a regular response" - assert result.has_tool_calls is False - - def test_single_tool_call(self): - text = '[TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}}]' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert result.tool_calls[0].function.name == "get_weather" - assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Paris"} - assert result.content is None - - def test_multiple_tool_calls_in_one_envelope(self): - # The defining feature of Mistral: multiple calls share one - # `[TOOL_CALLS]` envelope, separated by commas inside a JSON array. - text = '[TOOL_CALLS][{"name": "a", "arguments": {"x": 1}}, {"name": "b", "arguments": {"y": 2}}]' - result = self.parser.parse(text) - assert [tc.function.name for tc in result.tool_calls] == ["a", "b"] - assert json.loads(result.tool_calls[0].function.arguments) == {"x": 1} - assert json.loads(result.tool_calls[1].function.arguments) == {"y": 2} - - def test_tool_call_with_residual_text(self): - text = 'Sure, calling that.\n[TOOL_CALLS][{"name": "ping", "arguments": {}}]' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert result.content == "Sure, calling that.\n" - - def test_whitespace_only_preamble_nulls_content(self): - text = ' \n[TOOL_CALLS][{"name": "ping", "arguments": {}}]' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert result.content is None - - def test_object_arguments_passed_through(self): - text = '[TOOL_CALLS][{"name": "x", "arguments": {"a": 1, "b": [2, 3]}}]' - result = self.parser.parse(text) - assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1, "b": [2, 3]} - - def test_string_arguments_forwarded_verbatim(self): - text = '[TOOL_CALLS][{"name": "x", "arguments": "{\\"a\\": 1}"}]' - result = self.parser.parse(text) - assert result.tool_calls[0].function.arguments == '"{\\"a\\": 1}"' - - def test_braces_inside_string_args_do_not_break_split(self): - # The split walker tracks string state; literal `{` / `}` inside - # string values must not move the brace-depth counter. - text = '[TOOL_CALLS][{"name": "x", "arguments": {"q": "}{ literal"}}]' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert json.loads(result.tool_calls[0].function.arguments) == {"q": "}{ literal"} - - def test_block_without_extractable_name_is_dropped(self): - text = "[TOOL_CALLS][{not valid json}]" - result = self.parser.parse(text) - assert result.tool_calls == [] - - def test_missing_name_drops_call(self): - text = '[TOOL_CALLS][{"arguments": {}}]' - result = self.parser.parse(text) - assert result.tool_calls == [] - - def test_each_tool_call_gets_unique_id(self): - text = '[TOOL_CALLS][{"name": "a", "arguments": {}}, {"name": "b", "arguments": {}}]' - result = self.parser.parse(text) - assert result.tool_calls[0].id != result.tool_calls[1].id - - -class TestMistralStreamer: - """Stream a Mistral envelope incrementally and verify deltas. - - Mistral's region is EOS-bounded (empty `end_marker`), so completion - happens at finalize time. The split_payload override carves the JSON - array into per-call sub-blocks that flow through the same streamer - machinery used by Hermes. - """ - - def _feed(self, chunks: list[str]) -> tuple[ChatOutputStreamer, list]: - streamer = ChatOutputStreamer(MistralToolCallParser()) - deltas = [] - cumulative = "" - for chunk in chunks: - cumulative += chunk - d = streamer.extract_streaming(cumulative) - if d is not None: - deltas.append(d) - final = streamer.finalize() - if final is not None: - deltas.append(final) - return streamer, deltas - - def test_emits_name_before_args_for_single_call(self): - chunks = list('[TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}}]') - streamer, deltas = self._feed(chunks) - - tool_deltas = [tc for d in deltas for tc in d.tool_calls] - assert tool_deltas[0].function is not None - assert tool_deltas[0].function.name == "get_weather" - assert tool_deltas[0].id is not None - assert tool_deltas[0].function.arguments is None - joined_args = "".join(d.function.arguments or "" for d in tool_deltas[1:]) - assert json.loads(joined_args) == {"city": "Paris"} - # Region is EOS-bounded → finalize closes it out. - assert len(streamer.result.tool_calls) == 1 - - def test_two_calls_in_one_envelope_finalize_with_distinct_indices(self): - text = '[TOOL_CALLS][{"name": "a", "arguments": {"x": 1}}, {"name": "b", "arguments": {"y": 2}}]' - streamer, deltas = self._feed([text]) - - tool_deltas = [tc for d in deltas for tc in d.tool_calls] - indices = {d.index for d in tool_deltas} - assert indices == {0, 1} - names = [d.function.name for d in tool_deltas if d.function and d.function.name] - assert names == ["a", "b"] - assert [tc.function.name for tc in streamer.result.tool_calls] == ["a", "b"] - - def test_preamble_holds_back_marker_prefix(self): - # `[T` could be the start of `[TOOL_CALLS]`; the streamer must hold - # it until the byte after `[T` proves it is or isn't the marker. - streamer = ChatOutputStreamer(MistralToolCallParser()) - mid = streamer.extract_streaming("hello [T") - assert mid is not None and mid.content == "hello " - final = streamer.finalize() - assert final is not None and final.content == "[T" - - def test_no_envelope_streams_as_pure_content(self): - _, deltas = self._feed(["plain ", "answer ", "no tools"]) - assert "".join(d.content or "" for d in deltas) == "plain answer no tools" - - def test_unterminated_array_finalizes_complete_calls(self): - # The model emitted one full call and started a second that never - # closed. The first should finalize; the second should be dropped. - text = '[TOOL_CALLS][{"name": "done", "arguments": {"x": 1}}, {"name": "wip"' - streamer, _ = self._feed([text]) - assert [tc.function.name for tc in streamer.result.tool_calls] == ["done"] - - -class TestSplitPayloadDefault: - """Regression: the default `split_payload` keeps Hermes one-call-per-region behavior.""" - - def test_default_returns_single_entry(self): - parser = HermesToolCallParser() - assert parser.split_payload("anything", True) == [("anything", True)] - assert parser.split_payload("anything", False) == [("anything", False)] - - -class TestMarkersAreSpecialsFlag: - """Each parser's ``markers_are_specials`` flag gates the loader-side flip.""" - - def test_hermes_marker_is_regular_text(self): - assert HermesToolCallParser().markers_are_specials is False - - def test_llama3_json_marker_is_regular_text(self): - from modelship.openai.parsers.tool_calling.parsers import Llama3JsonToolCallParser - - assert Llama3JsonToolCallParser().markers_are_specials is False - - def test_mistral_marker_is_a_special_token(self): - # Drives loaders that detokenize raw output to flip - # ``skip_special_tokens=False`` at startup so ``[TOOL_CALLS]`` - # survives detokenization. - assert MistralToolCallParser().markers_are_specials is True - - -class TestStreamerNoiseStripping: - """When the loader keeps specials, the streamer drops every noise marker - (everything other than the parser's own start/end markers) before scanning. - - The cumulative-text design means a special token that was split across - HF chunks is reassembled before the streamer ever sees it, so per-chunk - holdback isn't needed at this layer — we just ``replace`` over the full - accumulated buffer every diff pass. - """ - - def test_eos_token_does_not_leak_into_content(self): - streamer = ChatOutputStreamer(MistralToolCallParser(), noise_specials=("", "<|eot_id|>")) - # Pure content path with the model's EOS appended. - d1 = streamer.extract_streaming("hello world") - # Content delta should exclude the EOS bytes. - content = "".join(d.content or "" for d in [d1] if d is not None) - assert content == "hello world" - - def test_tool_marker_is_kept_when_listed_as_keep(self): - # ``[TOOL_CALLS]`` is the parser's own start marker — even if it's - # also "special" in the tokenizer, it must survive because the - # loader includes it in the keep set, NOT in noise_specials. - streamer = ChatOutputStreamer( - MistralToolCallParser(), - noise_specials=("[INST]", "[/INST]", ""), - ) - text = '[TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}}]' - streamer.extract_streaming(text) - streamer.finalize() - result = streamer.result - assert len(result.tool_calls) == 1 - assert result.tool_calls[0].function.name == "get_weather" - # The trailing must not have leaked into content. - assert result.content is None or "" not in result.content - - def test_noise_split_across_cumulative_passes_is_caught(self): - # First pass sees `<|eot_i`, the next pass sees the full `<|eot_id|>`. - # Because the streamer re-strips against the full cumulative buffer - # every diff pass, the partial isn't yielded and the full marker - # is removed once it arrives. - streamer = ChatOutputStreamer(MistralToolCallParser(), noise_specials=("<|eot_id|>",)) - d1 = streamer.extract_streaming("done <|eot_i") - d2 = streamer.extract_streaming("done <|eot_id|>") - final = streamer.finalize() - all_content = "".join(d.content or "" for d in (d1, d2, final) if d is not None) - assert "<|eot_id|>" not in all_content - # The "<|eot_i" partial must not have been forwarded as content. - assert "<|eot_i" not in all_content - - def test_empty_noise_specials_is_a_no_op(self): - streamer = ChatOutputStreamer(HermesToolCallParser(), noise_specials=()) - d = streamer.extract_streaming("plain text") - assert d is not None and d.content == "plain text" - - -class TestLlama3JsonParser: - parser = Llama3JsonToolCallParser() - - def test_no_tool_calls_returns_text_unchanged(self): - result = self.parser.parse("just a regular response") - assert result.tool_calls == [] - assert result.content == "just a regular response" - assert result.has_tool_calls is False - - def test_single_tool_call_with_parameters_field(self): - # Meta's official Llama-3.1 JSON template uses ``parameters``. - text = '{"name": "get_weather", "parameters": {"city": "Paris"}}' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert result.tool_calls[0].function.name == "get_weather" - assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Paris"} - assert result.content is None - - def test_single_tool_call_with_arguments_field(self): - # Some community fine-tunes emit ``arguments`` instead — accept both. - text = '{"name": "get_weather", "arguments": {"city": "Paris"}}' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert result.tool_calls[0].function.name == "get_weather" - assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Paris"} - - def test_multiple_tool_calls_separated_by_semicolon(self): - # vLLM convention for multi-call output. - text = '{"name": "a", "parameters": {"x": 1}}; {"name": "b", "parameters": {"y": 2}}' - result = self.parser.parse(text) - assert [tc.function.name for tc in result.tool_calls] == ["a", "b"] - assert json.loads(result.tool_calls[0].function.arguments) == {"x": 1} - assert json.loads(result.tool_calls[1].function.arguments) == {"y": 2} - - def test_tool_call_with_residual_text(self): - text = 'Sure, calling that.\n{"name": "ping", "parameters": {}}' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert result.content == "Sure, calling that.\n" - - def test_whitespace_only_preamble_nulls_content(self): - text = ' \n{"name": "ping", "parameters": {}}' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert result.content is None - - def test_object_arguments_passed_through(self): - text = '{"name": "x", "parameters": {"a": 1, "b": [2, 3]}}' - result = self.parser.parse(text) - assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1, "b": [2, 3]} - - def test_string_arguments_forwarded_verbatim(self): - text = '{"name": "x", "parameters": "{\\"a\\": 1}"}' - result = self.parser.parse(text) - assert result.tool_calls[0].function.arguments == '"{\\"a\\": 1}"' - - def test_braces_inside_string_args_do_not_break_split(self): - # The split walker tracks string state so literal `{` / `}` inside - # string values must not move the brace-depth counter. - text = '{"name": "x", "parameters": {"q": "}{ literal"}}' - result = self.parser.parse(text) - assert len(result.tool_calls) == 1 - assert json.loads(result.tool_calls[0].function.arguments) == {"q": "}{ literal"} - - def test_block_without_extractable_name_is_dropped(self): - # The bare-JSON marker is `{"name"` — a payload that doesn't even - # contain `"name"` won't trip the marker, so no tool call surfaces. - text = '{"foo": "bar"}' - result = self.parser.parse(text) - assert result.tool_calls == [] - - def test_each_tool_call_gets_unique_id(self): - text = '{"name": "a", "parameters": {}}; {"name": "b", "parameters": {}}' - result = self.parser.parse(text) - assert result.tool_calls[0].id != result.tool_calls[1].id - - -class TestLlama3JsonStreamer: - """Stream a Llama-3.1 JSON tool call incrementally and verify deltas. - - The region is EOS-bounded (empty `end_marker`) like Mistral, so completion - happens at finalize time. The marker `{"name"` is NOT consumed by the - streamer so the JSON object stays intact for the body extractors. - """ - - def _feed(self, chunks: list[str]) -> tuple[ChatOutputStreamer, list]: - streamer = ChatOutputStreamer(Llama3JsonToolCallParser()) - deltas = [] - cumulative = "" - for chunk in chunks: - cumulative += chunk - d = streamer.extract_streaming(cumulative) - if d is not None: - deltas.append(d) - final = streamer.finalize() - if final is not None: - deltas.append(final) - return streamer, deltas - - def test_emits_name_before_args_for_single_call(self): - chunks = list('{"name": "get_weather", "parameters": {"city": "Paris"}}') - streamer, deltas = self._feed(chunks) - - tool_deltas = [tc for d in deltas for tc in d.tool_calls] - assert tool_deltas[0].function is not None - assert tool_deltas[0].function.name == "get_weather" - assert tool_deltas[0].id is not None - assert tool_deltas[0].function.arguments is None - joined_args = "".join(d.function.arguments or "" for d in tool_deltas[1:]) - assert json.loads(joined_args) == {"city": "Paris"} - # Region is EOS-bounded → finalize closes it out. - assert len(streamer.result.tool_calls) == 1 - - def test_two_calls_separated_by_semicolon_finalize_with_distinct_indices(self): - text = '{"name": "a", "parameters": {"x": 1}}; {"name": "b", "parameters": {"y": 2}}' - streamer, deltas = self._feed([text]) - - tool_deltas = [tc for d in deltas for tc in d.tool_calls] - indices = {d.index for d in tool_deltas} - assert indices == {0, 1} - names = [d.function.name for d in tool_deltas if d.function and d.function.name] - assert names == ["a", "b"] - assert [tc.function.name for tc in streamer.result.tool_calls] == ["a", "b"] - - def test_preamble_holds_back_marker_prefix(self): - # A leading `{` could be the first byte of `{"name"`; the streamer must - # hold it until the next byte proves it is or isn't the marker. - streamer = ChatOutputStreamer(Llama3JsonToolCallParser()) - mid = streamer.extract_streaming("hello {") - assert mid is not None and mid.content == "hello " - final = streamer.finalize() - assert final is not None and final.content == "{" - - def test_no_marker_streams_as_pure_content(self): - # Plain text response with no JSON at all goes through as content. - _, deltas = self._feed(["plain ", "answer ", "no tools"]) - assert "".join(d.content or "" for d in deltas) == "plain answer no tools" - - def test_unterminated_call_drops_partial_at_finalize(self): - # Model emitted a full call and started a second that never closed. - # The first should finalize; the second is dropped. - text = '{"name": "done", "parameters": {"x": 1}}; {"name": "wip"' - streamer, _ = self._feed([text]) - assert [tc.function.name for tc in streamer.result.tool_calls] == ["done"] - - def test_arguments_stream_incrementally(self): - prefix = '{"name": "ping", "parameters": ' - suffix = '{"x": 42}}' - _, deltas = self._feed([prefix, *list(suffix)]) - - arg_deltas = [tc for d in deltas for tc in d.tool_calls if tc.function and tc.function.arguments is not None] - assert len(arg_deltas) >= 3 - joined = "".join(d.function.arguments or "" for d in arg_deltas) - assert json.loads(joined) == {"x": 42}