From c1fe9f3688b1deabbca71a41b2e1229dfac7f2bf Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Wed, 10 Sep 2025 00:58:45 -0700
Subject: [PATCH 01/19] add content

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../deployment-serve-llm/gpt-oss/Dockerfile   |   8 +
 .../deployment-serve-llm/gpt-oss/README.md    | 386 ++++++++++++++
 .../deployment-serve-llm/gpt-oss/client.py    |  30 ++
 .../gpt-oss/notebook.ipynb                    | 488 ++++++++++++++++++
 .../gpt-oss/serve_gpt_oss.py                  |  46 ++
 .../deployment-serve-llm/gpt-oss/service.yaml |  10 +
 6 files changed, 968 insertions(+)
 create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/Dockerfile
 create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
 create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/client.py
 create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
 create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
 create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/service.yaml

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/Dockerfile b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/Dockerfile
new file mode 100644
index 000000000000..a48046d90fb6
--- /dev/null
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/Dockerfile
@@ -0,0 +1,8 @@
+FROM anyscale/ray:2.49.1-slim-py312-cu128
+
+# C compiler for Triton’s runtime build step (vLLM V1 engine)
+# https://github.com/vllm-project/vllm/issues/2997
+RUN sudo apt-get update && \
+    sudo apt-get install -y --no-install-recommends build-essential
+
+RUN pip install vllm==0.10.1
\ No newline at end of file
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
new file mode 100644
index 000000000000..d6792b3feac5
--- /dev/null
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
@@ -0,0 +1,386 @@
+---
+orphan: true
+---
+
+<!--
+Do not modify this README. This file is a copy of the notebook and is not used to display the content.
+Modify notebook.ipynb instead, then regenerate this file with:
+jupyter nbconvert "$notebook.ipynb" --to markdown --output "README.md"
+-->
+
+# Deploy gpt-oss
+
+[gpt-oss](https://huggingface.co/collections/openai/gpt-oss-68911959590a1634ba11c7a4) is a family of open-source models designed for general-purpose language understanding and generation. The 20&nbsp;B parameter variant (`gpt-oss-20b`) offers strong reasoning capabilities with lower latency, making it well-suited for local or specialized use cases. The larger 120&nbsp;B parameter variant (`gpt-oss-120b`) is designed for production-scale, high-reasoning workloads.
+
+---
+
+## Configure Ray Serve LLM
+
+Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object.
+
+Below are example configurations for both gpt-oss-20b and gpt-oss-120b, depending on your hardware and use case.
+
+---
+
+### gpt-oss-20b
+
+To deploy a small-sized model such as gpt-oss-20b, a single GPU is sufficient:
+
+
+```python
+# serve_gpt_oss.py
+from ray.serve.llm import LLMConfig, build_openai_app
+
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="my-gpt-oss",
+        model_source="openai/gpt-oss-20b",
+    ),
+    accelerator_type="L4",
+    deployment_config=dict(
+        autoscaling_config=dict(
+            min_replicas=1,
+            max_replicas=2,
+        )
+    ),
+    engine_kwargs=dict(
+        max_model_len=32768
+    ),
+)
+
+app = build_openai_app({"llm_configs": [llm_config]})
+```
+
+
+---
+
+### gpt-oss-120b
+
+To deploy a medium-sized model such as `gpt-oss-120b`, a single node with multiple GPUs is sufficient. Set `tensor_parallel_size` to distribute the model’s weights across the GPUs in your instance:
+
+
+```python
+# serve_gpt_oss.py
+from ray.serve.llm import LLMConfig, build_openai_app
+
+llm_config = LLMConfig(
+    model_loading_config=dict(
+        model_id="my-gpt-oss",
+        model_source="openai/gpt-oss-120b",
+    ),
+    accelerator_type="A100-80G",
+    deployment_config=dict(
+        autoscaling_config=dict(
+            min_replicas=1,
+            max_replicas=2,
+        )
+    ),
+    engine_kwargs=dict(
+        max_model_len=32768,
+        tensor_parallel_size=2,
+    ),
+)
+
+app = build_openai_app({"llm_configs": [llm_config]})
+```
+
+**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example.
+
+---
+
+## Deploy locally
+
+**Prerequisites**
+
+* Access to GPU compute.
+
+**Dependencies:**
+gpt-oss integration is available starting from `ray>=2.49.0` and `vllm>=0.10.1`
+```bash
+pip install "ray[serve,llm]>=2.49.0"
+pip install "vllm>=0.10.1"
+```
+
+---
+
+### Launch
+
+Follow the instructions in [Configure Ray Serve LLM](#configure-ray-serve-llm) according to the model size you choose, and define your app in a Python module `serve_gpt_oss.py`.
+
+In a terminal, run:
+
+
+```bash
+%%bash
+serve run serve_gpt_oss:app --non-blocking
+```
+
+Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded.
+
+---
+
+### Send requests
+
+Your endpoint is available locally at `http://localhost:8000`. You can use a placeholder authentication token for the OpenAI client, for example `"FAKE_KEY"`.
+
+**Example curl:**
+
+
+```bash
+%%bash
+curl -X POST http://localhost:8000/v1/chat/completions \
+  -H "Authorization: Bearer FAKE_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{ "model": "my-gpt-oss", "messages": [{"role": "user", "content": "How many Rs in strawberry ?"}] }'
+```
+
+**Example Python:**
+
+
+```python
+#client.py
+from urllib.parse import urljoin
+from openai import OpenAI
+
+api_key = "FAKE_KEY"
+base_url = "http://localhost:8000"
+
+client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key)
+
+# Example query
+response = client.chat.completions.create(
+    model="my-gpt-oss",
+    messages=[
+        {"role": "user", "content": "How many r's in strawberry"}
+    ],
+    stream=True
+)
+
+# Stream
+for chunk in response:
+    # Stream reasoning content
+    if hasattr(chunk.choices[0].delta, "reasoning_content"):
+        data_reasoning = chunk.choices[0].delta.reasoning_content
+        if data_reasoning:
+            print(data_reasoning, end="", flush=True)
+    # Later, stream the final answer
+    if hasattr(chunk.choices[0].delta, "content"):
+        data_content = chunk.choices[0].delta.content
+        if data_content:
+            print(data_content, end="", flush=True)
+```
+
+
+---
+
+### Shutdown
+
+Shutdown your LLM service: 
+
+
+```bash
+%%bash
+serve shutdown -y
+```
+
+
+---
+
+## Deploy to production with Anyscale Services
+
+For production deployment, use Anyscale Services to deploy the Ray Serve app to a dedicated cluster without modifying the code. Anyscale ensures scalability, fault tolerance, and load balancing, keeping the service resilient against node failures, high traffic, and rolling updates.
+
+---
+
+### Launch the service
+
+Anyscale provides out-of-the-box images (`anyscale/ray-llm`) which come pre-loaded with Ray Serve LLM, vLLM, and all required GPU/runtime dependencies. See the [Anyscale base images](https://docs.anyscale.com/reference/base-images) for details on what each image includes.
+
+Build a minimal Dockerfile:
+```Dockerfile
+FROM anyscale/ray:2.49.1-slim-py312-cu128
+
+# C compiler for Triton’s runtime build step (vLLM V1 engine)
+# https://github.com/vllm-project/vllm/issues/2997
+RUN sudo apt-get update && \
+    sudo apt-get install -y --no-install-recommends build-essential
+
+RUN pip install vllm==0.10.1
+```
+
+Create your Anyscale Service configuration in a new `service.yaml` file and reference the Dockerfile with `containerfile`:
+
+```yaml
+# service.yaml
+name: deploy-gpt-oss
+containerfile: ./Dockerfile # Build Ray Serve LLM with vllm==0.10.1
+compute_config:
+  auto_select_worker_config: true 
+working_dir: .
+cloud:
+applications:
+  # Point to your app in your Python module
+  - import_path: serve_gpt_oss:app
+```
+
+
+Deploy your service:
+
+
+```bash
+%%bash
+anyscale service deploy -f service.yaml
+```
+
+
+---
+
+### Send requests 
+
+The `anyscale service deploy` command output shows both the endpoint and authentication token:
+```console
+(anyscale +3.9s) curl -H "Authorization: Bearer <YOUR-TOKEN>" <YOUR-ENDPOINT>
+```
+You can also retrieve both from the service page in the Anyscale Console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token.  
+
+---
+
+### Access the Serve LLM dashboard
+
+See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling LLM-specific logging. To open the Ray Serve LLM Dashboard from an Anyscale Service:
+1. In the Anyscale console, go to your **Service** or **Workspace**.
+2. Navigate to the **Metrics** tab.
+3. Expand **View in Grafana** and click **Serve LLM Dashboard**.
+
+---
+
+### Shutdown
+
+Shutdown your Anyscale Service:
+
+
+```bash
+%%bash
+anyscale service terminate -n deploy-gpt-oss
+```
+
+
+---
+
+## Enable LLM monitoring
+
+The *Serve LLM Dashboard* offers deep visibility into model performance, latency, and system behavior, including:
+
+- Token throughput (tokens/sec).
+- Latency metrics: Time To First Token (TTFT), Time Per Output Token (TPOT).
+- KV cache utilization.
+
+To enable these metrics, go to your LLM config and set `log_engine_metrics: true`:
+
+```yaml
+applications:
+- ...
+  args:
+    llm_configs:
+      - ...
+        log_engine_metrics: true
+```
+
+---
+
+## Improve concurrency
+
+Ray Serve LLM uses [vLLM](https://docs.vllm.ai/en/stable/) as its backend engine, which logs the *maximum concurrency* it can support based on your configuration.
+
+Example log for gpt-oss-20b with 1xL4:
+```console
+INFO 09-08 17:34:28 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 tokens per request: 5.22x
+```
+
+Example log for gpt-oss-120b with 2xA100-80G:
+```console
+INFO 09-09 00:32:32 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 tokens per request: 44.08x
+```
+
+To improve concurrency for gpt-oss models, see [Deploy a small-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#improve-concurrency) for small-sized models such as `gpt-oss-20b`, and [Deploy a medium-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#improve-concurrency) for medium-sized models such as `gpt-oss-120b`.
+
+---
+
+## Reasoning configuration
+
+You can control how `gpt-oss` handles reasoning in its responses. This includes accessing reasoning outputs and adjusting the reasoning effort.
+
+For more details on deploying reasoning models with Ray Serve LLM, see [Deploy a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html).
+
+---
+
+### Access reasoning output
+
+You don’t need to set a reasoning parser when deploying `gpt-oss` with Ray Serve LLM. The reasoning content is available directly in the `reasoning_content` field of the response:
+
+```python
+response = client.chat.completions.create(
+    model="my-gpt-oss",
+    messages=[
+        ...
+    ]
+)
+reasoning_content = response.choices[0].message.reasoning_content
+content = response.choices[0].message.content
+```
+
+---
+
+### Control reasoning effort
+
+`gpt-oss` supports [three reasoning levels](https://huggingface.co/openai/gpt-oss-20b#reasoning-levels): **low**, **medium**, and **high**. The default level is **medium**.
+
+You can set a level explicitly in the system prompt:
+
+```python
+response = client.chat.completions.create(
+    model="my-gpt-oss",
+    messages=[
+        {"role": "system", "content": "Reasoning: low. You are an AI travel assistant."},
+        {"role": "user", "content": "What are the three main touristic spots to see in Paris?"}
+    ]
+)
+```
+
+You can also control reasoning with the `reasoning_effort` request parameter:
+
+```python
+response = client.chat.completions.create(
+    model="my-gpt-oss",
+    messages=[
+        {"role": "user", "content": "What are the three main touristic spots to see in Paris?"}
+    ],
+    reasoning_effort="low"
+)
+```
+
+**Note:** There is no reliable way to completely disable reasoning.
+
+---
+
+## Troubleshooting
+
+**Can't download the vocab file**  
+```error
+openai_harmony.HarmonyError: error downloading or loading vocab file: failed to download or load vocab
+```
+
+This is a known bug in the `openai_harmony` library.  
+Download the *tiktoken* encoding files in advance and set the `TIKTOKEN_ENCODINGS_BASE` environment variable:
+```bash
+mkdir -p tiktoken_encodings
+wget -O tiktoken_encodings/o200k_base.tiktoken "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken"
+wget -O tiktoken_encodings/cl100k_base.tiktoken "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"
+export TIKTOKEN_ENCODINGS_BASE=${PWD}/tiktoken_encodings
+```
+
+---
+
+## Summary
+
+In this tutorial, you learned how to deploy `gpt-oss` models with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on a Ray cluster, send requests, and monitor your service.
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/client.py b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/client.py
new file mode 100644
index 000000000000..5fae2d5431ac
--- /dev/null
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/client.py
@@ -0,0 +1,30 @@
+#client_streaming.py
+from urllib.parse import urljoin
+from openai import OpenAI
+
+api_key = "FAKE_KEY"
+base_url = "http://localhost:8000"
+
+client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key)
+
+# Example: Complex query with thinking process
+response = client.chat.completions.create(
+    model="my-gpt-oss",
+    messages=[
+        {"role": "user", "content": "How many r's in strawberry"}
+    ],
+    stream=True
+)
+
+# Stream
+for chunk in response:
+    # Stream reasoning content
+    if hasattr(chunk.choices[0].delta, "reasoning_content"):
+        data_reasoning = chunk.choices[0].delta.reasoning_content
+        if data_reasoning:
+            print(data_reasoning, end="", flush=True)
+    # Later, stream the final answer
+    if hasattr(chunk.choices[0].delta, "content"):
+        data_content = chunk.choices[0].delta.content
+        if data_content:
+            print(data_content, end="", flush=True)
\ No newline at end of file
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
new file mode 100644
index 000000000000..6db016c82f8f
--- /dev/null
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
@@ -0,0 +1,488 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "6a51548b",
+   "metadata": {},
+   "source": [
+    "# Deploy gpt-oss\n",
+    "\n",
+    "[gpt-oss](https://huggingface.co/collections/openai/gpt-oss-68911959590a1634ba11c7a4) is a family of open-source models designed for general-purpose language understanding and generation. The 20&nbsp;B parameter variant (`gpt-oss-20b`) offers strong reasoning capabilities with lower latency, making it well-suited for local or specialized use cases. The larger 120&nbsp;B parameter variant (`gpt-oss-120b`) is designed for production-scale, high-reasoning workloads.\n",
+    "\n",
+    "---\n",
+    "\n",
+    "## Configure Ray Serve LLM\n",
+    "\n",
+    "Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object.\n",
+    "\n",
+    "Below are example configurations for both gpt-oss-20b and gpt-oss-120b, depending on your hardware and use case.\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### gpt-oss-20b\n",
+    "\n",
+    "To deploy a small-sized model such as gpt-oss-20b, a single GPU is sufficient:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "86070ffe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# serve_gpt_oss.py\n",
+    "from ray.serve.llm import LLMConfig, build_openai_app\n",
+    "\n",
+    "llm_config = LLMConfig(\n",
+    "    model_loading_config=dict(\n",
+    "        model_id=\"my-gpt-oss\",\n",
+    "        model_source=\"openai/gpt-oss-20b\",\n",
+    "    ),\n",
+    "    accelerator_type=\"L4\",\n",
+    "    deployment_config=dict(\n",
+    "        autoscaling_config=dict(\n",
+    "            min_replicas=1,\n",
+    "            max_replicas=2,\n",
+    "        )\n",
+    "    ),\n",
+    "    engine_kwargs=dict(\n",
+    "        max_model_len=32768\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "app = build_openai_app({\"llm_configs\": [llm_config]})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "adeb0b16",
+   "metadata": {},
+   "source": [
+    "\n",
+    "---\n",
+    "\n",
+    "### gpt-oss-120b\n",
+    "\n",
+    "To deploy a medium-sized model such as `gpt-oss-120b`, a single node with multiple GPUs is sufficient. Set `tensor_parallel_size` to distribute the model’s weights across the GPUs in your instance:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4ac648e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# serve_gpt_oss.py\n",
+    "from ray.serve.llm import LLMConfig, build_openai_app\n",
+    "\n",
+    "llm_config = LLMConfig(\n",
+    "    model_loading_config=dict(\n",
+    "        model_id=\"my-gpt-oss\",\n",
+    "        model_source=\"openai/gpt-oss-120b\",\n",
+    "    ),\n",
+    "    accelerator_type=\"A100-80G\",\n",
+    "    deployment_config=dict(\n",
+    "        autoscaling_config=dict(\n",
+    "            min_replicas=1,\n",
+    "            max_replicas=2,\n",
+    "        )\n",
+    "    ),\n",
+    "    engine_kwargs=dict(\n",
+    "        max_model_len=32768,\n",
+    "        tensor_parallel_size=2,\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "app = build_openai_app({\"llm_configs\": [llm_config]})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b17a7140",
+   "metadata": {},
+   "source": [
+    "**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example.\n",
+    "\n",
+    "---\n",
+    "\n",
+    "## Deploy locally\n",
+    "\n",
+    "**Prerequisites**\n",
+    "\n",
+    "* Access to GPU compute.\n",
+    "\n",
+    "**Dependencies:**\n",
+    "gpt-oss integration is available starting from `ray>=2.49.0` and `vllm>=0.10.1`\n",
+    "```bash\n",
+    "pip install \"ray[serve,llm]>=2.49.0\"\n",
+    "pip install \"vllm>=0.10.1\"\n",
+    "```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### Launch\n",
+    "\n",
+    "Follow the instructions in [Configure Ray Serve LLM](#configure-ray-serve-llm) according to the model size you choose, and define your app in a Python module `serve_gpt_oss.py`.\n",
+    "\n",
+    "In a terminal, run:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dbdb0921",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "serve run serve_gpt_oss:app --non-blocking"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "df944967",
+   "metadata": {},
+   "source": [
+    "Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded.\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### Send requests\n",
+    "\n",
+    "Your endpoint is available locally at `http://localhost:8000`. You can use a placeholder authentication token for the OpenAI client, for example `\"FAKE_KEY\"`.\n",
+    "\n",
+    "**Example curl:**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5309437",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "curl -X POST http://localhost:8000/v1/chat/completions \\\n",
+    "  -H \"Authorization: Bearer FAKE_KEY\" \\\n",
+    "  -H \"Content-Type: application/json\" \\\n",
+    "  -d '{ \"model\": \"my-gpt-oss\", \"messages\": [{\"role\": \"user\", \"content\": \"How many Rs in strawberry ?\"}] }'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d623a30f",
+   "metadata": {},
+   "source": [
+    "**Example Python:**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75bedc22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#client.py\n",
+    "from urllib.parse import urljoin\n",
+    "from openai import OpenAI\n",
+    "\n",
+    "api_key = \"FAKE_KEY\"\n",
+    "base_url = \"http://localhost:8000\"\n",
+    "\n",
+    "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n",
+    "\n",
+    "# Example query\n",
+    "response = client.chat.completions.create(\n",
+    "    model=\"my-gpt-oss\",\n",
+    "    messages=[\n",
+    "        {\"role\": \"user\", \"content\": \"How many r's in strawberry\"}\n",
+    "    ],\n",
+    "    stream=True\n",
+    ")\n",
+    "\n",
+    "# Stream\n",
+    "for chunk in response:\n",
+    "    # Stream reasoning content\n",
+    "    if hasattr(chunk.choices[0].delta, \"reasoning_content\"):\n",
+    "        data_reasoning = chunk.choices[0].delta.reasoning_content\n",
+    "        if data_reasoning:\n",
+    "            print(data_reasoning, end=\"\", flush=True)\n",
+    "    # Later, stream the final answer\n",
+    "    if hasattr(chunk.choices[0].delta, \"content\"):\n",
+    "        data_content = chunk.choices[0].delta.content\n",
+    "        if data_content:\n",
+    "            print(data_content, end=\"\", flush=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b095ebf3",
+   "metadata": {},
+   "source": [
+    "\n",
+    "---\n",
+    "\n",
+    "### Shutdown\n",
+    "\n",
+    "Shutdown your LLM service: "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4fd3dacf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "serve shutdown -y"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fb81fa41",
+   "metadata": {},
+   "source": [
+    "\n",
+    "---\n",
+    "\n",
+    "## Deploy to production with Anyscale Services\n",
+    "\n",
+    "For production deployment, use Anyscale Services to deploy the Ray Serve app to a dedicated cluster without modifying the code. Anyscale ensures scalability, fault tolerance, and load balancing, keeping the service resilient against node failures, high traffic, and rolling updates.\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### Launch the service\n",
+    "\n",
+    "Anyscale provides out-of-the-box images (`anyscale/ray-llm`) which come pre-loaded with Ray Serve LLM, vLLM, and all required GPU/runtime dependencies. See the [Anyscale base images](https://docs.anyscale.com/reference/base-images) for details on what each image includes.\n",
+    "\n",
+    "Build a minimal Dockerfile:\n",
+    "```Dockerfile\n",
+    "FROM anyscale/ray:2.49.1-slim-py312-cu128\n",
+    "\n",
+    "# C compiler for Triton’s runtime build step (vLLM V1 engine)\n",
+    "# https://github.com/vllm-project/vllm/issues/2997\n",
+    "RUN sudo apt-get update && \\\n",
+    "    sudo apt-get install -y --no-install-recommends build-essential\n",
+    "\n",
+    "RUN pip install vllm==0.10.1\n",
+    "```\n",
+    "\n",
+    "Create your Anyscale Service configuration in a new `service.yaml` file and reference the Dockerfile with `containerfile`:\n",
+    "\n",
+    "```yaml\n",
+    "# service.yaml\n",
+    "name: deploy-gpt-oss\n",
+    "containerfile: ./Dockerfile # Build Ray Serve LLM with vllm==0.10.1\n",
+    "compute_config:\n",
+    "  auto_select_worker_config: true \n",
+    "working_dir: .\n",
+    "cloud:\n",
+    "applications:\n",
+    "  # Point to your app in your Python module\n",
+    "  - import_path: serve_gpt_oss:app\n",
+    "```\n",
+    "\n",
+    "\n",
+    "Deploy your service:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1fa0556b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "anyscale service deploy -f service.yaml"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7e6de36c",
+   "metadata": {},
+   "source": [
+    "\n",
+    "---\n",
+    "\n",
+    "### Send requests \n",
+    "\n",
+    "The `anyscale service deploy` command output shows both the endpoint and authentication token:\n",
+    "```console\n",
+    "(anyscale +3.9s) curl -H \"Authorization: Bearer <YOUR-TOKEN>\" <YOUR-ENDPOINT>\n",
+    "```\n",
+    "You can also retrieve both from the service page in the Anyscale Console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token.  \n",
+    "\n",
+    "---\n",
+    "\n",
+    "### Access the Serve LLM dashboard\n",
+    "\n",
+    "See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling LLM-specific logging. To open the Ray Serve LLM Dashboard from an Anyscale Service:\n",
+    "1. In the Anyscale console, go to your **Service** or **Workspace**.\n",
+    "2. Navigate to the **Metrics** tab.\n",
+    "3. Expand **View in Grafana** and click **Serve LLM Dashboard**.\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### Shutdown\n",
+    "\n",
+    "Shutdown your Anyscale Service:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "474b2764",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%bash\n",
+    "anyscale service terminate -n deploy-gpt-oss"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "49f67c39",
+   "metadata": {},
+   "source": [
+    "\n",
+    "---\n",
+    "\n",
+    "## Enable LLM monitoring\n",
+    "\n",
+    "The *Serve LLM Dashboard* offers deep visibility into model performance, latency, and system behavior, including:\n",
+    "\n",
+    "- Token throughput (tokens/sec).\n",
+    "- Latency metrics: Time To First Token (TTFT), Time Per Output Token (TPOT).\n",
+    "- KV cache utilization.\n",
+    "\n",
+    "To enable these metrics, go to your LLM config and set `log_engine_metrics: true`:\n",
+    "\n",
+    "```yaml\n",
+    "applications:\n",
+    "- ...\n",
+    "  args:\n",
+    "    llm_configs:\n",
+    "      - ...\n",
+    "        log_engine_metrics: true\n",
+    "```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "## Improve concurrency\n",
+    "\n",
+    "Ray Serve LLM uses [vLLM](https://docs.vllm.ai/en/stable/) as its backend engine, which logs the *maximum concurrency* it can support based on your configuration.\n",
+    "\n",
+    "Example log for gpt-oss-20b with 1xL4:\n",
+    "```console\n",
+    "INFO 09-08 17:34:28 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 tokens per request: 5.22x\n",
+    "```\n",
+    "\n",
+    "Example log for gpt-oss-120b with 2xA100-80G:\n",
+    "```console\n",
+    "INFO 09-09 00:32:32 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 tokens per request: 44.08x\n",
+    "```\n",
+    "\n",
+    "To improve concurrency for gpt-oss models, see [Deploy a small-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#improve-concurrency) for small-sized models such as `gpt-oss-20b`, and [Deploy a medium-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#improve-concurrency) for medium-sized models such as `gpt-oss-120b`.\n",
+    "\n",
+    "---\n",
+    "\n",
+    "## Reasoning configuration\n",
+    "\n",
+    "You can control how `gpt-oss` handles reasoning in its responses. This includes accessing reasoning outputs and adjusting the reasoning effort.\n",
+    "\n",
+    "For more details on deploying reasoning models with Ray Serve LLM, see [Deploy a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html).\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### Access reasoning output\n",
+    "\n",
+    "You don’t need to set a reasoning parser when deploying `gpt-oss` with Ray Serve LLM. The reasoning content is available directly in the `reasoning_content` field of the response:\n",
+    "\n",
+    "```python\n",
+    "response = client.chat.completions.create(\n",
+    "    model=\"my-gpt-oss\",\n",
+    "    messages=[\n",
+    "        ...\n",
+    "    ]\n",
+    ")\n",
+    "reasoning_content = response.choices[0].message.reasoning_content\n",
+    "content = response.choices[0].message.content\n",
+    "```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### Control reasoning effort\n",
+    "\n",
+    "`gpt-oss` supports [three reasoning levels](https://huggingface.co/openai/gpt-oss-20b#reasoning-levels): **low**, **medium**, and **high**. The default level is **medium**.\n",
+    "\n",
+    "You can set a level explicitly in the system prompt:\n",
+    "\n",
+    "```python\n",
+    "response = client.chat.completions.create(\n",
+    "    model=\"my-gpt-oss\",\n",
+    "    messages=[\n",
+    "        {\"role\": \"system\", \"content\": \"Reasoning: low. You are an AI travel assistant.\"},\n",
+    "        {\"role\": \"user\", \"content\": \"What are the three main touristic spots to see in Paris?\"}\n",
+    "    ]\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "You can also control reasoning with the `reasoning_effort` request parameter:\n",
+    "\n",
+    "```python\n",
+    "response = client.chat.completions.create(\n",
+    "    model=\"my-gpt-oss\",\n",
+    "    messages=[\n",
+    "        {\"role\": \"user\", \"content\": \"What are the three main touristic spots to see in Paris?\"}\n",
+    "    ],\n",
+    "    reasoning_effort=\"low\"\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "**Note:** There is no reliable way to completely disable reasoning.\n",
+    "\n",
+    "---\n",
+    "\n",
+    "## Troubleshooting\n",
+    "\n",
+    "**Can't download the vocab file**  \n",
+    "```error\n",
+    "openai_harmony.HarmonyError: error downloading or loading vocab file: failed to download or load vocab\n",
+    "```\n",
+    "\n",
+    "This is a known bug in the `openai_harmony` library.  \n",
+    "Download the *tiktoken* encoding files in advance and set the `TIKTOKEN_ENCODINGS_BASE` environment variable:\n",
+    "```bash\n",
+    "mkdir -p tiktoken_encodings\n",
+    "wget -O tiktoken_encodings/o200k_base.tiktoken \"https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken\"\n",
+    "wget -O tiktoken_encodings/cl100k_base.tiktoken \"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken\"\n",
+    "export TIKTOKEN_ENCODINGS_BASE=${PWD}/tiktoken_encodings\n",
+    "```\n",
+    "\n",
+    "---\n",
+    "\n",
+    "## Summary\n",
+    "\n",
+    "In this tutorial, you learned how to deploy `gpt-oss` models with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on a Ray cluster, send requests, and monitor your service."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "repo_ray_docs",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
new file mode 100644
index 000000000000..c0ff078c9ec0
--- /dev/null
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
@@ -0,0 +1,46 @@
+# serve_gpt_oss.py
+from ray.serve.llm import LLMConfig, build_openai_app
+
+GPT_OSS_SIZE = "20b"  # or "120b"
+
+if GPT_OSS_SIZE == "20b":
+    llm_config = LLMConfig(
+        model_loading_config=dict(
+            model_id="my-gpt-oss",
+            model_source="openai/gpt-oss-20b",
+        ),
+        accelerator_type="L4",
+        deployment_config=dict(
+            autoscaling_config=dict(
+                min_replicas=1,
+                max_replicas=2,
+            )
+        ),
+        engine_kwargs=dict(
+            max_model_len=32768,
+        ),
+    )
+
+elif GPT_OSS_SIZE == "120b":
+    llm_config = LLMConfig(
+        model_loading_config=dict(
+            model_id="my-gpt-oss",
+            model_source="openai/gpt-oss-120b",
+        ),
+        accelerator_type="A100-80G",
+        deployment_config=dict(
+            autoscaling_config=dict(
+                min_replicas=1,
+                max_replicas=2,
+            )
+        ),
+        engine_kwargs=dict(
+            max_model_len=65536,
+            tensor_parallel_size=2,
+        ),
+    )
+
+else:
+    raise ValueError("GPT_OSS_SIZE must be either '20b' or '120b'")
+
+app = build_openai_app({"llm_configs": [llm_config]})
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/service.yaml b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/service.yaml
new file mode 100644
index 000000000000..5e6be0427f11
--- /dev/null
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/service.yaml
@@ -0,0 +1,10 @@
+# service.yaml
+name: deploy-gpt-oss
+containerfile: ./Dockerfile # Build Ray Serve LLM with vllm==0.10.1
+compute_config:
+  auto_select_worker_config: true 
+working_dir: .
+cloud:
+applications:
+  # Point to your app in your Python module
+  - import_path: serve_gpt_oss:app
\ No newline at end of file

From 251b7852475b0d7b3ee9f4953584a11b71177292 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Wed, 10 Sep 2025 00:59:00 -0700
Subject: [PATCH 02/19] add reference link

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../serve/tutorials/deployment-serve-llm/README.ipynb      | 7 ++++++-
 doc/source/serve/tutorials/deployment-serve-llm/README.md  | 5 +++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb
index 9dfa5c6f1980..f3b04b0d9587 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb
@@ -39,7 +39,12 @@
     "---\n",
     "\n",
     "**[Deploy a hybrid reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.html)**  \n",
-    "Deploy models that can switch between reasoning and non-reasoning modes for flexible usage, such as Qwen-3."
+    "Deploy models that can switch between reasoning and non-reasoning modes for flexible usage, such as Qwen-3.\n",
+    "\n",
+    "---\n",
+    "\n",
+    "**[Deploy gpt-oss](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/gpt-oss/README.html)**  \n",
+    "Deploy gpt-oss reasoning models, including `gpt-oss-20b` for lower latency use cases and `gpt-oss-120b` for high-reasoning, production-scale workloads."
    ]
   }
  ],
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/README.md
index 666a99af10c5..d3bee1c38855 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/README.md
@@ -39,3 +39,8 @@ Deploy models with reasoning capabilities designed for long-context tasks, codin
 
 **[Deploy a hybrid reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.html)**  
 Deploy models that can switch between reasoning and non-reasoning modes for flexible usage, such as Qwen-3.
+
+---
+
+**[Deploy gpt-oss](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/gpt-oss/README.html)**  
+Deploy gpt-oss reasoning models, including `gpt-oss-20b` for lower latency use cases and `gpt-oss-120b` for high-reasoning, production-scale workloads.

From 0c4ca8587094149fa7426affa4125bff17d4222a Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Wed, 10 Sep 2025 00:59:08 -0700
Subject: [PATCH 03/19] add test

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh b/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh
index 79e90ee0905e..ab97e1c41d1e 100755
--- a/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh
+++ b/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh
@@ -12,7 +12,8 @@ for nb in \
   "large-size-llm/notebook" \
   "vision-llm/notebook" \
   "reasoning-llm/notebook" \
-  "hybrid-reasoning-llm/notebook"
+  "hybrid-reasoning-llm/notebook" \
+  "gpt-oss/notebook" \
 do
   python ci/nb2py.py "${nb}.ipynb" "${nb}.py" --ignore-cmds
   python "${nb}.py"

From 618acbe3d1a695d17dbfa3d45cf5e7aaa639b426 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Wed, 10 Sep 2025 01:20:19 -0700
Subject: [PATCH 04/19] black lint + sphinx fix

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../serve/tutorials/deployment-serve-llm/ci/tests.sh   |  2 +-
 .../tutorials/deployment-serve-llm/gpt-oss/README.md   |  2 +-
 .../tutorials/deployment-serve-llm/gpt-oss/client.py   | 10 ++++------
 .../deployment-serve-llm/gpt-oss/notebook.ipynb        |  2 +-
 4 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh b/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh
index ab97e1c41d1e..6f005cc384a7 100755
--- a/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh
+++ b/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh
@@ -13,7 +13,7 @@ for nb in \
   "vision-llm/notebook" \
   "reasoning-llm/notebook" \
   "hybrid-reasoning-llm/notebook" \
-  "gpt-oss/notebook" \
+  "gpt-oss/notebook" 
 do
   python ci/nb2py.py "${nb}.ipynb" "${nb}.py" --ignore-cmds
   python "${nb}.py"
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
index d6792b3feac5..8696569c89cf 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
@@ -366,7 +366,7 @@ response = client.chat.completions.create(
 ## Troubleshooting
 
 **Can't download the vocab file**  
-```error
+```console
 openai_harmony.HarmonyError: error downloading or loading vocab file: failed to download or load vocab
 ```
 
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/client.py b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/client.py
index 5fae2d5431ac..674b0f101b2c 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/client.py
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/client.py
@@ -1,4 +1,4 @@
-#client_streaming.py
+# client_streaming.py
 from urllib.parse import urljoin
 from openai import OpenAI
 
@@ -10,10 +10,8 @@
 # Example: Complex query with thinking process
 response = client.chat.completions.create(
     model="my-gpt-oss",
-    messages=[
-        {"role": "user", "content": "How many r's in strawberry"}
-    ],
-    stream=True
+    messages=[{"role": "user", "content": "How many r's in strawberry"}],
+    stream=True,
 )
 
 # Stream
@@ -27,4 +25,4 @@
     if hasattr(chunk.choices[0].delta, "content"):
         data_content = chunk.choices[0].delta.content
         if data_content:
-            print(data_content, end="", flush=True)
\ No newline at end of file
+            print(data_content, end="", flush=True)
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
index 6db016c82f8f..e0ede97f076e 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
@@ -451,7 +451,7 @@
     "## Troubleshooting\n",
     "\n",
     "**Can't download the vocab file**  \n",
-    "```error\n",
+    "```console\n",
     "openai_harmony.HarmonyError: error downloading or loading vocab file: failed to download or load vocab\n",
     "```\n",
     "\n",

From d8a8945482cfcccac1ab2a46db912afa393c6cbb Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Mon, 22 Sep 2025 11:47:24 -0700
Subject: [PATCH 05/19] use L40S insead of A100 + fix model len oversight

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../deployment-serve-llm/gpt-oss/README.md        | 15 ++++++++++++---
 .../deployment-serve-llm/gpt-oss/notebook.ipynb   | 15 ++++++++++++---
 .../deployment-serve-llm/gpt-oss/serve_gpt_oss.py |  4 ++--
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
index 8696569c89cf..4b53d968fc82 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
@@ -68,7 +68,7 @@ llm_config = LLMConfig(
         model_id="my-gpt-oss",
         model_source="openai/gpt-oss-120b",
     ),
-    accelerator_type="A100-80G",
+    accelerator_type="L40S",
     deployment_config=dict(
         autoscaling_config=dict(
             min_replicas=1,
@@ -297,9 +297,9 @@ Example log for gpt-oss-20b with 1xL4:
 INFO 09-08 17:34:28 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 tokens per request: 5.22x
 ```
 
-Example log for gpt-oss-120b with 2xA100-80G:
+Example log for gpt-oss-120b with 2xL40S:
 ```console
-INFO 09-09 00:32:32 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 tokens per request: 44.08x
+INFO 09-09 00:32:32 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 tokens per request: 6.18x
 ```
 
 To improve concurrency for gpt-oss models, see [Deploy a small-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#improve-concurrency) for small-sized models such as `gpt-oss-20b`, and [Deploy a medium-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#improve-concurrency) for medium-sized models such as `gpt-oss-120b`.
@@ -379,6 +379,15 @@ wget -O tiktoken_encodings/cl100k_base.tiktoken "https://openaipublic.blob.core.
 export TIKTOKEN_ENCODINGS_BASE=${PWD}/tiktoken_encodings
 ```
 
+**`gpt-oss` architecture not recognized**  
+```console
+Value error, The checkpoint you are trying to load has model type `gpt_oss` but Transformers does not recognize this architecture. This could be because of an issue with the checkpoint, or because your version of Transformers is out of date.
+```
+Older vLLM and transformers versions don't register `gpt_oss`, raising an error when vLLM hands off to Transformers. Upgrade **vLLM ≥ 0.10.1** and let your package resolver such as `pip` handle the other dependencies.
+```bash
+pip install -U "vllm>=0.10.1"
+```
+
 ---
 
 ## Summary
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
index e0ede97f076e..18dd24366a44 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
@@ -82,7 +82,7 @@
     "        model_id=\"my-gpt-oss\",\n",
     "        model_source=\"openai/gpt-oss-120b\",\n",
     "    ),\n",
-    "    accelerator_type=\"A100-80G\",\n",
+    "    accelerator_type=\"L40S\",\n",
     "    deployment_config=dict(\n",
     "        autoscaling_config=dict(\n",
     "            min_replicas=1,\n",
@@ -382,9 +382,9 @@
     "INFO 09-08 17:34:28 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 tokens per request: 5.22x\n",
     "```\n",
     "\n",
-    "Example log for gpt-oss-120b with 2xA100-80G:\n",
+    "Example log for gpt-oss-120b with 2xL40S:\n",
     "```console\n",
-    "INFO 09-09 00:32:32 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 tokens per request: 44.08x\n",
+    "INFO 09-09 00:32:32 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 tokens per request: 6.18x\n",
     "```\n",
     "\n",
     "To improve concurrency for gpt-oss models, see [Deploy a small-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#improve-concurrency) for small-sized models such as `gpt-oss-20b`, and [Deploy a medium-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#improve-concurrency) for medium-sized models such as `gpt-oss-120b`.\n",
@@ -464,6 +464,15 @@
     "export TIKTOKEN_ENCODINGS_BASE=${PWD}/tiktoken_encodings\n",
     "```\n",
     "\n",
+    "**`gpt-oss` architecture not recognized**  \n",
+    "```console\n",
+    "Value error, The checkpoint you are trying to load has model type `gpt_oss` but Transformers does not recognize this architecture. This could be because of an issue with the checkpoint, or because your version of Transformers is out of date.\n",
+    "```\n",
+    "Older vLLM and transformers versions don't register `gpt_oss`, raising an error when vLLM hands off to Transformers. Upgrade **vLLM ≥ 0.10.1** and let your package resolver such as `pip` handle the other dependencies.\n",
+    "```bash\n",
+    "pip install -U \"vllm>=0.10.1\"\n",
+    "```\n",
+    "\n",
     "---\n",
     "\n",
     "## Summary\n",
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
index c0ff078c9ec0..b5596d150ced 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
@@ -27,7 +27,7 @@
             model_id="my-gpt-oss",
             model_source="openai/gpt-oss-120b",
         ),
-        accelerator_type="A100-80G",
+        accelerator_type="L40S",
         deployment_config=dict(
             autoscaling_config=dict(
                 min_replicas=1,
@@ -35,7 +35,7 @@
             )
         ),
         engine_kwargs=dict(
-            max_model_len=65536,
+            max_model_len=32768,
             tensor_parallel_size=2,
         ),
     )

From d14b0178d122b8afff7a0d25da0dae2e03103389 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Mon, 22 Sep 2025 14:05:48 -0700
Subject: [PATCH 06/19] kunling review

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../deployment-serve-llm/gpt-oss/README.md    | 37 ++++++++++---------
 .../gpt-oss/notebook.ipynb                    | 37 ++++++++++---------
 .../gpt-oss/serve_gpt_oss.py                  |  2 +-
 3 files changed, 41 insertions(+), 35 deletions(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
index 4b53d968fc82..4bba5dd68262 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
@@ -68,7 +68,7 @@ llm_config = LLMConfig(
         model_id="my-gpt-oss",
         model_source="openai/gpt-oss-120b",
     ),
-    accelerator_type="L40S",
+    accelerator_type="L40S", # Or "A100-40G"
     deployment_config=dict(
         autoscaling_config=dict(
             min_replicas=1,
@@ -198,7 +198,7 @@ Anyscale provides out-of-the-box images (`anyscale/ray-llm`) which come pre-load
 
 Build a minimal Dockerfile:
 ```Dockerfile
-FROM anyscale/ray:2.49.1-slim-py312-cu128
+FROM anyscale/ray:2.49.0-slim-py312-cu128
 
 # C compiler for Triton’s runtime build step (vLLM V1 engine)
 # https://github.com/vllm-project/vllm/issues/2997
@@ -304,19 +304,21 @@ INFO 09-09 00:32:32 [kv_cache_utils.py:1017] Maximum concurrency for 32,768 toke
 
 To improve concurrency for gpt-oss models, see [Deploy a small-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#improve-concurrency) for small-sized models such as `gpt-oss-20b`, and [Deploy a medium-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#improve-concurrency) for medium-sized models such as `gpt-oss-120b`.
 
+**Note:** Some example guides recommend using quantization to boost concurrency. `gpt-oss` weights are already 4-bit by default, so further quantization typically isn’t applicable.  
+
+For broader guidance, also see [Choose a GPU for LLM serving](https://docs.anyscale.com/llm/serving/gpu-guidance) and [Optimize performance for Ray Serve LLM](https://docs.anyscale.com/llm/serving/performance-optimization).
+
 ---
 
 ## Reasoning configuration
 
-You can control how `gpt-oss` handles reasoning in its responses. This includes accessing reasoning outputs and adjusting the reasoning effort.
-
-For more details on deploying reasoning models with Ray Serve LLM, see [Deploy a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html).
+You don’t need a custom reasoning parser when deploying `gpt-oss` with Ray Serve LLM, you can access the reasoning content in the model's response directly. You can also control the reasoning effort of the model in the request.
 
 ---
 
 ### Access reasoning output
 
-You don’t need to set a reasoning parser when deploying `gpt-oss` with Ray Serve LLM. The reasoning content is available directly in the `reasoning_content` field of the response:
+The reasoning content is available directly in the `reasoning_content` field of the response:
 
 ```python
 response = client.chat.completions.create(
@@ -335,27 +337,25 @@ content = response.choices[0].message.content
 
 `gpt-oss` supports [three reasoning levels](https://huggingface.co/openai/gpt-oss-20b#reasoning-levels): **low**, **medium**, and **high**. The default level is **medium**.
 
-You can set a level explicitly in the system prompt:
-
+You can control reasoning with the `reasoning_effort` request parameter:  
 ```python
 response = client.chat.completions.create(
     model="my-gpt-oss",
     messages=[
-        {"role": "system", "content": "Reasoning: low. You are an AI travel assistant."},
         {"role": "user", "content": "What are the three main touristic spots to see in Paris?"}
-    ]
+    ],
+    reasoning_effort="low" # Or "medium", "high"
 )
 ```
 
-You can also control reasoning with the `reasoning_effort` request parameter:
-
+You can also set a level explicitly in the system prompt:  
 ```python
 response = client.chat.completions.create(
     model="my-gpt-oss",
     messages=[
+        {"role": "system", "content": "Reasoning: low. You are an AI travel assistant."},
         {"role": "user", "content": "What are the three main touristic spots to see in Paris?"}
-    ],
-    reasoning_effort="low"
+    ]
 )
 ```
 
@@ -370,8 +370,11 @@ response = client.chat.completions.create(
 openai_harmony.HarmonyError: error downloading or loading vocab file: failed to download or load vocab
 ```
 
-This is a known bug in the `openai_harmony` library.  
-Download the *tiktoken* encoding files in advance and set the `TIKTOKEN_ENCODINGS_BASE` environment variable:
+The `openai_harmony` library needs the *tiktoken* encoding files and tries to fetch them from OpenAI's public host. Common cause includes:
+- Corporate firewall or proxy blocks `openaipublic.blob.core.windows.net`; you may need to allowlist this domain.
+- Race conditions when multiple processes try to download to the same cache. This can happen when [deploying multiple models at the same time](https://github.com/openai/harmony/pull/41).
+
+You can also directly download the *tiktoken* encoding files in advance and set the `TIKTOKEN_ENCODINGS_BASE` environment variable:
 ```bash
 mkdir -p tiktoken_encodings
 wget -O tiktoken_encodings/o200k_base.tiktoken "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken"
@@ -383,7 +386,7 @@ export TIKTOKEN_ENCODINGS_BASE=${PWD}/tiktoken_encodings
 ```console
 Value error, The checkpoint you are trying to load has model type `gpt_oss` but Transformers does not recognize this architecture. This could be because of an issue with the checkpoint, or because your version of Transformers is out of date.
 ```
-Older vLLM and transformers versions don't register `gpt_oss`, raising an error when vLLM hands off to Transformers. Upgrade **vLLM ≥ 0.10.1** and let your package resolver such as `pip` handle the other dependencies.
+Older vLLM and transformers versions don't register `gpt_oss`, raising an error when vLLM hands off to transformers. Upgrade **vLLM ≥ 0.10.1** and let your package resolver such as `pip` handle the other dependencies.
 ```bash
 pip install -U "vllm>=0.10.1"
 ```
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
index 18dd24366a44..74e33ea24b7e 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
@@ -82,7 +82,7 @@
     "        model_id=\"my-gpt-oss\",\n",
     "        model_source=\"openai/gpt-oss-120b\",\n",
     "    ),\n",
-    "    accelerator_type=\"L40S\",\n",
+    "    accelerator_type=\"L40S\", # Or \"A100-40G\"\n",
     "    deployment_config=dict(\n",
     "        autoscaling_config=dict(\n",
     "            min_replicas=1,\n",
@@ -261,7 +261,7 @@
     "\n",
     "Build a minimal Dockerfile:\n",
     "```Dockerfile\n",
-    "FROM anyscale/ray:2.49.1-slim-py312-cu128\n",
+    "FROM anyscale/ray:2.49.0-slim-py312-cu128\n",
     "\n",
     "# C compiler for Triton’s runtime build step (vLLM V1 engine)\n",
     "# https://github.com/vllm-project/vllm/issues/2997\n",
@@ -389,19 +389,21 @@
     "\n",
     "To improve concurrency for gpt-oss models, see [Deploy a small-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#improve-concurrency) for small-sized models such as `gpt-oss-20b`, and [Deploy a medium-sized LLM: Improve concurrency](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#improve-concurrency) for medium-sized models such as `gpt-oss-120b`.\n",
     "\n",
+    "**Note:** Some example guides recommend using quantization to boost concurrency. `gpt-oss` weights are already 4-bit by default, so further quantization typically isn’t applicable.  \n",
+    "\n",
+    "For broader guidance, also see [Choose a GPU for LLM serving](https://docs.anyscale.com/llm/serving/gpu-guidance) and [Optimize performance for Ray Serve LLM](https://docs.anyscale.com/llm/serving/performance-optimization).\n",
+    "\n",
     "---\n",
     "\n",
     "## Reasoning configuration\n",
     "\n",
-    "You can control how `gpt-oss` handles reasoning in its responses. This includes accessing reasoning outputs and adjusting the reasoning effort.\n",
-    "\n",
-    "For more details on deploying reasoning models with Ray Serve LLM, see [Deploy a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html).\n",
+    "You don’t need a custom reasoning parser when deploying `gpt-oss` with Ray Serve LLM, you can access the reasoning content in the model's response directly. You can also control the reasoning effort of the model in the request.\n",
     "\n",
     "---\n",
     "\n",
     "### Access reasoning output\n",
     "\n",
-    "You don’t need to set a reasoning parser when deploying `gpt-oss` with Ray Serve LLM. The reasoning content is available directly in the `reasoning_content` field of the response:\n",
+    "The reasoning content is available directly in the `reasoning_content` field of the response:\n",
     "\n",
     "```python\n",
     "response = client.chat.completions.create(\n",
@@ -420,27 +422,25 @@
     "\n",
     "`gpt-oss` supports [three reasoning levels](https://huggingface.co/openai/gpt-oss-20b#reasoning-levels): **low**, **medium**, and **high**. The default level is **medium**.\n",
     "\n",
-    "You can set a level explicitly in the system prompt:\n",
-    "\n",
+    "You can control reasoning with the `reasoning_effort` request parameter:  \n",
     "```python\n",
     "response = client.chat.completions.create(\n",
     "    model=\"my-gpt-oss\",\n",
     "    messages=[\n",
-    "        {\"role\": \"system\", \"content\": \"Reasoning: low. You are an AI travel assistant.\"},\n",
     "        {\"role\": \"user\", \"content\": \"What are the three main touristic spots to see in Paris?\"}\n",
-    "    ]\n",
+    "    ],\n",
+    "    reasoning_effort=\"low\" # Or \"medium\", \"high\"\n",
     ")\n",
     "```\n",
     "\n",
-    "You can also control reasoning with the `reasoning_effort` request parameter:\n",
-    "\n",
+    "You can also set a level explicitly in the system prompt:  \n",
     "```python\n",
     "response = client.chat.completions.create(\n",
     "    model=\"my-gpt-oss\",\n",
     "    messages=[\n",
+    "        {\"role\": \"system\", \"content\": \"Reasoning: low. You are an AI travel assistant.\"},\n",
     "        {\"role\": \"user\", \"content\": \"What are the three main touristic spots to see in Paris?\"}\n",
-    "    ],\n",
-    "    reasoning_effort=\"low\"\n",
+    "    ]\n",
     ")\n",
     "```\n",
     "\n",
@@ -455,8 +455,11 @@
     "openai_harmony.HarmonyError: error downloading or loading vocab file: failed to download or load vocab\n",
     "```\n",
     "\n",
-    "This is a known bug in the `openai_harmony` library.  \n",
-    "Download the *tiktoken* encoding files in advance and set the `TIKTOKEN_ENCODINGS_BASE` environment variable:\n",
+    "The `openai_harmony` library needs the *tiktoken* encoding files and tries to fetch them from OpenAI's public host. Common cause includes:\n",
+    "- Corporate firewall or proxy blocks `openaipublic.blob.core.windows.net`; you may need to allowlist this domain.\n",
+    "- Race conditions when multiple processes try to download to the same cache. This can happen when [deploying multiple models at the same time](https://github.com/openai/harmony/pull/41).\n",
+    "\n",
+    "You can also directly download the *tiktoken* encoding files in advance and set the `TIKTOKEN_ENCODINGS_BASE` environment variable:\n",
     "```bash\n",
     "mkdir -p tiktoken_encodings\n",
     "wget -O tiktoken_encodings/o200k_base.tiktoken \"https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken\"\n",
@@ -468,7 +471,7 @@
     "```console\n",
     "Value error, The checkpoint you are trying to load has model type `gpt_oss` but Transformers does not recognize this architecture. This could be because of an issue with the checkpoint, or because your version of Transformers is out of date.\n",
     "```\n",
-    "Older vLLM and transformers versions don't register `gpt_oss`, raising an error when vLLM hands off to Transformers. Upgrade **vLLM ≥ 0.10.1** and let your package resolver such as `pip` handle the other dependencies.\n",
+    "Older vLLM and transformers versions don't register `gpt_oss`, raising an error when vLLM hands off to transformers. Upgrade **vLLM ≥ 0.10.1** and let your package resolver such as `pip` handle the other dependencies.\n",
     "```bash\n",
     "pip install -U \"vllm>=0.10.1\"\n",
     "```\n",
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
index b5596d150ced..ce6a3faa1820 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
@@ -27,7 +27,7 @@
             model_id="my-gpt-oss",
             model_source="openai/gpt-oss-120b",
         ),
-        accelerator_type="L40S",
+        accelerator_type="L40S", # Or "A100-40G"
         deployment_config=dict(
             autoscaling_config=dict(
                 min_replicas=1,

From 48bcf3bbcf20eb8ff8b147b1511ece1f5c7739f8 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Mon, 22 Sep 2025 14:13:08 -0700
Subject: [PATCH 07/19] add more details to openai harmony bug

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../serve/tutorials/deployment-serve-llm/gpt-oss/README.md     | 3 ++-
 .../tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb      | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
index 4bba5dd68262..74fd97809c4f 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
@@ -371,7 +371,8 @@ openai_harmony.HarmonyError: error downloading or loading vocab file: failed to
 ```
 
 The `openai_harmony` library needs the *tiktoken* encoding files and tries to fetch them from OpenAI's public host. Common cause includes:
-- Corporate firewall or proxy blocks `openaipublic.blob.core.windows.net`; you may need to allowlist this domain.
+- Corporate firewall or proxy blocks `openaipublic.blob.core.windows.net`; you may need to whitelist this domain.
+- Intermittent network issues.
 - Race conditions when multiple processes try to download to the same cache. This can happen when [deploying multiple models at the same time](https://github.com/openai/harmony/pull/41).
 
 You can also directly download the *tiktoken* encoding files in advance and set the `TIKTOKEN_ENCODINGS_BASE` environment variable:
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
index 74e33ea24b7e..03166f48daea 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
@@ -456,7 +456,8 @@
     "```\n",
     "\n",
     "The `openai_harmony` library needs the *tiktoken* encoding files and tries to fetch them from OpenAI's public host. Common cause includes:\n",
-    "- Corporate firewall or proxy blocks `openaipublic.blob.core.windows.net`; you may need to allowlist this domain.\n",
+    "- Corporate firewall or proxy blocks `openaipublic.blob.core.windows.net`; you may need to whitelist this domain.\n",
+    "- Intermittent network issues.\n",
     "- Race conditions when multiple processes try to download to the same cache. This can happen when [deploying multiple models at the same time](https://github.com/openai/harmony/pull/41).\n",
     "\n",
     "You can also directly download the *tiktoken* encoding files in advance and set the `TIKTOKEN_ENCODINGS_BASE` environment variable:\n",

From 5b4257ff81a7e132a867a6fb118a24ff695fcaf0 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Thu, 2 Oct 2025 16:56:57 -0700
Subject: [PATCH 08/19] [cursor] style/editorial rules + fix vllm to 0.10.1

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../deployment-serve-llm/gpt-oss/README.md    | 27 ++++++++++++-------
 .../gpt-oss/notebook.ipynb                    | 27 ++++++++++++-------
 2 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
index 74fd97809c4f..34d6fded0eb9 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
@@ -10,7 +10,9 @@ jupyter nbconvert "$notebook.ipynb" --to markdown --output "README.md"
 
 # Deploy gpt-oss
 
-[gpt-oss](https://huggingface.co/collections/openai/gpt-oss-68911959590a1634ba11c7a4) is a family of open-source models designed for general-purpose language understanding and generation. The 20&nbsp;B parameter variant (`gpt-oss-20b`) offers strong reasoning capabilities with lower latency, making it well-suited for local or specialized use cases. The larger 120&nbsp;B parameter variant (`gpt-oss-120b`) is designed for production-scale, high-reasoning workloads.
+*gpt-oss* is a family of open-source models designed for general-purpose language understanding and generation. The 20 B parameter variant (`gpt-oss-20b`) offers strong reasoning capabilities with lower latency. This makes it well-suited for local or specialized use cases. The larger 120 B parameter variant (`gpt-oss-120b`) is designed for production-scale, high-reasoning workloads.
+
+For more information, see the [gpt-oss collection](https://huggingface.co/collections/openai/gpt-oss-68911959590a1634ba11c7a4).
 
 ---
 
@@ -84,21 +86,23 @@ llm_config = LLMConfig(
 app = build_openai_app({"llm_configs": [llm_config]})
 ```
 
-**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example.
+**Note:** Before moving to a production setup, migrate to using a Serve config file to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. For an example, see [_](/serve/llm/quick-start.md#production-deployment).
 
 ---
 
 ## Deploy locally
 
-**Prerequisites**
+**Prerequisites:**
 
 * Access to GPU compute.
 
 **Dependencies:**
-gpt-oss integration is available starting from `ray>=2.49.0` and `vllm>=0.10.1`
+
+gpt-oss integration is available starting from `ray>=2.49.0` and `vllm==0.10.1`.
+
 ```bash
 pip install "ray[serve,llm]>=2.49.0"
-pip install "vllm>=0.10.1"
+pip install "vllm==0.10.1"
 ```
 
 ---
@@ -175,7 +179,7 @@ for chunk in response:
 
 ### Shutdown
 
-Shutdown your LLM service: 
+To shutdown your LLM service: 
 
 
 ```bash
@@ -238,25 +242,28 @@ anyscale service deploy -f service.yaml
 ### Send requests 
 
 The `anyscale service deploy` command output shows both the endpoint and authentication token:
+
 ```console
 (anyscale +3.9s) curl -H "Authorization: Bearer <YOUR-TOKEN>" <YOUR-ENDPOINT>
 ```
+
 You can also retrieve both from the service page in the Anyscale Console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token.  
 
 ---
 
 ### Access the Serve LLM dashboard
 
-See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling LLM-specific logging. To open the Ray Serve LLM Dashboard from an Anyscale Service:
+For instructions on enabling LLM-specific logging, see [Enable LLM monitoring](#enable-llm-monitoring). To open the Ray Serve LLM Dashboard from an Anyscale Service:
+
 1. In the Anyscale console, go to your **Service** or **Workspace**.
-2. Navigate to the **Metrics** tab.
-3. Expand **View in Grafana** and click **Serve LLM Dashboard**.
+1. Navigate to the **Metrics** tab.
+1. Expand **View in Grafana** and click **Serve LLM Dashboard**.
 
 ---
 
 ### Shutdown
 
-Shutdown your Anyscale Service:
+To shutdown your Anyscale Service:
 
 
 ```bash
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
index 03166f48daea..9814217929b7 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
@@ -7,7 +7,9 @@
    "source": [
     "# Deploy gpt-oss\n",
     "\n",
-    "[gpt-oss](https://huggingface.co/collections/openai/gpt-oss-68911959590a1634ba11c7a4) is a family of open-source models designed for general-purpose language understanding and generation. The 20&nbsp;B parameter variant (`gpt-oss-20b`) offers strong reasoning capabilities with lower latency, making it well-suited for local or specialized use cases. The larger 120&nbsp;B parameter variant (`gpt-oss-120b`) is designed for production-scale, high-reasoning workloads.\n",
+    "*gpt-oss* is a family of open-source models designed for general-purpose language understanding and generation. The 20 B parameter variant (`gpt-oss-20b`) offers strong reasoning capabilities with lower latency. This makes it well-suited for local or specialized use cases. The larger 120 B parameter variant (`gpt-oss-120b`) is designed for production-scale, high-reasoning workloads.\n",
+    "\n",
+    "For more information, see the [gpt-oss collection](https://huggingface.co/collections/openai/gpt-oss-68911959590a1634ba11c7a4).\n",
     "\n",
     "---\n",
     "\n",
@@ -103,21 +105,23 @@
    "id": "b17a7140",
    "metadata": {},
    "source": [
-    "**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example.\n",
+    "**Note:** Before moving to a production setup, migrate to using a Serve config file to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. For an example, see [_](/serve/llm/quick-start.md#production-deployment).\n",
     "\n",
     "---\n",
     "\n",
     "## Deploy locally\n",
     "\n",
-    "**Prerequisites**\n",
+    "**Prerequisites:**\n",
     "\n",
     "* Access to GPU compute.\n",
     "\n",
     "**Dependencies:**\n",
-    "gpt-oss integration is available starting from `ray>=2.49.0` and `vllm>=0.10.1`\n",
+    "\n",
+    "gpt-oss integration is available starting from `ray>=2.49.0` and `vllm==0.10.1`.\n",
+    "\n",
     "```bash\n",
     "pip install \"ray[serve,llm]>=2.49.0\"\n",
-    "pip install \"vllm>=0.10.1\"\n",
+    "pip install \"vllm==0.10.1\"\n",
     "```\n",
     "\n",
     "---\n",
@@ -227,7 +231,7 @@
     "\n",
     "### Shutdown\n",
     "\n",
-    "Shutdown your LLM service: "
+    "To shutdown your LLM service: "
    ]
   },
   {
@@ -312,25 +316,28 @@
     "### Send requests \n",
     "\n",
     "The `anyscale service deploy` command output shows both the endpoint and authentication token:\n",
+    "\n",
     "```console\n",
     "(anyscale +3.9s) curl -H \"Authorization: Bearer <YOUR-TOKEN>\" <YOUR-ENDPOINT>\n",
     "```\n",
+    "\n",
     "You can also retrieve both from the service page in the Anyscale Console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token.  \n",
     "\n",
     "---\n",
     "\n",
     "### Access the Serve LLM dashboard\n",
     "\n",
-    "See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling LLM-specific logging. To open the Ray Serve LLM Dashboard from an Anyscale Service:\n",
+    "For instructions on enabling LLM-specific logging, see [Enable LLM monitoring](#enable-llm-monitoring). To open the Ray Serve LLM Dashboard from an Anyscale Service:\n",
+    "\n",
     "1. In the Anyscale console, go to your **Service** or **Workspace**.\n",
-    "2. Navigate to the **Metrics** tab.\n",
-    "3. Expand **View in Grafana** and click **Serve LLM Dashboard**.\n",
+    "1. Navigate to the **Metrics** tab.\n",
+    "1. Expand **View in Grafana** and click **Serve LLM Dashboard**.\n",
     "\n",
     "---\n",
     "\n",
     "### Shutdown\n",
     "\n",
-    "Shutdown your Anyscale Service:"
+    "To shutdown your Anyscale Service:"
    ]
   },
   {

From 7a1ebb5982d566a9aba71fb58a0413b4eb541e98 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Fri, 3 Oct 2025 12:57:40 -0700
Subject: [PATCH 09/19] black lint

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
index ce6a3faa1820..5024ebc0f43b 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
@@ -27,7 +27,7 @@
             model_id="my-gpt-oss",
             model_source="openai/gpt-oss-120b",
         ),
-        accelerator_type="L40S", # Or "A100-40G"
+        accelerator_type="L40S",  # Or "A100-40G"
         deployment_config=dict(
             autoscaling_config=dict(
                 min_replicas=1,

From b25fe07ff70387655583b94418d9fd9e0207e5f9 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Fri, 3 Oct 2025 13:00:01 -0700
Subject: [PATCH 10/19] fix link

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../serve/tutorials/deployment-serve-llm/gpt-oss/README.md      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
index 34d6fded0eb9..ee5591f7987a 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
@@ -86,7 +86,7 @@ llm_config = LLMConfig(
 app = build_openai_app({"llm_configs": [llm_config]})
 ```
 
-**Note:** Before moving to a production setup, migrate to using a Serve config file to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. For an example, see [_](/serve/llm/quick-start.md#production-deployment).
+**Note:** Before moving to a production setup, migrate to using a Serve config file to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. For an example, see [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment).
 
 ---
 

From 4a8e52f8c2fe87b7e6b5368bf54162b3e594ad54 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Fri, 3 Oct 2025 14:01:48 -0700
Subject: [PATCH 11/19] fix link

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb      | 2 +-
 .../tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py    | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
index 9814217929b7..899c72788287 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
@@ -105,7 +105,7 @@
    "id": "b17a7140",
    "metadata": {},
    "source": [
-    "**Note:** Before moving to a production setup, migrate to using a Serve config file to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. For an example, see [_](/serve/llm/quick-start.md#production-deployment).\n",
+    "**Note:** Before moving to a production setup, migrate to using a Serve config file to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. For an example, see [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment).\n",
     "\n",
     "---\n",
     "\n",
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
index 5024ebc0f43b..60057f84c4c3 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
@@ -1,7 +1,8 @@
 # serve_gpt_oss.py
+import os
 from ray.serve.llm import LLMConfig, build_openai_app
 
-GPT_OSS_SIZE = "20b"  # or "120b"
+GPT_OSS_SIZE = os.environ.get("GPT_OSS_SIZE", "20b") # set to "20b" or "120b"
 
 if GPT_OSS_SIZE == "20b":
     llm_config = LLMConfig(

From 100a5b60e99e0a585b6960c18b09e14d9065fbd5 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Fri, 3 Oct 2025 14:17:57 -0700
Subject: [PATCH 12/19] add explicit env variable to control gpt oss size

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../deployment-serve-llm/gpt-oss/serve_gpt_oss.py        | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
index 60057f84c4c3..95e38a018058 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/serve_gpt_oss.py
@@ -2,7 +2,14 @@
 import os
 from ray.serve.llm import LLMConfig, build_openai_app
 
-GPT_OSS_SIZE = os.environ.get("GPT_OSS_SIZE", "20b") # set to "20b" or "120b"
+# Configure model size via environment variable:
+# export GPT_OSS_SIZE=20b   # for gpt-oss-20b (default)
+# export GPT_OSS_SIZE=120b  # for gpt-oss-120b
+GPT_OSS_SIZE = os.environ.get("GPT_OSS_SIZE", "20b")
+print(
+    f"Set the 'GPT_OSS_SIZE' environment variable to '20b' or '120b' to use the appropriate config for your model."
+)
+print(f"Using GPT-OSS size: {GPT_OSS_SIZE}")
 
 if GPT_OSS_SIZE == "20b":
     llm_config = LLMConfig(

From 8c61525b386b8a4f2323b25647a41264a7dd6957 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Fri, 3 Oct 2025 14:30:28 -0700
Subject: [PATCH 13/19] consistent docker image with notebook cell

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../serve/tutorials/deployment-serve-llm/gpt-oss/Dockerfile     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/Dockerfile b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/Dockerfile
index a48046d90fb6..cb8e884572c0 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/Dockerfile
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/Dockerfile
@@ -1,4 +1,4 @@
-FROM anyscale/ray:2.49.1-slim-py312-cu128
+FROM anyscale/ray:2.49.0-slim-py312-cu128
 
 # C compiler for Triton’s runtime build step (vLLM V1 engine)
 # https://github.com/vllm-project/vllm/issues/2997

From 51cee8b30cb542d167845578871088b1ea90aebd Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Fri, 3 Oct 2025 19:48:25 -0700
Subject: [PATCH 14/19] rendering on ray docs

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 doc/source/serve/examples.yml                             | 8 ++++++++
 doc/source/serve/llm/index.md                             | 3 ++-
 .../tutorials/deployment-serve-llm/gpt-oss/README.md      | 5 +++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/doc/source/serve/examples.yml b/doc/source/serve/examples.yml
index 56416a1e0b4c..8ef50ad58aae 100644
--- a/doc/source/serve/examples.yml
+++ b/doc/source/serve/examples.yml
@@ -122,6 +122,14 @@ examples:
       - natural language processing
     link: tutorials/deployment-serve-llm/hybrid-reasoning-llm/README
     related_technology: llm applications
+  - title: Deploy gpt-oss
+    skill_level: beginner
+    use_cases:
+      - generative ai
+      - large language models
+      - natural language processing
+    link: tutorials/deployment-serve-llm/gpt-oss/README
+    related_technology: llm applications
   - title: Serve a Chatbot with Request and Response Streaming
     skill_level: intermediate
     use_cases:
diff --git a/doc/source/serve/llm/index.md b/doc/source/serve/llm/index.md
index 382a107f81ff..5915e6c8712d 100644
--- a/doc/source/serve/llm/index.md
+++ b/doc/source/serve/llm/index.md
@@ -67,4 +67,5 @@ Cache-aware request routing <prefix-aware-request-router>
 - {doc}`Deploy a large-sized LLM <../tutorials/deployment-serve-llm/large-size-llm/README>`
 - {doc}`Deploy a vision LLM <../tutorials/deployment-serve-llm/vision-llm/README>`
 - {doc}`Deploy a reasoning LLM <../tutorials/deployment-serve-llm/reasoning-llm/README>`
-- {doc}`Deploy a hybrid reasoning LLM <../tutorials/deployment-serve-llm/hybrid-reasoning-llm/README>`
\ No newline at end of file
+- {doc}`Deploy a hybrid reasoning LLM <../tutorials/deployment-serve-llm/hybrid-reasoning-llm/README>`
+- {doc}`Deploy gpt-oss <../tutorials/deployment-serve-llm/gpt-oss/README>`
\ No newline at end of file
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
index ee5591f7987a..e42cdeec4c94 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
@@ -10,6 +10,11 @@ jupyter nbconvert "$notebook.ipynb" --to markdown --output "README.md"
 
 # Deploy gpt-oss
 
+<div align="left">
+<a target="_blank" href="https://console.anyscale.com/template-preview/deployment-serve-llm?file=%252Ffiles%252Fgpt-oss"><img src="https://img.shields.io/badge/🚀 Run_on-Anyscale-9hf"></a>&nbsp;
+<a href="https://github.com/ray-project/ray/tree/master/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss" role="button"><img src="https://img.shields.io/static/v1?label=&amp;message=View%20On%20GitHub&amp;color=586069&amp;logo=github&amp;labelColor=2f363d"></a>&nbsp;
+</div>
+
 *gpt-oss* is a family of open-source models designed for general-purpose language understanding and generation. The 20 B parameter variant (`gpt-oss-20b`) offers strong reasoning capabilities with lower latency. This makes it well-suited for local or specialized use cases. The larger 120 B parameter variant (`gpt-oss-120b`) is designed for production-scale, high-reasoning workloads.
 
 For more information, see the [gpt-oss collection](https://huggingface.co/collections/openai/gpt-oss-68911959590a1634ba11c7a4).

From bb5a72dd44da8fbbb4c2cf9a9bfb90674877d181 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Mon, 6 Oct 2025 10:42:42 -0700
Subject: [PATCH 15/19] remove %%bash

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../deployment-serve-llm/gpt-oss/README.md    | 20 +++++--------------
 .../gpt-oss/notebook.ipynb                    |  5 -----
 .../hybrid-reasoning-llm/README.md            | 12 ++++-------
 .../hybrid-reasoning-llm/notebook.ipynb       |  4 ----
 .../large-size-llm/README.md                  | 15 +++++---------
 .../large-size-llm/notebook.ipynb             |  5 -----
 .../medium-size-llm/README.md                 | 15 +++++---------
 .../medium-size-llm/notebook.ipynb            |  5 -----
 .../reasoning-llm/README.md                   |  9 +++------
 .../reasoning-llm/notebook.ipynb              |  3 ---
 .../small-size-llm/README.md                  | 15 +++++---------
 .../small-size-llm/notebook.ipynb             |  5 -----
 .../deployment-serve-llm/vision-llm/README.md |  9 +++------
 .../vision-llm/notebook.ipynb                 |  3 ---
 14 files changed, 30 insertions(+), 95 deletions(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
index e42cdeec4c94..134d3c03df52 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/README.md
@@ -10,11 +10,6 @@ jupyter nbconvert "$notebook.ipynb" --to markdown --output "README.md"
 
 # Deploy gpt-oss
 
-<div align="left">
-<a target="_blank" href="https://console.anyscale.com/template-preview/deployment-serve-llm?file=%252Ffiles%252Fgpt-oss"><img src="https://img.shields.io/badge/🚀 Run_on-Anyscale-9hf"></a>&nbsp;
-<a href="https://github.com/ray-project/ray/tree/master/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss" role="button"><img src="https://img.shields.io/static/v1?label=&amp;message=View%20On%20GitHub&amp;color=586069&amp;logo=github&amp;labelColor=2f363d"></a>&nbsp;
-</div>
-
 *gpt-oss* is a family of open-source models designed for general-purpose language understanding and generation. The 20 B parameter variant (`gpt-oss-20b`) offers strong reasoning capabilities with lower latency. This makes it well-suited for local or specialized use cases. The larger 120 B parameter variant (`gpt-oss-120b`) is designed for production-scale, high-reasoning workloads.
 
 For more information, see the [gpt-oss collection](https://huggingface.co/collections/openai/gpt-oss-68911959590a1634ba11c7a4).
@@ -119,8 +114,7 @@ Follow the instructions in [Configure Ray Serve LLM](#configure-ray-serve-llm) a
 In a terminal, run:
 
 
-```bash
-%%bash
+```python
 serve run serve_gpt_oss:app --non-blocking
 ```
 
@@ -135,8 +129,7 @@ Your endpoint is available locally at `http://localhost:8000`. You can use a pla
 **Example curl:**
 
 
-```bash
-%%bash
+```python
 curl -X POST http://localhost:8000/v1/chat/completions \
   -H "Authorization: Bearer FAKE_KEY" \
   -H "Content-Type: application/json" \
@@ -187,8 +180,7 @@ for chunk in response:
 To shutdown your LLM service: 
 
 
-```bash
-%%bash
+```python
 serve shutdown -y
 ```
 
@@ -236,8 +228,7 @@ applications:
 Deploy your service:
 
 
-```bash
-%%bash
+```python
 anyscale service deploy -f service.yaml
 ```
 
@@ -271,8 +262,7 @@ For instructions on enabling LLM-specific logging, see [Enable LLM monitoring](#
 To shutdown your Anyscale Service:
 
 
-```bash
-%%bash
+```python
 anyscale service terminate -n deploy-gpt-oss
 ```
 
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
index 899c72788287..5c28d1749e01 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
@@ -140,7 +140,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "serve run serve_gpt_oss:app --non-blocking"
    ]
   },
@@ -167,7 +166,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "curl -X POST http://localhost:8000/v1/chat/completions \\\n",
     "  -H \"Authorization: Bearer FAKE_KEY\" \\\n",
     "  -H \"Content-Type: application/json\" \\\n",
@@ -241,7 +239,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "serve shutdown -y"
    ]
   },
@@ -301,7 +298,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "anyscale service deploy -f service.yaml"
    ]
   },
@@ -347,7 +343,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "anyscale service terminate -n deploy-gpt-oss"
    ]
   },
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md
index 53283909a5cf..852a989b564f 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md
@@ -181,8 +181,7 @@ Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) t
 In a terminal, run:  
 
 
-```bash
-%%bash
+```python
 serve run serve_qwen_3_32b:app --non-blocking
 ```
 
@@ -201,8 +200,7 @@ You can disable thinking in Qwen-3 by either adding a `/no_think` tag in the pro
 Example curl with `/no_think`:
 
 
-```bash
-%%bash
+```python
 curl -X POST http://localhost:8000/v1/chat/completions \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer FAKE_KEY" \
@@ -247,8 +245,7 @@ You can enable thinking in Qwen-3 by either adding a `/think` tag in the prompt
 Example curl with `/think`:
 
 
-```bash
-%%bash
+```python
 curl -X POST http://localhost:8000/v1/chat/completions \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer FAKE_KEY" \
@@ -290,8 +287,7 @@ If you configure a valid reasoning parser, the reasoning output should appear in
 Shutdown your LLM service:
 
 
-```bash
-%%bash
+```python
 serve shutdown -y
 ```
 
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb
index c9f79cce9e31..c693108a27bf 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb
@@ -195,7 +195,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "serve run serve_qwen_3_32b:app --non-blocking"
    ]
   },
@@ -226,7 +225,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "curl -X POST http://localhost:8000/v1/chat/completions \\\n",
     "  -H \"Content-Type: application/json\" \\\n",
     "  -H \"Authorization: Bearer FAKE_KEY\" \\\n",
@@ -294,7 +292,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "curl -X POST http://localhost:8000/v1/chat/completions \\\n",
     "  -H \"Content-Type: application/json\" \\\n",
     "  -H \"Authorization: Bearer FAKE_KEY\" \\\n",
@@ -359,7 +356,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "serve shutdown -y"
    ]
   },
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md
index ff3d49b1d5e8..61557ab44be1 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md
@@ -98,8 +98,7 @@ Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) t
 In a terminal, run:  
 
 
-```bash
-%%bash
+```python
 serve run serve_deepseek_r1:app --non-blocking
 ```
 
@@ -114,8 +113,7 @@ Your endpoint is available locally at `http://localhost:8000` and you can use a
 Example curl:
 
 
-```bash
-%%bash
+```python
 curl -X POST http://localhost:8000/v1/chat/completions \
   -H "Authorization: Bearer FAKE_KEY" \
   -H "Content-Type: application/json" \
@@ -163,8 +161,7 @@ for chunk in response:
 Shutdown your LLM service: 
 
 
-```bash
-%%bash
+```python
 serve shutdown -y
 ```
 
@@ -225,8 +222,7 @@ applications:
 Deploy your service
 
 
-```bash
-%%bash
+```python
 anyscale service deploy -f service.yaml
 ```
 
@@ -272,8 +268,7 @@ See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling
 Shutdown your Anyscale service:
 
 
-```bash
-%%bash
+```python
 anyscale service terminate -n deploy-deepseek-r1
 ```
 
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb
index c1fc5ba09fac..e03c02b97204 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb
@@ -112,7 +112,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "serve run serve_deepseek_r1:app --non-blocking"
    ]
   },
@@ -139,7 +138,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "curl -X POST http://localhost:8000/v1/chat/completions \\\n",
     "  -H \"Authorization: Bearer FAKE_KEY\" \\\n",
     "  -H \"Content-Type: application/json\" \\\n",
@@ -210,7 +208,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "serve shutdown -y"
    ]
   },
@@ -285,7 +282,6 @@
    },
    "outputs": [],
    "source": [
-    "%%bash\n",
     "anyscale service deploy -f service.yaml"
    ]
   },
@@ -343,7 +339,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "anyscale service terminate -n deploy-deepseek-r1"
    ]
   },
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md
index 1762ebdca98b..6bdd151e0f8b 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md
@@ -90,8 +90,7 @@ Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) t
 In a terminal, run:  
 
 
-```bash
-%%bash
+```python
 export HF_TOKEN=<YOUR-HUGGINGFACE-TOKEN>
 serve run serve_llama_3_1_70b:app --non-blocking
 ```
@@ -107,8 +106,7 @@ Your endpoint is available locally at `http://localhost:8000` and you can use a
 Example curl:
 
 
-```bash
-%%bash
+```python
 curl -X POST http://localhost:8000/v1/chat/completions \
   -H "Authorization: Bearer FAKE_KEY" \
   -H "Content-Type: application/json" \
@@ -148,8 +146,7 @@ for chunk in response:
 Shutdown your LLM service: 
 
 
-```bash
-%%bash
+```python
 serve shutdown -y
 ```
 
@@ -183,8 +180,7 @@ applications:
 Deploy your service. Make sure you forward your Hugging Face token to the command.
 
 
-```bash
-%%bash
+```python
 anyscale service deploy -f service.yaml --env HF_TOKEN=<YOUR-HUGGINGFACE-TOKEN>
 ```
 
@@ -228,8 +224,7 @@ See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling
 Shutdown your Anyscale service:
 
 
-```bash
-%%bash
+```python
 anyscale service terminate -n deploy-llama-3-70b
 ```
 
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb
index 0283fa9e483d..f63015fef80f 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb
@@ -104,7 +104,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "export HF_TOKEN=<YOUR-HUGGINGFACE-TOKEN>\n",
     "serve run serve_llama_3_1_70b:app --non-blocking"
    ]
@@ -132,7 +131,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "curl -X POST http://localhost:8000/v1/chat/completions \\\n",
     "  -H \"Authorization: Bearer FAKE_KEY\" \\\n",
     "  -H \"Content-Type: application/json\" \\\n",
@@ -195,7 +193,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "serve shutdown -y"
    ]
   },
@@ -243,7 +240,6 @@
    },
    "outputs": [],
    "source": [
-    "%%bash\n",
     "anyscale service deploy -f service.yaml --env HF_TOKEN=<YOUR-HUGGINGFACE-TOKEN>"
    ]
   },
@@ -299,7 +295,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "anyscale service terminate -n deploy-llama-3-70b"
    ]
   },
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md
index 94bbdd36b858..77f1a3d118f9 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md
@@ -156,8 +156,7 @@ Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) t
 In a terminal, run:  
 
 
-```bash
-%%bash
+```python
 serve run serve_qwq_32b:app --non-blocking
 ```
 
@@ -172,8 +171,7 @@ Your endpoint is available locally at `http://localhost:8000` and you can use a
 Example curl:
 
 
-```bash
-%%bash
+```python
 curl -X POST http://localhost:8000/v1/chat/completions \
   -H "Authorization: Bearer FAKE_KEY" \
   -H "Content-Type: application/json" \
@@ -213,8 +211,7 @@ If you configure a valid reasoning parser, the reasoning output should appear in
 Shutdown your LLM service:
 
 
-```bash
-%%bash
+```python
 serve shutdown -y
 ```
 
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb
index 62d4c4646f4c..ae9e798687ad 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb
@@ -170,7 +170,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "serve run serve_qwq_32b:app --non-blocking"
    ]
   },
@@ -197,7 +196,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "curl -X POST http://localhost:8000/v1/chat/completions \\\n",
     "  -H \"Authorization: Bearer FAKE_KEY\" \\\n",
     "  -H \"Content-Type: application/json\" \\\n",
@@ -260,7 +258,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "serve shutdown -y"
    ]
   },
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md
index 1a7e017b7df9..e0e7e4b5594b 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md
@@ -85,8 +85,7 @@ Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) t
 In a terminal, run:  
 
 
-```bash
-%%bash
+```python
 export HF_TOKEN=<YOUR-HUGGINGFACE-TOKEN>
 serve run serve_llama_3_1_8b:app --non-blocking
 ```
@@ -102,8 +101,7 @@ Your endpoint is available locally at `http://localhost:8000`. You can use a pla
 Example curl:
 
 
-```bash
-%%bash
+```python
 curl -X POST http://localhost:8000/v1/chat/completions \
   -H "Authorization: Bearer FAKE_KEY" \
   -H "Content-Type: application/json" \
@@ -143,8 +141,7 @@ for chunk in response:
 Shutdown your LLM service: 
 
 
-```bash
-%%bash
+```python
 serve shutdown -y
 ```
 
@@ -180,8 +177,7 @@ applications:
 Deploy your service with the following command. Make sure to forward your Hugging Face token:
 
 
-```bash
-%%bash
+```python
 anyscale service deploy -f service.yaml --env HF_TOKEN=<YOUR-HUGGINGFACE-TOKEN>
 ```
 
@@ -225,8 +221,7 @@ See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling
 Shutdown your Anyscale Service:
 
 
-```bash
-%%bash
+```python
 anyscale service terminate -n deploy-llama-3-8b
 ```
 
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb
index b53ba2accc67..2ca1fbedee84 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb
@@ -99,7 +99,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "export HF_TOKEN=<YOUR-HUGGINGFACE-TOKEN>\n",
     "serve run serve_llama_3_1_8b:app --non-blocking"
    ]
@@ -127,7 +126,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "curl -X POST http://localhost:8000/v1/chat/completions \\\n",
     "  -H \"Authorization: Bearer FAKE_KEY\" \\\n",
     "  -H \"Content-Type: application/json\" \\\n",
@@ -190,7 +188,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "serve shutdown -y"
    ]
   },
@@ -240,7 +237,6 @@
    },
    "outputs": [],
    "source": [
-    "%%bash\n",
     "anyscale service deploy -f service.yaml --env HF_TOKEN=<YOUR-HUGGINGFACE-TOKEN>"
    ]
   },
@@ -296,7 +292,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%bash\n",
     "anyscale service terminate -n deploy-llama-3-8b"
    ]
   },
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md
index aa18ede4a755..440bdea734af 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md
@@ -81,8 +81,7 @@ Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) t
 In a terminal, run:   
 
 
-```bash
-%%bash
+```python
 serve run serve_qwen_VL:app --non-blocking
 ```
 
@@ -97,8 +96,7 @@ Your endpoint is available locally at `http://localhost:8000` and you can use a
 Example curl with image URL:
 
 
-```bash
-%%bash
+```python
 curl -X POST http://localhost:8000/v1/chat/completions \
   -H "Authorization: Bearer FAKE_KEY" \
   -H "Content-Type: application/json" \
@@ -187,8 +185,7 @@ for chunk in response:
 Shutdown your LLM service:
 
 
-```bash
-%%bash
+```python
 serve shutdown -y
 ```
 
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb
index 1678fdd7a57e..ada641b97c97 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb
@@ -95,7 +95,6 @@
             "metadata": {},
             "outputs": [],
             "source": [
-                "%%bash\n",
                 "serve run serve_qwen_VL:app --non-blocking"
             ]
         },
@@ -122,7 +121,6 @@
             "metadata": {},
             "outputs": [],
             "source": [
-                "%%bash\n",
                 "curl -X POST http://localhost:8000/v1/chat/completions \\\n",
                 "  -H \"Authorization: Bearer FAKE_KEY\" \\\n",
                 "  -H \"Content-Type: application/json\" \\\n",
@@ -245,7 +243,6 @@
             "metadata": {},
             "outputs": [],
             "source": [
-                "%%bash\n",
                 "serve shutdown -y"
             ]
         },

From c61955810b580b2e68726c1edc1607c52e54251e Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Mon, 6 Oct 2025 13:07:53 -0700
Subject: [PATCH 16/19] update python conversion script

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../serve/tutorials/deployment-serve-llm/ci/nb2py.py       | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py b/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py
index 2c94f8270b9e..8968d35b66b0 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py
+++ b/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py
@@ -42,7 +42,10 @@ def convert_notebook(
             else:
                 # Detect any IPython '!' shell commands in code lines
                 has_bang = any(line.lstrip().startswith("!") for line in lines)
-                if has_bang:
+                # Start with "serve run" "serve shutdown" "curl" or "anyscale service" commands
+                to_ignore_cmd = ("serve run", "serve shutdown", "curl", "anyscale service")
+                has_illegal_start = any(line.lstrip().startswith(to_ignore_cmd) for line in lines)
+                if has_bang or has_illegal_start:
                     if ignore_cmds:
                         continue
                     out.write("import subprocess\n")
@@ -58,7 +61,7 @@ def convert_notebook(
                             )
                         else:
                             out.write(line.rstrip() + "\n")
-                    out.write("\n")
+                    out.write("\n")                
                 else:
                     # Regular Python cell:
                     code = cell.source.rstrip()

From 9e03d017a9c485ae2326650c4e987f2ecee4cce6 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Mon, 6 Oct 2025 13:37:40 -0700
Subject: [PATCH 17/19] black lint

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../tutorials/deployment-serve-llm/ci/nb2py.py    | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py b/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py
index 8968d35b66b0..ec78ed993725 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py
+++ b/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py
@@ -43,9 +43,16 @@ def convert_notebook(
                 # Detect any IPython '!' shell commands in code lines
                 has_bang = any(line.lstrip().startswith("!") for line in lines)
                 # Start with "serve run" "serve shutdown" "curl" or "anyscale service" commands
-                to_ignore_cmd = ("serve run", "serve shutdown", "curl", "anyscale service")
-                has_illegal_start = any(line.lstrip().startswith(to_ignore_cmd) for line in lines)
-                if has_bang or has_illegal_start:
+                to_ignore_cmd = (
+                    "serve run",
+                    "serve shutdown",
+                    "curl",
+                    "anyscale service",
+                )
+                has_ignored_start = any(
+                    line.lstrip().startswith(to_ignore_cmd) for line in lines
+                )
+                if has_bang or has_ignored_start:
                     if ignore_cmds:
                         continue
                     out.write("import subprocess\n")
@@ -61,7 +68,7 @@ def convert_notebook(
                             )
                         else:
                             out.write(line.rstrip() + "\n")
-                    out.write("\n")                
+                    out.write("\n")
                 else:
                     # Regular Python cell:
                     code = cell.source.rstrip()

From 11490e311176f7dce8bd7a35f168dd690d0b0fb8 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <62435714+Aydin-ab@users.noreply.github.com>
Date: Mon, 6 Oct 2025 17:39:15 -0700
Subject: [PATCH 18/19] angelina review: apply suggestions from code review

Co-authored-by: angelinalg <122562471+angelinalg@users.noreply.github.com>
Signed-off-by: Aydin Abiar <62435714+Aydin-ab@users.noreply.github.com>
---
 .../deployment-serve-llm/README.ipynb         |  2 +-
 .../tutorials/deployment-serve-llm/README.md  |  2 +-
 .../gpt-oss/notebook.ipynb                    | 44 +++++++++----------
 3 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb
index f3b04b0d9587..e3cd22014286 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb
@@ -44,7 +44,7 @@
     "---\n",
     "\n",
     "**[Deploy gpt-oss](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/gpt-oss/README.html)**  \n",
-    "Deploy gpt-oss reasoning models, including `gpt-oss-20b` for lower latency use cases and `gpt-oss-120b` for high-reasoning, production-scale workloads."
+    "Deploy gpt-oss reasoning models for high-reasoning, production-scale workloads, for lower latency (`gpt-oss-20b`) and high-reasoning (`gpt-oss-120b`) use cases."
    ]
   }
  ],
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/README.md
index d3bee1c38855..016a4b49a507 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/README.md
+++ b/doc/source/serve/tutorials/deployment-serve-llm/README.md
@@ -43,4 +43,4 @@ Deploy models that can switch between reasoning and non-reasoning modes for flex
 ---
 
 **[Deploy gpt-oss](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/gpt-oss/README.html)**  
-Deploy gpt-oss reasoning models, including `gpt-oss-20b` for lower latency use cases and `gpt-oss-120b` for high-reasoning, production-scale workloads.
+Deploy gpt-oss reasoning models for high-reasoning, production-scale workloads, for lower latency (`gpt-oss-20b`) and high-reasoning (`gpt-oss-120b`) use cases.
diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
index 5c28d1749e01..77227defef69 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
@@ -7,7 +7,7 @@
    "source": [
     "# Deploy gpt-oss\n",
     "\n",
-    "*gpt-oss* is a family of open-source models designed for general-purpose language understanding and generation. The 20 B parameter variant (`gpt-oss-20b`) offers strong reasoning capabilities with lower latency. This makes it well-suited for local or specialized use cases. The larger 120 B parameter variant (`gpt-oss-120b`) is designed for production-scale, high-reasoning workloads.\n",
+    "*gpt-oss* is a family of open-source models designed for general-purpose language understanding and generation. The 20B parameter variant (`gpt-oss-20b`) offers strong reasoning capabilities with lower latency. This makes it well-suited for local or specialized use cases. The larger 120B parameter variant (`gpt-oss-120b`) is designed for production-scale, high-reasoning workloads.\n",
     "\n",
     "For more information, see the [gpt-oss collection](https://huggingface.co/collections/openai/gpt-oss-68911959590a1634ba11c7a4).\n",
     "\n",
@@ -111,11 +111,11 @@
     "\n",
     "## Deploy locally\n",
     "\n",
-    "**Prerequisites:**\n",
+    "### Prerequisites\n",
     "\n",
     "* Access to GPU compute.\n",
     "\n",
-    "**Dependencies:**\n",
+    "### Dependencies\n",
     "\n",
     "gpt-oss integration is available starting from `ray>=2.49.0` and `vllm==0.10.1`.\n",
     "\n",
@@ -126,7 +126,7 @@
     "\n",
     "---\n",
     "\n",
-    "### Launch\n",
+    "### Launch the service\n",
     "\n",
     "Follow the instructions in [Configure Ray Serve LLM](#configure-ray-serve-llm) according to the model size you choose, and define your app in a Python module `serve_gpt_oss.py`.\n",
     "\n",
@@ -148,7 +148,7 @@
    "id": "df944967",
    "metadata": {},
    "source": [
-    "Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded.\n",
+    "Deployment typically takes a few minutes as Ray provisions the cluster, the vLLM server starts, and Ray Serve downloads the model.\n",
     "\n",
     "---\n",
     "\n",
@@ -156,7 +156,7 @@
     "\n",
     "Your endpoint is available locally at `http://localhost:8000`. You can use a placeholder authentication token for the OpenAI client, for example `\"FAKE_KEY\"`.\n",
     "\n",
-    "**Example curl:**"
+    "#### Example curl"
    ]
   },
   {
@@ -177,7 +177,7 @@
    "id": "d623a30f",
    "metadata": {},
    "source": [
-    "**Example Python:**"
+    "#### Example Python"
    ]
   },
   {
@@ -227,7 +227,7 @@
     "\n",
     "---\n",
     "\n",
-    "### Shutdown\n",
+    "### Shut down the service\n",
     "\n",
     "To shutdown your LLM service: "
    ]
@@ -250,15 +250,15 @@
     "\n",
     "---\n",
     "\n",
-    "## Deploy to production with Anyscale Services\n",
+    "## Deploy to production with Anyscale services\n",
     "\n",
-    "For production deployment, use Anyscale Services to deploy the Ray Serve app to a dedicated cluster without modifying the code. Anyscale ensures scalability, fault tolerance, and load balancing, keeping the service resilient against node failures, high traffic, and rolling updates.\n",
+    "For production deployment, use Anyscale services to deploy the Ray Serve app to a dedicated cluster without modifying the code. Anyscale ensures scalability, fault tolerance, and load balancing, keeping the service resilient against node failures, high traffic, and rolling updates.\n",
     "\n",
     "---\n",
     "\n",
     "### Launch the service\n",
     "\n",
-    "Anyscale provides out-of-the-box images (`anyscale/ray-llm`) which come pre-loaded with Ray Serve LLM, vLLM, and all required GPU/runtime dependencies. See the [Anyscale base images](https://docs.anyscale.com/reference/base-images) for details on what each image includes.\n",
+    "Anyscale provides out-of-the-box images (`anyscale/ray-llm`), which come pre-loaded with Ray Serve LLM, vLLM, and all required GPU and runtime dependencies. See the [Anyscale base images](https://docs.anyscale.com/reference/base-images) for details on what each image includes.\n",
     "\n",
     "Build a minimal Dockerfile:\n",
     "```Dockerfile\n",
@@ -272,7 +272,7 @@
     "RUN pip install vllm==0.10.1\n",
     "```\n",
     "\n",
-    "Create your Anyscale Service configuration in a new `service.yaml` file and reference the Dockerfile with `containerfile`:\n",
+    "Create your Anyscale service configuration in a new `service.yaml` file and reference the Dockerfile with `containerfile`:\n",
     "\n",
     "```yaml\n",
     "# service.yaml\n",
@@ -317,17 +317,17 @@
     "(anyscale +3.9s) curl -H \"Authorization: Bearer <YOUR-TOKEN>\" <YOUR-ENDPOINT>\n",
     "```\n",
     "\n",
-    "You can also retrieve both from the service page in the Anyscale Console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token.  \n",
+    "You can also retrieve both from the service page in the Anyscale console. Click **Query** at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token.  \n",
     "\n",
     "---\n",
     "\n",
     "### Access the Serve LLM dashboard\n",
     "\n",
-    "For instructions on enabling LLM-specific logging, see [Enable LLM monitoring](#enable-llm-monitoring). To open the Ray Serve LLM Dashboard from an Anyscale Service:\n",
+    "For instructions on enabling LLM-specific logging, see [Enable LLM monitoring](#enable-llm-monitoring). To open the Ray Serve LLM Dashboard from an Anyscale service:\n",
     "\n",
-    "1. In the Anyscale console, go to your **Service** or **Workspace**.\n",
+    "1. In the Anyscale console, go to the **Service** or **Workspace** tab.\n",
     "1. Navigate to the **Metrics** tab.\n",
-    "1. Expand **View in Grafana** and click **Serve LLM Dashboard**.\n",
+    "1. Click **View in Grafana** and click **Serve LLM Dashboard**.\n",
     "\n",
     "---\n",
     "\n",
@@ -446,19 +446,19 @@
     ")\n",
     "```\n",
     "\n",
-    "**Note:** There is no reliable way to completely disable reasoning.\n",
+    "**Note:** There's no reliable way to completely disable reasoning.\n",
     "\n",
     "---\n",
     "\n",
     "## Troubleshooting\n",
     "\n",
-    "**Can't download the vocab file**  \n",
+    "### Can't download the vocab file  \n",
     "```console\n",
     "openai_harmony.HarmonyError: error downloading or loading vocab file: failed to download or load vocab\n",
     "```\n",
     "\n",
-    "The `openai_harmony` library needs the *tiktoken* encoding files and tries to fetch them from OpenAI's public host. Common cause includes:\n",
-    "- Corporate firewall or proxy blocks `openaipublic.blob.core.windows.net`; you may need to whitelist this domain.\n",
+    "The `openai_harmony` library needs the *tiktoken* encoding files and tries to fetch them from OpenAI's public host. Common causes includes:\n",
+    "- Corporate firewall or proxy blocks `openaipublic.blob.core.windows.net`. You may need to whitelist this domain.\n",
     "- Intermittent network issues.\n",
     "- Race conditions when multiple processes try to download to the same cache. This can happen when [deploying multiple models at the same time](https://github.com/openai/harmony/pull/41).\n",
     "\n",
@@ -470,11 +470,11 @@
     "export TIKTOKEN_ENCODINGS_BASE=${PWD}/tiktoken_encodings\n",
     "```\n",
     "\n",
-    "**`gpt-oss` architecture not recognized**  \n",
+    "### `gpt-oss` architecture not recognized \n",
     "```console\n",
     "Value error, The checkpoint you are trying to load has model type `gpt_oss` but Transformers does not recognize this architecture. This could be because of an issue with the checkpoint, or because your version of Transformers is out of date.\n",
     "```\n",
-    "Older vLLM and transformers versions don't register `gpt_oss`, raising an error when vLLM hands off to transformers. Upgrade **vLLM ≥ 0.10.1** and let your package resolver such as `pip` handle the other dependencies.\n",
+    "Older vLLM and Transformers versions don't register `gpt_oss`, raising an error when vLLM hands off to Transformers. Upgrade **vLLM ≥ 0.10.1** and let your package resolver such as `pip` handle the other dependencies.\n",
     "```bash\n",
     "pip install -U \"vllm>=0.10.1\"\n",
     "```\n",

From 85bf55e6d4631b6011a33fee2449f981f3363798 Mon Sep 17 00:00:00 2001
From: Aydin Abiar <aydin@anyscale.com>
Date: Mon, 6 Oct 2025 17:41:15 -0700
Subject: [PATCH 19/19] add serve config file link

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
---
 .../serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
index 77227defef69..74fa42722638 100644
--- a/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
+++ b/doc/source/serve/tutorials/deployment-serve-llm/gpt-oss/notebook.ipynb
@@ -105,7 +105,7 @@
    "id": "b17a7140",
    "metadata": {},
    "source": [
-    "**Note:** Before moving to a production setup, migrate to using a Serve config file to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. For an example, see [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment).\n",
+    "**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. For an example, see [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment).\n",
     "\n",
     "---\n",
     "\n",