@@ -148,10 +279,24 @@ function RankedRow({ entry, onRemeasure, onDelete, busy }) {
{entry.modelId}
- {BACKEND_LABEL[entry.backend] || entry.backend}
+ {runtimeLabel}
+ {entry.tuningLabel || 'backend defaults'}
+
+ {/* The numbers below describe SOME OTHER configuration when the launch
+ knobs never reached the daemon — say so rather than filing them
+ under the tuning that was asked for. */}
+ {entry.tuningApplied === false && entry.tuningNotApplied && (
+
+ Tuning was not applied — {entry.tuningNotApplied}. These numbers describe the configuration that was actually running.
+
+ )}
{entry.explanation}
{entry.staleness?.stale && (
@@ -171,7 +316,7 @@ function RankedRow({ entry, onRemeasure, onDelete, busy }) {
onRemeasure(entry)}
disabled={busy}
- title="Measure again"
+ title="Measure again (and adjust tuning)"
aria-label={`Measure ${entry.modelId} again`}
className="p-1.5 text-gray-400 hover:text-white transition-colors disabled:opacity-50"
>
@@ -243,11 +388,91 @@ function RankedRow({ entry, onRemeasure, onDelete, busy }) {
);
}
+/**
+ * Which launch tuning won, per model.
+ *
+ * Only rendered for models measured under two or more tunings — one reading is
+ * not a comparison, and presenting it as "the best tuning" would dress a single
+ * measurement up as a conclusion. The server enforces the same rule.
+ */
+function TuningComparison({ rows, runtimeLabelFor }) {
+ if (!rows?.length) return null;
+ return (
+
+
Tuning comparison
+
+ Throughput of each launch configuration, relative to the best one measured for that model.
+
+ {rows.map((row) => (
+
+
+ {row.modelId}
+ {runtimeLabelFor(row.backend)}
+
+ {row.variants.map((variant, index) => (
+
+
+
+ {variant.label}
+ {index === 0 && best }
+
+
+
+
+
+ {variant.charsPerSecond} chars/s
+ {variant.deltaPercent}%
+
+
+ ))}
+
+ ))}
+
+ {runtimes.map((runtime) => (
+
+ {runtime.label}
+
+ {runtime.modelCount === null ? 'unreachable' : `${runtime.modelCount} model${runtime.modelCount === 1 ? '' : 's'}`}
+
+
+ ))}
+
+ );
+}
+
export function LocalModelAssessments() {
const [intent, setIntent] = useState('balanced');
const [report, setReport] = useState(null);
const [loading, setLoading] = useState(true);
const [pendingTarget, setPendingTarget] = useState(null);
+ // Tuning for the run being set up. Kept beside `pendingTarget` rather than
+ // inside the modal so re-measuring can pre-fill it from the existing record,
+ // and so it survives the collapse/expand of the tuning section.
+ const [tuningDraft, setTuningDraft] = useState({});
// Per-sample progress for the run in flight. `null` = no frame yet, which is
// rendered as "no progress bar" rather than as 0 of N.
const [progress, setProgress] = useState(null);
@@ -309,7 +534,7 @@ export function LocalModelAssessments() {
// That is OUR abort, not a failure, so swallow it here rather than letting
// useAsyncAction toast an error the user just asked for.
const result = await runLocalLlmAssessment(
- { backend: target.backend, modelId: target.modelId },
+ { backend: target.backend, modelId: target.modelId, tuning: compactTuning(target.tuning) },
{ silent: true, signal: controller.signal },
).catch((err) => {
if (controller.signal.aborted) return { cancelled: true };
@@ -323,30 +548,49 @@ export function LocalModelAssessments() {
}, { errorMessage: 'Assessment failed' });
const confirmRun = async () => {
- const target = pendingTarget;
+ const target = { ...pendingTarget, tuning: compactTuning(tuningDraft) };
activeTargetRef.current = target;
setProgress(null);
const result = await runAssessment(target);
activeTargetRef.current = null;
setProgress(null);
setPendingTarget(null);
-// An aborted run recorded nothing on either side, so there is no verdict
- // to report — marked `cancelled` by the abort catch above, or by the server
+ setTuningDraft({});
+ // An aborted run recorded nothing on either side, so there is no verdict to
+ // report — marked `cancelled` by the abort catch above, or by the server
// when it saw the signal drop mid-run.
if (result && !result.cancelled) {
const verdict = VERDICT_META[result.verdict]?.label || result.verdict;
+ // A tuning that could not be applied means the verdict describes a
+ // different configuration — surface that at the point of the result
+ // rather than only in the row the user has to go find.
+ if (result.tuningApplied === false && result.tuningNotApplied) {
+ toast.warning(`${target.modelId}: ${verdict} — tuning not applied (${result.tuningNotApplied})`);
+ return;
+ }
toast.success(`${target.modelId}: ${verdict}`);
}
};
const [removeAssessment, removing] = useAsyncAction(async (entry) => {
- await deleteLocalLlmAssessment(entry.backend, entry.modelId, { silent: true });
- setReport((prev) => (prev ? {
- ...prev,
- ranked: prev.ranked.filter((r) => !(r.backend === entry.backend && r.modelId === entry.modelId)),
- assessments: prev.assessments.filter((a) => !(a.backend === entry.backend && a.modelId === entry.modelId)),
- unassessed: [...prev.unassessed, { backend: entry.backend, modelId: entry.modelId, params: null }],
- } : prev));
+ const tuningKey = entry.tuningKey || '';
+ await deleteLocalLlmAssessment(entry.backend, entry.modelId, tuningKey, { silent: true });
+ const isDropped = (r) => r.backend === entry.backend && r.modelId === entry.modelId && (r.tuningKey || '') === tuningKey;
+ setReport((prev) => {
+ if (!prev) return prev;
+ const assessments = prev.assessments.filter((a) => !isDropped(a));
+ // The model only returns to "not yet measured" when its LAST tuning is
+ // gone — dropping one of several still leaves evidence for it.
+ const stillMeasured = assessments.some((a) => a.backend === entry.backend && a.modelId === entry.modelId);
+ return {
+ ...prev,
+ ranked: prev.ranked.filter((r) => !isDropped(r)),
+ assessments,
+ unassessed: stillMeasured
+ ? prev.unassessed
+ : [...prev.unassessed, { backend: entry.backend, modelId: entry.modelId, params: null }],
+ };
+ });
return true;
}, { errorMessage: 'Could not discard that measurement' });
@@ -357,6 +601,15 @@ export function LocalModelAssessments() {
activeTargetRef.current = null;
setProgress(null);
setPendingTarget(null);
+ setTuningDraft({});
+ };
+
+ // Re-measuring starts from the tuning that produced the existing record, so
+ // "run it again" reproduces the same configuration by default and adjusting
+ // one knob is a one-field edit rather than re-entering the whole set.
+ const openTarget = (entry) => {
+ setTuningDraft(entry?.tuning && typeof entry.tuning === 'object' ? { ...entry.tuning } : {});
+ setPendingTarget(entry);
};
const busy = running || removing;
@@ -383,10 +636,14 @@ export function LocalModelAssessments() {
The install catalog estimates fit from a model's file size. This measures it: one short
generation at each of several context lengths, recording throughput, time to first token, and how
- far throughput falls off as context grows. Results stay on this machine — they describe this
- hardware, so they are never synced to a peer.
+ far throughput falls off as context grows — across every local runtime PortOS can reach
+ (Ollama, LM Studio, llama.cpp, MTPLX, vLLM). Measure a model under more than one launch tuning to
+ see which configuration this machine actually prefers. Results stay on this machine — they
+ describe this hardware, so they are never synced to a peer.
+
Rank for
0 && (
)}
@@ -423,10 +680,11 @@ export function LocalModelAssessments() {
{report.ranked.map((entry) => (
))}
@@ -442,8 +700,13 @@ export function LocalModelAssessments() {
Measured, but not recommended
{report.excluded.map((entry) => (
-
+
{entry.modelId}
+ {/* Several rows can name the same model — one per tuning — so
+ the configuration is what tells them apart. */}
+
+ {entry.tuningLabel || 'backend defaults'}
+
{entry.reason} }
@@ -451,6 +714,11 @@ export function LocalModelAssessments() {
)}
+
backendLabel(report, id)}
+ />
+
{report.unassessed?.length > 0 && (
@@ -464,10 +732,10 @@ export function LocalModelAssessments() {
{entry.modelId}
- {BACKEND_LABEL[entry.backend] || entry.backend}
+ {backendLabel(report, entry.backend)}
setPendingTarget(entry)}
+ onClick={() => openTarget(entry)}
disabled={busy}
className="flex items-center gap-1 px-2 py-1 text-[11px] rounded border border-port-border text-gray-300 hover:border-port-accent hover:text-white transition-colors disabled:opacity-50 shrink-0"
>
@@ -483,7 +751,11 @@ export function LocalModelAssessments() {
({
deleteLocalLlmAssessment: vi.fn(),
}));
-vi.mock('../ui/Toast', () => ({ default: { success: vi.fn(), error: vi.fn() } }));
+vi.mock('../ui/Toast', () => ({ default: { success: vi.fn(), error: vi.fn(), warning: vi.fn() } }));
// Per-sample run progress arrives on the shared `localLlm:progress` socket
// event; the tests below replay frames through the registered handler.
@@ -31,9 +31,26 @@ const report = (overrides = {}) => ({
readError: null,
ranked: [],
excluded: [],
+ runtimes: RUNTIMES,
+ tuningComparison: [],
+ uninstalled: [],
...overrides,
});
+// The runtime roster is server-derived — label, reachability, and the knob
+// catalog all ride on the report, so the panel has no hardcoded backend list to
+// drift from.
+const RUNTIMES = [
+ { id: 'ollama', label: 'Ollama', managed: true, modelCount: 1, error: null, tuningSpecs: [
+ { id: 'numCtx', label: 'Context size', type: 'number', applies: 'request', min: 512, max: 1048576, unit: 'tokens', hint: 'Sent with the request.' },
+ ] },
+ { id: 'llama', label: 'llama.cpp', managed: false, modelCount: 3, error: null, tuningSpecs: [
+ { id: 'ubatchSize', label: 'Micro-batch size', type: 'number', applies: 'launch', min: 1, max: 8192, hint: 'Physical micro-batch.' },
+ { id: 'flashAttn', label: 'Flash attention', type: 'boolean', applies: 'launch', hint: 'Fused attention kernel.' },
+ ] },
+ { id: 'mtplx', label: 'MTPLX', managed: false, modelCount: null, error: 'not reachable at http://127.0.0.1:8000/v1 (ECONNREFUSED)', tuningSpecs: [] },
+];
+
const rankedEntry = (overrides = {}) => ({
backend: 'ollama',
modelId: 'example-model:7b',
@@ -76,7 +93,7 @@ describe('LocalModelAssessments', () => {
render(
Media models
-
Local LLM settings
+
Manage LLMs
, activeTab: string, ariaLabel: string }} props
+ */
+export default function RouteTabsHeader({ tabs, activeTab, ariaLabel }) {
+ const navigate = useNavigate();
+
+ const handleChange = (tabId) => {
+ const target = tabs.find((t) => t.id === tabId);
+ if (target) navigate(target.to);
+ };
+
+ return (
+
-
+
@@ -469,6 +470,11 @@ export default function LocalLlmPlayground() {
+ {/* This page keeps its own `/local-llm/playground` path (it predates the
+ Models section and lives in ⌘K history), so without the section's tab
+ bar arriving here would strand the user outside it. */}
+
+
diff --git a/client/src/pages/Models.jsx b/client/src/pages/Models.jsx
new file mode 100644
index 000000000..80d287237
--- /dev/null
+++ b/client/src/pages/Models.jsx
@@ -0,0 +1,53 @@
+import { useParams, Navigate } from 'react-router';
+import { Cpu } from 'lucide-react';
+import PageHeader from '../components/PageHeader';
+import ModelsTabsHeader from '../components/models/ModelsTabsHeader';
+import MemoryManagement from '../components/settings/MemoryManagement.jsx';
+import LocalModelAssessments from '../components/settings/LocalModelAssessments.jsx';
+import { LocalLlmTab } from '../components/settings/LocalLlmTab';
+
+/**
+ * Models — the top-level home for everything about the models this machine runs.
+ *
+ * Three tabs, each of which was previously a card buried in
+ * `/settings/local-llm`:
+ *
+ * - **LLMs** — backends, the install catalog, the llama.cpp launcher.
+ * - **Performance** — measured assessments and launch-tuning comparison.
+ * - **Status** — what is resident in memory right now.
+ *
+ * `?tab` is a route param, not local state, so every one is deep-linkable and
+ * reachable from ⌘K and voice (`client/src/CLAUDE.md`).
+ *
+ * Only LLM model management lives here so far. Media models (LoRAs, image/video
+ * checkpoints, embeddings) are tracked for the same treatment in #4728; the
+ * design record is `docs/plans/2026-08-21-models-navigation.md`.
+ */
+const TAB_CONTENT = {
+ llms: LocalLlmTab,
+ performance: LocalModelAssessments,
+ status: MemoryManagement,
+};
+
+export default function Models() {
+ const { tab } = useParams();
+ // An unknown slug lands on Performance rather than rendering a blank page —
+ // it is the tab that answers "which model should I use?", which is what most
+ // people arrive here for.
+ const activeTab = tab && TAB_CONTENT[tab] ? tab : null;
+ if (!activeTab) return memory panel
}));
+vi.mock('../components/settings/LocalModelAssessments.jsx', () => ({ default: () => assessments panel
}));
+vi.mock('../components/settings/LocalLlmTab', () => ({ LocalLlmTab: () => llms panel
}));
+
+import Models from './Models';
+
+const renderAt = (path) => render(
+
+
+
+
+);
+
+describe('Models', () => {
+ it.each([
+ ['/models/performance', 'assessments panel'],
+ ['/models/status', 'memory panel'],
+ ['/models/llms', 'llms panel'],
+ ])('renders %s from the route param, not from local state', (path, expected) => {
+ renderAt(path);
+ expect(screen.getByText(expected)).toBeInTheDocument();
+ });
+
+ // A stale ⌘K entry or a typo must not produce a blank page — Performance is
+ // the tab that answers "which model should I use?", which is why people land
+ // here at all.
+ it('redirects an unknown tab slug to Performance', () => {
+ renderAt('/models/not-a-tab');
+ expect(screen.getByText('assessments panel')).toBeInTheDocument();
+ });
+
+ it('offers every Models destination in the sub-nav', () => {
+ renderAt('/models/performance');
+ for (const label of ['LLMs', 'Performance', 'Playground', 'Status']) {
+ expect(screen.getByRole('tab', { name: label })).toBeInTheDocument();
+ }
+ });
+});
diff --git a/client/src/pages/Settings.jsx b/client/src/pages/Settings.jsx
index 56287d240..d7d1700b4 100644
--- a/client/src/pages/Settings.jsx
+++ b/client/src/pages/Settings.jsx
@@ -9,7 +9,6 @@ import { CatalogTypesTab } from '../components/settings/CatalogTypesTab';
import CodeReviewersTab from '../components/settings/CodeReviewersTab';
import { DatabaseTab } from '../components/settings/DatabaseTab';
import EmbeddingsTab from '../components/settings/EmbeddingsTab';
-import { LocalLlmTab } from '../components/settings/LocalLlmTab';
import { TelegramTab } from '../components/settings/TelegramTab';
import { GeneralTab } from '../components/settings/GeneralTab';
import { MortalLoomTab } from '../components/settings/MortalLoomTab';
@@ -48,7 +47,6 @@ export default function Settings() {
case 'code-reviewers': return ` | Install one runtime from the installer's fixed table, streaming installer output as SSE. Rejects any id not in the table. |
| GET | `/providers/readiness` | Requirements checklist per provider backed by a LOCAL daemon (llama.cpp / Ollama / LM Studio / MTPLX), keyed by provider id: is the daemon installed, is it answering at the endpoint THIS provider points at, and is it serving the provider's default model. Each entry also carries `setup` — what the one-click fix below can do about the unmet checks (`null` when nothing is auto-fixable here). Providers with no local dependency are absent from the map. Complements `/providers/runtimes` (which answers "can PortOS run this CLI?"). Booleans, labels, and the provider's own endpoint only — never a resolved binary path. Skips disabled providers; 15s endpoint-probe cache (one probe per distinct endpoint), 60s binary-PATH cache, both dropped by the llama-server start/stop/install routes. |
| POST | `/providers/readiness/setup?provider=` | Install and/or start the LOCAL DAEMON that provider points at (llama.cpp / Ollama / LM Studio / MTPLX), streaming progress as SSE — the "do it for me" half of `/providers/readiness`, so an unmet requirement is fixed from the card instead of from a vendor setup doc. The request names a PROVIDER only: the runtime kind and endpoint are re-derived server-side from the stored record, so no query value reaches a spawn argument (an optional `runtime=` is cross-checked and 409s on a mismatch). Every command comes from a fixed per-runtime table. Never downloads model weights, never starts llama-server (it needs a checkpoint you choose), and never runs MTPLX's privileged fan-control helper. Single-flight. |
diff --git a/docs/PORTS.md b/docs/PORTS.md
index c32f71faf..063531aba 100644
--- a/docs/PORTS.md
+++ b/docs/PORTS.md
@@ -34,7 +34,7 @@ Common port labels:
| 5559 | portos-autofixer | api | Autofixer daemon API |
| 5560 | portos-autofixer-ui | ui | Autofixer web UI |
| 5561 | portos-db (Docker container) | - | Infrastructure dependency: PostgreSQL Docker container provisioned by `scripts/setup-db.js` / Docker Compose (not a PM2 process in `server/services/apps.js`; native mode uses system pg on 5432). |
-| 5568 | llama-server | - | Loopback speculative-decoding server managed from Settings → Local LLM |
+| 5568 | llama-server | - | Loopback speculative-decoding server managed from Models → LLMs |
| 18020 | vLLM (Docker) | - | Loopback vLLM Qwen3.8-27B / DFlash 2 container on an RTX 3090 host. Operator-started (`docker compose --profile single up -d`) — PortOS never brings it up on boot. See [features/qwen38-rtx3090.md](./features/qwen38-rtx3090.md). |
## How `:5555`, `:5553`, and `:5554` Relate
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md
index 6f4889599..6267f886a 100644
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@@ -139,7 +139,7 @@ Two caveats:
`ollama serve` runs from its own unit file, so when a window is configured
PortOS starts (or restarts) Ollama itself.
-The Local LLM settings card shows the window loaded models are actually running
+The Models → LLMs page shows the window loaded models are actually running
at, and flags it when it's below what an agent harness needs.
## Chief of Staff Issues
diff --git a/docs/features/dflash2.md b/docs/features/dflash2.md
index cf1f53de7..bb6d48071 100644
--- a/docs/features/dflash2.md
+++ b/docs/features/dflash2.md
@@ -19,7 +19,7 @@ PortOS integrates both through the **OpenCode llama TUI** provider preset and th
2. **Model Refresh**:
- Support for dynamic model discovery via the **Refresh Models** button on AI Providers, querying the local `llama-server` `/v1/models` endpoint.
3. **Local LLMs & AI Providers Guidance**:
- - UI instructions, command templates, and copyable run lines surfaced in **Settings → Local LLMs** and **AI Providers**.
+ - UI instructions, command templates, and copyable run lines surfaced in **Models → LLMs** and **AI Providers**.
---
@@ -27,7 +27,7 @@ PortOS integrates both through the **OpenCode llama TUI** provider preset and th
### 1. Download Base & Draft Models
-**From the UI (recommended).** **Settings → Local LLMs → Speculative Decoding**
+**From the UI (recommended).** **Models → LLMs → Speculative Decoding**
lists each preset's two GGUFs with their on-disk state and a **Download** button
per file — PortOS fetches the weights from Hugging Face straight into the path
the launcher passes `llama.cpp`, so a missing file is visible (and fixable)
@@ -126,7 +126,7 @@ can't fight your choice:
Vocabulary reference: llama.cpp `docs/speculative.md`.
### 3. Use in PortOS
-1. Navigate to **AI Providers** (`/ai`) or **Settings → Local LLMs**.
+1. Navigate to **AI Providers** (`/ai`) or **Models → LLMs**.
2. Verify **OpenCode llama TUI** is enabled.
3. Click **Refresh Models** to pull the live aliases from `llama-server`, or use the default `dflash` model.
4. Select **OpenCode llama TUI** in the CoS task creator or terminal runner to execute coding and agent tasks with speculative acceleration.
@@ -171,7 +171,7 @@ every 20s:
asking for `dspark` fails here rather than inside a dead agent run.
Until all three pass, the card says what is missing and links to
-**Settings → Local LLM**. The same failure previously surfaced only as
+**Models → LLMs**. The same failure previously surfaced only as
`Cannot connect to API: Unable to connect` inside the agent transcript.
The GGUF weights are a separate download from the binary: `llama-server` will
diff --git a/docs/plans/2026-08-21-models-navigation.md b/docs/plans/2026-08-21-models-navigation.md
new file mode 100644
index 000000000..30a2c3bed
--- /dev/null
+++ b/docs/plans/2026-08-21-models-navigation.md
@@ -0,0 +1,127 @@
+# Models navigation + multi-runtime measured assessments
+
+**Date:** 2026-08-21
+
+## Context
+
+Two problems, one shape.
+
+**1. Measured assessments only knew two runtimes.** `services/localModelAssessments.js`
+could measure a model on Ollama or LM Studio and nothing else, because the only
+measurement path it had was `runLocalLlmTest`, which resolves a configured PortOS
+*provider*. Meanwhile PortOS already knows how to reach three more local daemons
+— llama.cpp (`llama-server`, which PortOS itself launches), MTPLX, and vLLM — all
+speaking the same OpenAI-compatible wire protocol. Those were exactly the
+runtimes whose performance is most sensitive to how they were *launched*, and the
+feature had nothing to say about them.
+
+**2. The assessment recorded no configuration.** A throughput number for a GGUF
+is meaningless without the launch line that produced it: the same model on the
+same machine streams at wildly different rates depending on the micro-batch size,
+whether flash attention is on, and how much of the KV cache is quantized. Two
+readings of one model looked like noise when they were actually two different
+setups, and there was no way to ask "is `-ub 512` worth it here?".
+
+**3. Model management was a scroll position.** Memory residency, measured
+assessments, backend install/switch, the llama.cpp launcher, and the install
+catalog were all cards stacked on `/settings/local-llm` — one long page, none of
+it individually linkable, all of it filed under Settings even though "which model
+should I run?" is not a settings question.
+
+## Design
+
+### Runtimes
+
+`ASSESSABLE_RUNTIMES` (`server/lib/localProviderRuntime.js`) is the roster:
+`ollama`, `lmstudio`, `llama`, `mtplx`, `vllm`. The split is about *how PortOS
+reaches the model*, not about measurement quality:
+
+| Kind | Runtimes | Model list | Measurement |
+| --- | --- | --- | --- |
+| Managed | ollama, lmstudio | `listModels()` — a durable catalog on disk | `runLocalLlmTest` (provider path, lands in `/runs`) |
+| Endpoint | llama, mtplx, vllm | `GET /v1/models` on the live daemon | `runEndpointLlmTest` — direct, no `/runs` record |
+
+The distinction matters for the sentinel contract. A managed backend's list
+survives the daemon being down. An endpoint runtime has no catalog at all, so a
+stopped daemon means *the list could not be read* — an error, never an empty
+catalog. Reporting "0 models" for a daemon the user only needs to start would
+hide every model behind it.
+
+The SSE read loop moved to `server/lib/openAiChatStream.js` so both paths share
+one implementation of the reasoning-channel handling, the skip-a-malformed-frame
+rule, and partial-output-on-abort.
+
+### Tuning
+
+`server/lib/localModelTuning.js` holds the knob catalog. Every knob declares what
+PortOS can actually **do** with it:
+
+- `launch` — PortOS puts it on the daemon's command line (llama.cpp only:
+ `-b`, `-ub`, `-t`, `--flash-attn`, `--cache-type-k|v`, `--draft-max`).
+- `request` — sent with each measurement request (Ollama's `num_ctx`).
+- `record` — PortOS cannot set it; the user states how the daemon was launched so
+ two readings stay comparable (LM Studio, MTPLX, vLLM).
+
+A `record` knob is not a lie by omission — it changes nothing about the run and
+the UI says so. What it must never do is claim to have been applied, which is why
+the `applies` axis is on the spec rather than implied, and why a `request` knob
+must also declare the wire field name it maps to.
+
+`relaunchLlamaServerWithTuning` is the launch half: it reads the running config,
+merges the requested knobs, and restarts llama-server under PM2. It **refuses
+rather than guesses** when nothing is running (no model path to reuse) or when
+the process was started outside PortOS (stopping it would kill something the user
+owns) — returning `{ applied: false, reason }` so the run can still measure
+whatever is actually serving and record that the tuning was NOT applied. A
+reading taken under a tuning PortOS could not apply must never be filed as
+evidence for that tuning.
+
+### Store identity
+
+`assessmentKey(backend, modelId, tuningKey)` returns `backend:modelId` when the
+tuning signature is empty and `backend:modelId@` otherwise. That is
+deliberate: an untuned run keys byte-identically to the pre-tuning key, so every
+record already on disk keeps resolving with **no migration**, while a tuned run
+of the same model lands beside it instead of overwriting it. Two tunings are two
+answers to two different questions; re-running the *same* tuning still
+supersedes, because a stale reading of one configuration is worse than none.
+
+`compareTunings` groups by (backend, model) and reports each variant's throughput
+as a percentage of the winner. Models measured under only one tuning are omitted
+— one reading is not a comparison, and presenting it as "the best tuning" would
+dress a single measurement up as a conclusion.
+
+### Navigation
+
+A top-level **Models** section, with `/models/:tab`:
+
+- **LLMs** (`/models/llms`) — backends, install catalog, llama.cpp launcher.
+- **Performance** (`/models/performance`) — measured assessments + tuning comparison.
+- **Status** (`/models/status`) — what is resident in memory right now.
+- **Playground** (`/local-llm/playground`) — unchanged path, now listed here.
+
+`/settings/local-llm` redirects to `/models/llms` so bookmarks and stale ⌘K
+history keep working, and `Local LLMs` is gone from the Settings sub-nav. Each
+tab is a route param, so all four are deep-linkable and reachable from ⌘K and
+voice (`NAV_COMMANDS` gained `nav.models.performance` and `nav.models.status`;
+`nav.settings.local-llm` keeps its opaque id and moves to the `Models` section).
+
+## Verification
+
+- `server/lib/localModelTuning.test.js` — knob catalog invariants (every knob
+ declares `applies`; every `request` knob declares its wire name; only llama.cpp
+ carries launch knobs), normalize/clamp/coerce, signature stability, and the
+ comparison rules.
+- `server/services/localModelAssessments.test.js` — endpoint-runtime listing and
+ measurement, endpoint resolution from the live llama-server, two tunings
+ coexisting, same-tuning replacement, per-tuning delete, `tuningApplied: false`
+ recording, and the runtime roster's `null`-vs-`0` model count.
+- Full server (1540 files) and client (700 files) suites pass.
+
+## Follow-up
+
+Only LLM model management lives under Models so far. Media models — LoRAs,
+image/video checkpoints (`/media/models`), Three.js meshes, embeddings — are
+scattered across Create and Settings and belong in the same section. Tracked in
+[#4728](https://github.com/atomantic/PortOS/issues/4728) so this change stays
+reviewable.
diff --git a/server/lib/README.md b/server/lib/README.md
index c72a0735e..03343a4b2 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -139,6 +139,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `mtplxModels.js` | `listMtplxCachedModels({command?})` → `{models, error}` from `mtplx models --json` (a local directory listing — no network, no model load) and `pickMtplxCachedModel(models)` → the repo id to hand `mtplx serve --model`. `models: null` means the cache could not be READ (no binary, command failed, unparseable) and is deliberately distinct from `[]` (read, and empty), because `services/localRuntimeSetup.js` starts MTPLX on its own default in the first case and refuses with the `mtplx pull` command in the second. Exists because `mtplx serve` defaults `--model` to one hard-coded checkpoint and exits 1 before binding when that repo is not cached — even on a host holding a different MTP model that serves fine. Picks only entries MTPLX itself calls complete (`validation.ok !== false`, so a half-finished pull is not served), preferring one with a recorded `mtplx_runtime.json` exactness contract. |
| `vllmQwenProject.js` | `resolveVllmProjectDir()` / `inspectVllmQwenProject()` → `{dir, hasProject, composeFile, hasWeights, weightsRoot}` for the operator-cloned syv-ai/qwen38-27b-rtx3090 compose project, plus `vllmStartBlockedReason(project)` → the prose refusal (or `null`). Directory reads only — never runs docker or touches a registry. `hasWeights` is tri-state: `true` found / `false` caches read and empty / `null` no cache readable (a docker-volume cache is invisible from a native-Win32 PortOS), and `services/localRuntimeSetup.js` refuses to `docker compose up` on anything but `true` so the start button can never kick off the ~20 GB prepare. Overrides: `VLLM_QWEN_PROJECT_DIR`, `VLLM_QWEN_WEIGHTS_DIR`. |
| `openAiModelsProbe.js` | `probeOpenAiModels(baseUrl, { timeoutMs, apiKey })` → `{ reachable, models, error }` — the one `GET {base}/models` probe for the local OpenAI-compatible daemons, shared by `services/providerReadiness.js` and `services/llamaServerManager.js`. Distinguishes unreachable from reachable-but-unlistable (`models: null`) from up-with-nothing-loaded (`[]`), names the real transport failure via `describeFetchError` (undici reports every one as a bare `fetch failed`), and cancels an unread body on a non-OK response. `apiKey` attaches a Bearer header for a key-gated daemon (vLLM's compose stack), and a 401/403 answers `reachable: true` with `error: 'authentication required'` — a server that refused the request is definitively running, and calling it unreachable would send the user to start it again. Consolidated after the two copies drifted — one passed its timeout as a `timeout` key inside the fetch init object, where it is not an option, silently running a 500ms poll loop on the 15s default. |
+| `openAiChatStream.js` | `streamOpenAiChat({ endpoint, apiKey, model, messages, temperature, maxTokens, extraBody, signal, onChunk })` → the streamed text — one streaming `POST {base}/chat/completions` against any local OpenAI-compatible daemon, plus `buildMessages`, `extractStreamDelta` (skips a malformed SSE frame instead of aborting the stream) and `resolvePartialOutput` (content, else reasoning, else `''`). Sibling of `openAiModelsProbe.js`. Shared by `services/localLlmPlayground.js` (provider-backed runs with a `/runs` record) and its `runEndpointLlmTest` (a bare loopback daemon PortOS holds no provider record for, which is how `services/localModelAssessments.js` measures llama.cpp / MTPLX / vLLM). An abort mid-stream throws with `.partialOutput` carrying what already streamed. |
| `cliChildEnv.js` | The one place the AI-CLI child environment is composed, replacing the hand-rolled copy every spawn site carried — which made each env-level fix an N-file sweep (#3194). `buildCliChildEnv({ baseEnv, before, provider, model, cwd, extra, guard })` returns a COMPLETE env for `spawn`: layers `baseEnv → before → Ollama-Claude defaults → provider.envVars → buildOpencodeEnvVars → extra`, pins `PWD` to `cwd`, strips `CLAUDECODE`, and (with `guard: true`) prepends the pm2 guard shim onto the final `PATH`. The Ollama-Claude layer raises Claude Code's default output ceiling to 65,536 tokens so a thinking-capable local model cannot finish its reasoning past the stock 32K ceiling and die before its final tool call; an explicit provider env value wins. `composeProviderEnv({ before, provider, model, extra })` returns just the ordered provider layers, for sites that build a DELTA someone else bases and spawns (the CoS runner payload, a shell-session overlay). The two slots are not interchangeable: `before` sits UNDER `provider.envVars` (forgeTokenEnv/claudeSettingsEnv, so a provider override still wins), `extra` sits OVER it (TERM/COLORTERM for a PTY). `cliChildEnv.test.js` asserts the composed order per call site and **discovers** any new site that hand-rolls the tuple instead of calling these — so the call-site list stays in the test, not in prose here. |
| `cliProviderArgs.js` | Per-CLI argv conventions (`buildCliArgs`) for stdin prompt delivery — dependency-light extraction from runner.js so out-of-process callers (autofixer) can import it. |
| `cliProviderRun.js` | One-shot CLI provider invocation (`pickCliProvider` + `runCliProviderPrompt`) — lightweight path for the autofixer + calendar MCP sync to honor the configured provider/model. |
@@ -313,6 +314,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `ollamaContext.js` | Ollama runtime context-window reasoning. `resolveOllamaContextLength(provider, env)` → the window PortOS should hold the daemon at (`provider.numCtx`, then `OLLAMA_CONTEXT_LENGTH`, then `null` = leave Ollama's VRAM-based auto-pick alone); `withOllamaContextEnv(env, n)` builds the `ollama serve` child env; `parseOllamaContextOverflow(text)` recognizes the `exceed_context_size_error` rejection and pulls `{ promptTokens, contextLength }` out of either the JSON body or the human message; `describeOllamaContextOverflow` / `describeOllamaContextTooSmall` render the actionable one-liners; `isSameOllamaDaemon(providerBase, managedBase)` compares host+port so a provider pointed at a *remote* Ollama never triggers a local-daemon reload; `OLLAMA_AGENT_MIN_CONTEXT` is the window below which an agent harness is warned. Pure — the daemon side lives in `services/ollamaManager.js` (`ensureContextWindow`), the pre-spawn enforcement in `services/ollamaAgentContext.js`. |
| `localModelHeuristics.js` | Capability heuristics for untyped local (Ollama/LM Studio) models. `isEmbeddingModel`/`isGenerationModel` (so a generation/fallback run never picks an embedding model like `nomic-embed-text`); `isVisionModel(model)` (string id or model card — prefers explicit `type:'vlm'`/`capabilities:['vision']` metadata, falls back to id regex; used by the LoRA captioner); `recommendEditorialModel(models, { measured })` ranks installed models for editorial review/editing, dropping any a fresh measurement proved cannot run here. Pure. Mirror `isEmbeddingModel`/`isVisionModel` in `client/src/utils/providers.js`. |
| `localModelAssessment.js` | Scoring for MEASURED local-model assessments (recorded by `services/localModelAssessments.js`): `classifyFitVerdict` (`fits`/`does-not-fit`/`incompatible`/`unknown`), `summarizePerformance` (mean/peak chars-per-second, TTFT, max working context, cross-context degradation), `scoreAssessment` + `scoreForIntent` + `rankByIntent` for the balanced/smartest/fastest/lightweight pickers, `explainAssessment`, `compareEnvironments`/`describeStaleness` (is a stored reading still valid for THIS machine?), and `measuredFitVerdict`/`reconcileFit` (fold a measurement into the catalog's size-estimate fit badge, keeping the disagreement). Pure. `null` strictly means NOT MEASURED — never 0, never "failed". |
+| `localModelTuning.js` | Launch/runtime tuning knobs for the measurable local runtimes: `TUNING_SPECS` (per runtime — llama.cpp's `-b`/`-ub`/`-t`/`--flash-attn`/`--cache-type-k|v`/`--draft-max`, Ollama's `num_ctx`, plus the record-only knobs for LM Studio / MTPLX / vLLM), each declaring `applies: 'launch' \| 'request' \| 'record'` so the UI can never claim PortOS set something it cannot. `normalizeTuning` (coerce + clamp + drop unknown keys), `tuningSignature` (stable identity — `''` for backend defaults, so pre-tuning store keys keep resolving), `describeTuning`, `launchTuning`, `requestBody` (request knobs under their wire names), and `compareTunings` (which tuning won per model, with a `deltaPercent` against the winner). Pure. |
| `loraDataset.js` | Pure helpers for character LoRA training datasets (`data/lora-datasets/`): `sanitizeLoraDataset` (collectionStore sanitizer), `deriveTriggerWord` (name → single-token slug with collision suffix), `prefixCaption` (idempotent trigger-word prefixing), `buildVariationMatrix` (deterministic view/pose/expression/outfit tuples for batch generation), `computeDatasetReadiness` (trainable gate: ≥`MIN_TRAINING_IMAGES` ready+captioned images, plus an advisory `recommended`/`quality` tier via `datasetQualityTier` nudging toward `RECOMMENDED_TRAINING_IMAGES`), `analyzeCaptionInvariants` (flags identity fragments repeated across ≥`INVARIANT_SHARE_THRESHOLD` of captions — those bind to the caption phrases instead of the trigger token, issue #1320) + `stripSharedFragments` (rewrite one caption with those fragments removed, trigger preserved). Prompt building + I/O live in `services/loraDatasetGenerate.js` / `services/loraDatasets.js`. |
| `issueLength.js` | Per-issue size targets fed into text stages. |
| `musicDuration.js` | Lyric-aware MiniMax Music 3 duration analysis and ending-cushioned auto-duration recommendation; mirrors `client/src/lib/musicDuration.js`. |
diff --git a/server/lib/index.js b/server/lib/index.js
index 68525dee0..c28d34db7 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -155,6 +155,7 @@ export * from './localProviderRuntime.js';
export * from './mtplxModels.js';
export * from './vllmQwenProject.js';
export * from './openAiModelsProbe.js';
+export * from './openAiChatStream.js';
// `runners.js` re-defines `isFlux2`/`isZImage`/`isErnie` that also live in
// mediaModels.js — namespace it so the barrel surface is unambiguous.
export * as runners from './runners.js';
@@ -291,6 +292,7 @@ export * from './localLlmDisk.js';
export * from './specDecodePresets.js';
export * from './localModelHeuristics.js';
export * from './localModelAssessment.js';
+export * from './localModelTuning.js';
export * from './ollamaContext.js';
export * from './loraDataset.js';
export * from './issueLength.js';
diff --git a/server/lib/localModelAssessment.js b/server/lib/localModelAssessment.js
index 9d9bacadb..ec0777396 100644
--- a/server/lib/localModelAssessment.js
+++ b/server/lib/localModelAssessment.js
@@ -308,6 +308,8 @@ export function rankByIntent(assessments, intent = 'balanced') {
excluded.push({
backend: assessment?.backend || null,
modelId: assessment?.modelId || null,
+ tuningKey: assessment?.tuningKey || '',
+ tuningLabel: assessment?.tuningLabel || null,
verdict,
reason: assessment?.verdictReason || null,
});
@@ -319,6 +321,8 @@ export function rankByIntent(assessments, intent = 'balanced') {
excluded.push({
backend: assessment?.backend || null,
modelId: assessment?.modelId || null,
+ tuningKey: assessment?.tuningKey || '',
+ tuningLabel: assessment?.tuningLabel || null,
verdict,
reason: 'ran, but no axis of this intent was measured',
});
@@ -343,6 +347,19 @@ export function rankByIntent(assessments, intent = 'balanced') {
// `null` = the caller did not annotate staleness, which is not the same as
// "compared and current".
staleness: assessment?.staleness || null,
+ // The tuning IS part of this row's identity, not decoration. A model can
+ // hold several measurements, one per launch configuration, and the
+ // consumer uses these to key the row, to delete THIS measurement rather
+ // than the backend-defaults one, and to pre-fill a re-measure with the
+ // configuration that produced it. Dropping them collapsed every variant
+ // onto the default record.
+ tuningKey: assessment?.tuningKey || '',
+ tuning: assessment?.tuning || {},
+ tuningLabel: assessment?.tuningLabel || null,
+ // `null` = nothing was settable (see runAssessment); `false` means these
+ // numbers describe some OTHER configuration, which the row has to say.
+ tuningApplied: assessment?.tuningApplied ?? null,
+ tuningNotApplied: assessment?.tuningNotApplied || null,
explanation: explainAssessment(assessment, resolvedIntent),
});
}
@@ -361,7 +378,10 @@ export function rankByIntent(assessments, intent = 'balanced') {
(isStale(a) - isStale(b))
|| (b.score - a.score)
|| (b.coverage - a.coverage)
- || String(a.modelId).localeCompare(String(b.modelId)));
+ || String(a.modelId).localeCompare(String(b.modelId))
+ // Two tunings of ONE model tie on model id, so the signature is what makes
+ // their order stable across reloads.
+ || String(a.tuningKey).localeCompare(String(b.tuningKey)));
return { intent: resolvedIntent, ranked, excluded };
}
diff --git a/server/lib/localModelAssessment.test.js b/server/lib/localModelAssessment.test.js
index 2ab3baea4..7d5fd9765 100644
--- a/server/lib/localModelAssessment.test.js
+++ b/server/lib/localModelAssessment.test.js
@@ -228,6 +228,72 @@ describe('explainAssessment', () => {
});
});
+// The tuning IS the row's identity when a model holds several measurements. The
+// consumer keys the row on it, deletes THIS measurement with it, and pre-fills a
+// re-measure from it — so a projection that drops it collapses every variant
+// onto the backend-defaults record (wrong row deleted, re-measure loses its
+// settings, duplicate React keys).
+describe('rankByIntent — tuning identity', () => {
+ const tuned = (tuningKey, tuning, tuningLabel, charsPerSecond) => ({
+ backend: 'llama',
+ modelId: 'example-7b',
+ verdict: 'fits',
+ params: '7B',
+ tuningKey,
+ tuning,
+ tuningLabel,
+ performance: { meanCharsPerSecond: charsPerSecond, contextDegradation: 0.9, maxWorkingContextTokens: 16384 },
+ environment: { memoryBudgetGb: 64 },
+ residentGb: 5,
+ });
+
+ it('carries the tuning through to every ranked row', () => {
+ const { ranked } = rankByIntent([
+ tuned('', {}, null, 90),
+ tuned('ubatchSize=512', { ubatchSize: 512 }, 'Micro-batch size 512', 120),
+ ], 'fastest');
+ expect(ranked).toHaveLength(2);
+ expect(ranked.map((r) => r.tuningKey).sort()).toEqual(['', 'ubatchSize=512']);
+ const fastest = ranked[0];
+ expect(fastest.tuningKey).toBe('ubatchSize=512');
+ expect(fastest.tuning).toEqual({ ubatchSize: 512 });
+ expect(fastest.tuningLabel).toBe('Micro-batch size 512');
+ });
+
+ it('gives two tunings of one model distinct identities, not one collapsed row', () => {
+ const { ranked } = rankByIntent([
+ tuned('', {}, null, 90),
+ tuned('ubatchSize=512', { ubatchSize: 512 }, 'Micro-batch size 512', 120),
+ ], 'fastest');
+ const keys = ranked.map((r) => `${r.backend}:${r.modelId}@${r.tuningKey}`);
+ expect(new Set(keys).size).toBe(2);
+ });
+
+ it('reports that a tuning was not applied so the row can say the numbers are another config', () => {
+ const entry = tuned('ubatchSize=512', { ubatchSize: 512 }, 'Micro-batch size 512', 120);
+ const { ranked } = rankByIntent([
+ { ...entry, tuningApplied: false, tuningNotApplied: 'llama-server is not running' },
+ ], 'fastest');
+ expect(ranked[0].tuningApplied).toBe(false);
+ expect(ranked[0].tuningNotApplied).toBe('llama-server is not running');
+ });
+
+ it('breaks a model-id tie on the tuning so the order is stable across reloads', () => {
+ const a = tuned('a=1', {}, 'A', 100);
+ const b = tuned('b=2', { }, 'B', 100);
+ expect(rankByIntent([b, a], 'fastest').ranked.map((r) => r.tuningKey)).toEqual(['a=1', 'b=2']);
+ expect(rankByIntent([a, b], 'fastest').ranked.map((r) => r.tuningKey)).toEqual(['a=1', 'b=2']);
+ });
+
+ it('carries the tuning onto an excluded row too, so variants stay distinguishable there', () => {
+ const { excluded } = rankByIntent([
+ { ...tuned('ubatchSize=512', { ubatchSize: 512 }, 'Micro-batch size 512', 120), verdict: 'does-not-fit' },
+ ], 'fastest');
+ expect(excluded[0].tuningKey).toBe('ubatchSize=512');
+ expect(excluded[0].tuningLabel).toBe('Micro-batch size 512');
+ });
+});
+
describe('rankByIntent', () => {
const assessment = (modelId, verdict, performance, extra = {}) => ({
backend: 'ollama',
@@ -263,7 +329,7 @@ describe('rankByIntent', () => {
const { ranked, excluded } = rankByIntent(models, 'balanced');
expect(ranked.map((r) => r.modelId)).toEqual(['example-model:7b']);
expect(excluded).toEqual([
- { backend: 'ollama', modelId: 'example-model:70b', verdict: 'does-not-fit', reason: 'out of memory' },
+ { backend: 'ollama', modelId: 'example-model:70b', tuningKey: '', tuningLabel: null, verdict: 'does-not-fit', reason: 'out of memory' },
]);
});
diff --git a/server/lib/localModelTuning.js b/server/lib/localModelTuning.js
new file mode 100644
index 000000000..91704cdff
--- /dev/null
+++ b/server/lib/localModelTuning.js
@@ -0,0 +1,260 @@
+/**
+ * Launch/runtime tuning knobs for the local model runtimes PortOS can measure.
+ *
+ * A measured assessment answers "how did this model behave here?". That question
+ * is incomplete without "…configured how?" — the same GGUF on the same machine
+ * streams at wildly different rates depending on the micro-batch size, whether
+ * flash attention is on, and how much of the KV cache is quantized. Recording a
+ * throughput number with no record of the launch line makes two readings of the
+ * same model look like noise when they are actually two different setups.
+ *
+ * This module is the pure half of that: the per-runtime knob catalog, the
+ * normalizer that turns a request body into a knob set, and the stable signature
+ * that lets several tunings of ONE model coexist in the store and be ranked
+ * against each other.
+ *
+ * ## `applies` — the honesty axis (read before adding a knob)
+ *
+ * Every knob declares what PortOS can actually DO with it:
+ *
+ * - `'launch'` — PortOS starts this daemon, so it puts the knob on the
+ * command line (llama.cpp only today, via `llamaServerManager`).
+ * - `'request'` — PortOS sends it with each measurement request.
+ * - `'record'` — PortOS cannot set it. The user states how the daemon was
+ * launched so two readings are comparable.
+ *
+ * A `'record'` knob is NOT a lie by omission — it changes nothing about the run
+ * and the UI says so. What it must never do is claim to have been applied. Do
+ * not promote a knob to `'launch'`/`'request'` without a code path that sends it.
+ *
+ * ## The sentinel contract
+ *
+ * An ABSENT knob means "whatever the daemon defaults to", which is not a value
+ * we can name — it is never coerced to 0, `false`, or a guessed default. That is
+ * why `normalizeTuning` drops empty input instead of filling it in, and why the
+ * signature of an untuned run is `''` (so its store key is byte-identical to the
+ * pre-tuning key, and existing records keep resolving).
+ */
+
+/**
+ * KV-cache quantization types llama.cpp accepts for `--cache-type-k/-v`. Kept to
+ * the three that are universally compiled in; an exotic type the local build
+ * lacks would make the server exit on launch, which reads to the user as "this
+ * model does not fit".
+ */
+const CACHE_TYPES = ['f16', 'q8_0', 'q4_0'];
+
+/**
+ * Tuning knobs per runtime, in the order a UI should render them.
+ *
+ * `hint` is user-facing: it says what the knob trades away, because the point of
+ * a tuning sweep is finding the trade that suits this machine, not maximizing
+ * any single number.
+ */
+export const TUNING_SPECS = Object.freeze({
+ llama: Object.freeze([
+ { id: 'ctxSize', label: 'Context size', type: 'number', applies: 'launch', min: 512, max: 1048576, unit: 'tokens', hint: 'KV cache is allocated for the whole window up front — a larger one costs memory even when prompts are short.' },
+ { id: 'nGpuLayers', label: 'GPU layers', type: 'number', applies: 'launch', min: 0, max: 999, hint: 'Layers offloaded to the GPU. Fewer layers frees VRAM for a bigger context at the cost of throughput.' },
+ { id: 'batchSize', label: 'Batch size', type: 'number', applies: 'launch', min: 1, max: 8192, hint: 'Logical prompt batch (-b). Raising it speeds up prefill on long prompts and raises peak memory.' },
+ { id: 'ubatchSize', label: 'Micro-batch size', type: 'number', applies: 'launch', min: 1, max: 8192, hint: 'Physical micro-batch (-ub). The single knob that most often moves long-context throughput.' },
+ { id: 'threads', label: 'CPU threads', type: 'number', applies: 'launch', min: 1, max: 256, hint: 'Threads for the CPU-resident layers. More is not always faster once you pass the physical core count.' },
+ { id: 'flashAttn', label: 'Flash attention', type: 'boolean', applies: 'launch', hint: 'Fused attention kernel. Usually faster and lighter on memory, but not every build/GPU supports it.' },
+ { id: 'cacheTypeK', label: 'KV cache type (K)', type: 'enum', applies: 'launch', options: CACHE_TYPES, hint: 'Quantizing the key cache buys context length with a little quality.' },
+ { id: 'cacheTypeV', label: 'KV cache type (V)', type: 'enum', applies: 'launch', options: CACHE_TYPES, hint: 'Quantizing the value cache buys context length with a little quality.' },
+ { id: 'draftMax', label: 'Draft tokens', type: 'number', applies: 'launch', min: 0, max: 64, hint: 'Speculative-decoding lookahead. Only does anything when a drafter model is loaded.' },
+ ]),
+ ollama: Object.freeze([
+ { id: 'numCtx', label: 'Context size', type: 'number', applies: 'request', wire: 'num_ctx', min: 512, max: 1048576, unit: 'tokens', hint: 'Sent with the measurement request as `num_ctx`.' },
+ { id: 'numGpu', label: 'GPU layers', type: 'number', applies: 'record', min: 0, max: 999, hint: 'Set via the model\'s Modelfile or OLLAMA_NUM_GPU — recorded here so two readings are comparable.' },
+ { id: 'numThread', label: 'CPU threads', type: 'number', applies: 'record', min: 1, max: 256, hint: 'Set outside PortOS; recorded so a thread-count change does not read as noise.' },
+ { id: 'flashAttention', label: 'Flash attention', type: 'boolean', applies: 'record', hint: 'OLLAMA_FLASH_ATTENTION in the daemon\'s environment.' },
+ { id: 'kvCacheType', label: 'KV cache type', type: 'enum', applies: 'record', options: CACHE_TYPES, hint: 'OLLAMA_KV_CACHE_TYPE in the daemon\'s environment.' },
+ ]),
+ lmstudio: Object.freeze([
+ { id: 'contextLength', label: 'Context length', type: 'number', applies: 'record', min: 512, max: 1048576, unit: 'tokens', hint: 'Chosen in LM Studio when the model is loaded.' },
+ { id: 'gpuOffloadLayers', label: 'GPU offload layers', type: 'number', applies: 'record', min: 0, max: 999, hint: 'LM Studio\'s GPU offload slider.' },
+ { id: 'evalBatchSize', label: 'Eval batch size', type: 'number', applies: 'record', min: 1, max: 8192, hint: 'LM Studio\'s evaluation batch size.' },
+ { id: 'flashAttention', label: 'Flash attention', type: 'boolean', applies: 'record', hint: 'LM Studio\'s flash-attention toggle.' },
+ ]),
+ mtplx: Object.freeze([
+ { id: 'numDraftTokens', label: 'MTP draft tokens', type: 'number', applies: 'record', min: 0, max: 64, hint: 'Multi-token-prediction lookahead the server was started with.' },
+ { id: 'maxKvSize', label: 'Max KV size', type: 'number', applies: 'record', min: 512, max: 1048576, unit: 'tokens', hint: 'KV cache ceiling on the launch line.' },
+ { id: 'kvBits', label: 'KV cache bits', type: 'enum', applies: 'record', options: ['4', '8', '16'], hint: 'MLX KV-cache quantization width.' },
+ ]),
+ vllm: Object.freeze([
+ { id: 'maxModelLen', label: 'Max model length', type: 'number', applies: 'record', min: 512, max: 1048576, unit: 'tokens', hint: '--max-model-len on the container launch line.' },
+ { id: 'gpuMemoryUtilization', label: 'GPU memory utilization', type: 'number', applies: 'record', min: 0.1, max: 1, step: 0.05, hint: '--gpu-memory-utilization: the fraction of VRAM vLLM is allowed to claim.' },
+ { id: 'maxNumSeqs', label: 'Max concurrent sequences', type: 'number', applies: 'record', min: 1, max: 1024, hint: '--max-num-seqs: batching width, which trades single-stream latency for throughput.' },
+ ]),
+});
+
+/** Knob specs for one runtime, or `[]` for a runtime with none declared. */
+export const tuningSpecsFor = (runtimeId) => TUNING_SPECS[runtimeId] || [];
+
+const specById = (runtimeId, id) => tuningSpecsFor(runtimeId).find((s) => s.id === id) || null;
+
+const clamp = (value, min, max) => Math.min(max ?? Infinity, Math.max(min ?? -Infinity, value));
+
+/**
+ * Coerce one raw value against its spec. Returns `undefined` for anything that
+ * cannot be read as a value — which drops the knob entirely rather than
+ * substituting a default the daemon never saw.
+ */
+function coerceValue(spec, raw) {
+ if (raw === null || raw === undefined || raw === '') return undefined;
+ if (spec.type === 'boolean') {
+ if (typeof raw === 'boolean') return raw;
+ if (raw === 'true') return true;
+ if (raw === 'false') return false;
+ return undefined;
+ }
+ if (spec.type === 'enum') {
+ const value = String(raw);
+ return spec.options.includes(value) ? value : undefined;
+ }
+ const num = Number(raw);
+ if (!Number.isFinite(num)) return undefined;
+ const clamped = clamp(num, spec.min, spec.max);
+ // Integer knobs (everything but a utilization fraction) round rather than
+ // truncate — a launch line takes whole numbers, and `-ub 511.6` is not a thing.
+ return spec.step ? Number(clamped.toFixed(2)) : Math.round(clamped);
+}
+
+/**
+ * Reduce a raw tuning object to the knobs this runtime declares, coerced and
+ * clamped. Unknown keys are dropped silently: they cannot be applied, and
+ * persisting them would put an un-renderable field in the store forever.
+ *
+ * @param {string} runtimeId
+ * @param {object|null|undefined} tuning
+ * @returns {object} `{}` when nothing usable was supplied — which means "daemon
+ * defaults", NOT "every knob set to zero".
+ */
+export function normalizeTuning(runtimeId, tuning) {
+ if (!tuning || typeof tuning !== 'object') return {};
+ const out = {};
+ for (const spec of tuningSpecsFor(runtimeId)) {
+ const value = coerceValue(spec, tuning[spec.id]);
+ if (value !== undefined) out[spec.id] = value;
+ }
+ return out;
+}
+
+/**
+ * Stable identity for a tuning set: sorted `id=value` pairs.
+ *
+ * `''` for an empty set — deliberately, so an untuned assessment keys exactly
+ * as it did before tuning existed and every record already on disk keeps
+ * resolving without a migration.
+ */
+export function tuningSignature(tuning) {
+ const entries = Object.entries(tuning || {}).filter(([, v]) => v !== null && v !== undefined && v !== '');
+ if (entries.length === 0) return '';
+ return entries.sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join(',');
+}
+
+const formatValue = (spec, value) => {
+ if (spec.type === 'boolean') return value ? 'on' : 'off';
+ if (spec.type === 'number' && spec.unit === 'tokens' && value >= 1024) return `${Math.round(value / 1024)}k`;
+ return String(value);
+};
+
+/**
+ * Human label for a tuning set, e.g. `Context size 32k · Micro-batch size 512`.
+ * `null` when nothing was tuned — the caller renders "backend defaults" rather
+ * than an empty string that reads like a missing value.
+ */
+export function describeTuning(runtimeId, tuning) {
+ const parts = tuningSpecsFor(runtimeId)
+ .filter((spec) => tuning?.[spec.id] !== undefined && tuning?.[spec.id] !== null)
+ .map((spec) => `${spec.label} ${formatValue(spec, tuning[spec.id])}`);
+ return parts.length ? parts.join(' · ') : null;
+}
+
+/**
+ * The subset of a tuning set PortOS puts on the daemon's command line. Empty for
+ * every runtime PortOS does not start itself.
+ */
+export function launchTuning(runtimeId, tuning) {
+ const out = {};
+ for (const [id, value] of Object.entries(tuning || {})) {
+ if (specById(runtimeId, id)?.applies === 'launch') out[id] = value;
+ }
+ return out;
+}
+
+/**
+ * The request-applied knobs, rendered under their WIRE names, ready to merge
+ * into a chat-completions body.
+ *
+ * The wire name lives on the spec rather than in a lookup table here so a knob
+ * cannot be declared `applies: 'request'` without also declaring the field the
+ * daemon reads — which would produce a knob that silently does nothing while the
+ * UI reports it as applied.
+ */
+export function requestBody(runtimeId, tuning) {
+ const out = {};
+ for (const [id, value] of Object.entries(tuning || {})) {
+ const spec = specById(runtimeId, id);
+ if (spec?.applies === 'request' && spec.wire) out[spec.wire] = value;
+ }
+ return out;
+}
+
+const throughputOf = (a) => {
+ const value = a?.performance?.meanCharsPerSecond;
+ return Number.isFinite(value) ? value : null;
+};
+
+/**
+ * Group measured assessments by (backend, model) and report which tuning won.
+ *
+ * Only models with TWO OR MORE tunings that both produced throughput appear: a
+ * single reading has nothing to compare against, and presenting it as a "best
+ * tuning" would dress one measurement up as a conclusion.
+ *
+ * `deltaPercent` is each variant's throughput relative to the winner, so the
+ * table answers "was that knob worth it?" rather than just listing numbers.
+ *
+ * @param {Array} assessments records from the store
+ * @returns {Array<{backend:string, modelId:string, best:object, variants:Array}>}
+ */
+export function compareTunings(assessments) {
+ const groups = new Map();
+ for (const assessment of Array.isArray(assessments) ? assessments : []) {
+ if (!assessment?.backend || !assessment?.modelId) continue;
+ if (throughputOf(assessment) === null) continue;
+ const key = `${assessment.backend}:${assessment.modelId}`;
+ if (!groups.has(key)) groups.set(key, []);
+ groups.get(key).push(assessment);
+ }
+
+ const rows = [];
+ for (const list of groups.values()) {
+ if (list.length < 2) continue;
+ const sorted = [...list].sort((a, b) => throughputOf(b) - throughputOf(a));
+ const winner = throughputOf(sorted[0]);
+ rows.push({
+ backend: sorted[0].backend,
+ modelId: sorted[0].modelId,
+ best: {
+ tuning: sorted[0].tuning || {},
+ label: describeTuning(sorted[0].backend, sorted[0].tuning) || 'Backend defaults',
+ charsPerSecond: winner,
+ },
+ variants: sorted.map((a) => ({
+ tuning: a.tuning || {},
+ label: describeTuning(a.backend, a.tuning) || 'Backend defaults',
+ charsPerSecond: throughputOf(a),
+ maxWorkingContextTokens: Number.isFinite(a.performance?.maxWorkingContextTokens)
+ ? a.performance.maxWorkingContextTokens
+ : null,
+ assessedAt: a.assessedAt || null,
+ // Relative to the winner, so 100 is the best measured tuning and 74
+ // means "a quarter slower than the best this model managed here".
+ deltaPercent: winner > 0 ? Number(((throughputOf(a) / winner) * 100).toFixed(1)) : null,
+ })),
+ });
+ }
+ return rows.sort((a, b) => a.modelId.localeCompare(b.modelId));
+}
diff --git a/server/lib/localModelTuning.test.js b/server/lib/localModelTuning.test.js
new file mode 100644
index 000000000..4ae3c6c14
--- /dev/null
+++ b/server/lib/localModelTuning.test.js
@@ -0,0 +1,179 @@
+import { describe, it, expect } from 'vitest';
+import {
+ TUNING_SPECS,
+ compareTunings,
+ describeTuning,
+ launchTuning,
+ normalizeTuning,
+ requestBody,
+ tuningSignature,
+ tuningSpecsFor,
+} from './localModelTuning.js';
+
+describe('TUNING_SPECS', () => {
+ it('declares an applies mode for every knob', () => {
+ for (const [runtime, specs] of Object.entries(TUNING_SPECS)) {
+ for (const spec of specs) {
+ expect(['launch', 'request', 'record'], `${runtime}/${spec.id}`).toContain(spec.applies);
+ }
+ }
+ });
+
+ // A `request` knob with no wire name would be reported to the user as applied
+ // while `requestBody` silently dropped it.
+ it('gives every request-applied knob the field name the daemon reads', () => {
+ for (const [runtime, specs] of Object.entries(TUNING_SPECS)) {
+ for (const spec of specs.filter((s) => s.applies === 'request')) {
+ expect(spec.wire, `${runtime}/${spec.id}`).toBeTruthy();
+ }
+ }
+ });
+
+ it('only puts launch knobs on runtimes PortOS actually starts', () => {
+ for (const [runtime, specs] of Object.entries(TUNING_SPECS)) {
+ if (runtime === 'llama') continue;
+ expect(specs.some((s) => s.applies === 'launch'), runtime).toBe(false);
+ }
+ });
+
+ it('returns an empty list for an unknown runtime rather than throwing', () => {
+ expect(tuningSpecsFor('not-a-runtime')).toEqual([]);
+ });
+});
+
+describe('normalizeTuning', () => {
+ it('drops keys the runtime does not declare', () => {
+ expect(normalizeTuning('llama', { ubatchSize: 512, rmRf: '/' })).toEqual({ ubatchSize: 512 });
+ });
+
+ it('clamps a number to its declared range', () => {
+ expect(normalizeTuning('llama', { threads: 9999 })).toEqual({ threads: 256 });
+ expect(normalizeTuning('llama', { threads: 0 })).toEqual({ threads: 1 });
+ });
+
+ it('rounds an integer knob rather than truncating — a launch line takes whole numbers', () => {
+ expect(normalizeTuning('llama', { ubatchSize: 511.6 })).toEqual({ ubatchSize: 512 });
+ });
+
+ it('keeps a fractional knob fractional when the spec declares a step', () => {
+ expect(normalizeTuning('vllm', { gpuMemoryUtilization: 0.85 })).toEqual({ gpuMemoryUtilization: 0.85 });
+ });
+
+ it('coerces the string booleans a form posts', () => {
+ expect(normalizeTuning('llama', { flashAttn: 'true' })).toEqual({ flashAttn: true });
+ expect(normalizeTuning('llama', { flashAttn: 'false' })).toEqual({ flashAttn: false });
+ });
+
+ it('rejects an enum value outside the declared options', () => {
+ expect(normalizeTuning('llama', { cacheTypeK: 'q2_k' })).toEqual({});
+ expect(normalizeTuning('llama', { cacheTypeK: 'q8_0' })).toEqual({ cacheTypeK: 'q8_0' });
+ });
+
+ // ABSENT is not zero. An empty field must leave the daemon on its own default
+ // rather than pinning a value the user never chose.
+ it.each([undefined, null, '', {}])('treats %p as "no tuning", not as zeroes', (input) => {
+ expect(normalizeTuning('llama', input)).toEqual({});
+ });
+
+ it('drops a non-numeric value instead of recording NaN', () => {
+ expect(normalizeTuning('llama', { threads: 'lots' })).toEqual({});
+ });
+});
+
+describe('tuningSignature', () => {
+ it('is empty for backend defaults, so a pre-tuning store key is unchanged', () => {
+ expect(tuningSignature({})).toBe('');
+ expect(tuningSignature(null)).toBe('');
+ });
+
+ it('is stable regardless of key order', () => {
+ expect(tuningSignature({ ubatchSize: 512, threads: 8 }))
+ .toBe(tuningSignature({ threads: 8, ubatchSize: 512 }));
+ });
+
+ it('separates two different tunings', () => {
+ expect(tuningSignature({ ubatchSize: 512 })).not.toBe(tuningSignature({ ubatchSize: 256 }));
+ });
+});
+
+describe('describeTuning', () => {
+ it('renders labels in spec order with human units', () => {
+ expect(describeTuning('llama', { flashAttn: true, ctxSize: 32768 }))
+ .toBe('Context size 32k · Flash attention on');
+ });
+
+ it('is null for backend defaults so the caller can say so in its own words', () => {
+ expect(describeTuning('llama', {})).toBeNull();
+ });
+
+ it('renders a false boolean as off, not as absent', () => {
+ expect(describeTuning('llama', { flashAttn: false })).toBe('Flash attention off');
+ });
+});
+
+describe('launchTuning / requestBody', () => {
+ it('keeps only the knobs that reach the llama.cpp command line', () => {
+ expect(launchTuning('llama', { ubatchSize: 512, cacheTypeK: 'q8_0' }))
+ .toEqual({ ubatchSize: 512, cacheTypeK: 'q8_0' });
+ });
+
+ it('finds no launch knobs on a runtime PortOS does not start', () => {
+ expect(launchTuning('ollama', { numCtx: 8192, numGpu: 40 })).toEqual({});
+ });
+
+ it('renders request knobs under the wire name the daemon reads', () => {
+ expect(requestBody('ollama', { numCtx: 8192, numGpu: 40 })).toEqual({ num_ctx: 8192 });
+ });
+
+ it('sends nothing for a record-only runtime', () => {
+ expect(requestBody('lmstudio', { contextLength: 8192 })).toEqual({});
+ });
+});
+
+describe('compareTunings', () => {
+ const measured = (tuning, charsPerSecond, extra = {}) => ({
+ backend: 'llama',
+ modelId: 'example-7b',
+ tuning,
+ performance: { meanCharsPerSecond: charsPerSecond, maxWorkingContextTokens: 16384 },
+ assessedAt: '2026-08-01T00:00:00.000Z',
+ ...extra,
+ });
+
+ it('ranks a model\'s tunings and reports each against the winner', () => {
+ const [row] = compareTunings([
+ measured({ ubatchSize: 256 }, 90),
+ measured({ ubatchSize: 512 }, 120),
+ ]);
+ expect(row.modelId).toBe('example-7b');
+ expect(row.best.charsPerSecond).toBe(120);
+ expect(row.best.label).toBe('Micro-batch size 512');
+ expect(row.variants.map((v) => v.deltaPercent)).toEqual([100, 75]);
+ });
+
+ it('labels the untuned variant as backend defaults rather than an empty string', () => {
+ const [row] = compareTunings([measured({}, 120), measured({ ubatchSize: 512 }, 90)]);
+ expect(row.best.label).toBe('Backend defaults');
+ });
+
+ // One reading is not a comparison. Presenting it as "the best tuning" would
+ // dress a single measurement up as a conclusion.
+ it('omits a model measured under only one tuning', () => {
+ expect(compareTunings([measured({ ubatchSize: 512 }, 120)])).toEqual([]);
+ });
+
+ it('omits a variant that never produced throughput instead of scoring it zero', () => {
+ expect(compareTunings([
+ measured({ ubatchSize: 512 }, 120),
+ measured({ ubatchSize: 256 }, null),
+ ])).toEqual([]);
+ });
+
+ it('never mixes two models into one comparison', () => {
+ const rows = compareTunings([
+ measured({ ubatchSize: 512 }, 120),
+ measured({ ubatchSize: 512 }, 40, { modelId: 'other-70b' }),
+ ]);
+ expect(rows).toEqual([]);
+ });
+});
diff --git a/server/lib/localProviderRuntime.js b/server/lib/localProviderRuntime.js
index 89bc26e95..986f4b581 100644
--- a/server/lib/localProviderRuntime.js
+++ b/server/lib/localProviderRuntime.js
@@ -117,9 +117,9 @@ export function localBackendForProvider(provider) {
* probe a port nothing is on and call a working setup broken. LM Studio has no
* row there (nothing spawns OpenCode against it), so it carries its own.
*
- * `manageUrl` is the client route that installs/starts it — the Local LLM
- * settings tab owns every one of these flows, so an unmet requirement links
- * there rather than duplicating the install UI on the Providers page.
+ * `manageUrl` is the client route that installs/starts it — the Models → LLMs
+ * page owns every one of these flows, so an unmet requirement links there
+ * rather than duplicating the install UI on the Providers page.
*/
export const LOCAL_RUNTIMES = Object.freeze({
llama: Object.freeze({
@@ -129,7 +129,7 @@ export const LOCAL_RUNTIMES = Object.freeze({
// `llamaServerManager` resolves and starts.
command: 'llama-server',
defaultBaseUrl: opencodeLocalBaseUrl('llama'),
- manageUrl: '/settings/local-llm',
+ manageUrl: '/models/llms',
docsUrl: 'https://github.com/ggml-org/llama.cpp',
// Named so an unmet check can say what the user still has to fetch. GGUF
// weights are a separate download from the binary — the single most common
@@ -141,16 +141,16 @@ export const LOCAL_RUNTIMES = Object.freeze({
label: 'Ollama',
command: 'ollama',
defaultBaseUrl: opencodeLocalBaseUrl('ollama'),
- manageUrl: '/settings/local-llm',
+ manageUrl: '/models/llms',
docsUrl: 'https://ollama.com/download',
- modelsHint: 'Pull a model from Settings → Local LLM before an agent can use this provider.',
+ modelsHint: 'Pull a model from Models → LLMs before an agent can use this provider.',
}),
lmstudio: Object.freeze({
id: 'lmstudio',
label: 'LM Studio',
command: 'lms',
defaultBaseUrl: 'http://localhost:1234/v1',
- manageUrl: '/settings/local-llm',
+ manageUrl: '/models/llms',
docsUrl: 'https://lmstudio.ai/download',
modelsHint: 'Download a model in LM Studio and start its local server.',
}),
@@ -163,7 +163,7 @@ export const LOCAL_RUNTIMES = Object.freeze({
// answer about it.
command: 'docker',
defaultBaseUrl: opencodeLocalBaseUrl('vllm'),
- // No Local LLM tab entry — the weights and the compose project are an
+ // No Models → LLMs entry — the weights and the compose project are an
// operator-owned ~20 GB prepare step, not something PortOS downloads.
manageUrl: null,
docsUrl: 'https://github.com/atomantic/PortOS/blob/main/docs/features/qwen38-rtx3090.md',
@@ -174,7 +174,7 @@ export const LOCAL_RUNTIMES = Object.freeze({
label: 'MTPLX',
command: 'mtplx',
defaultBaseUrl: opencodeLocalBaseUrl('mtplx'),
- // No Local LLM tab entry — MTPLX has no model catalog inside PortOS. The
+ // No Models → LLMs entry — MTPLX has no model catalog inside PortOS. The
// one-click setup on the readiness checklist
// (`services/localRuntimeSetup.js`) is what installs and starts it.
manageUrl: null,
@@ -183,6 +183,32 @@ export const LOCAL_RUNTIMES = Object.freeze({
}),
});
+/**
+ * Local runtimes the measured-assessment feature can benchmark.
+ *
+ * Deliberately the same key space as `LOCAL_RUNTIMES` above, minus nothing:
+ * every local daemon PortOS knows how to *reach* is one it can also *measure*,
+ * because a measurement is just one bounded generation over the shared
+ * OpenAI-compatible wire protocol.
+ *
+ * The split below is about how PortOS gets to the model, not about how good the
+ * measurement is:
+ * - MANAGED — PortOS keeps a provider record and an installed-model catalog
+ * for it, so a run goes through the playground's provider path and lands in
+ * `/runs`.
+ * - ENDPOINT — a bare loopback daemon the user (or PortOS's llama-server
+ * launcher) started. Its "installed models" are whatever `GET /v1/models`
+ * reports right now, and a measurement talks to the endpoint directly.
+ */
+export const ASSESSABLE_RUNTIMES = Object.freeze(['ollama', 'lmstudio', 'llama', 'mtplx', 'vllm']);
+
+/** Assessable runtimes PortOS holds a provider record and model catalog for. */
+export const MANAGED_ASSESSMENT_BACKENDS = Object.freeze(['ollama', 'lmstudio']);
+
+/** True for an assessable runtime reached as a bare OpenAI-compatible endpoint. */
+export const isEndpointRuntime = (id) =>
+ ASSESSABLE_RUNTIMES.includes(id) && !MANAGED_ASSESSMENT_BACKENDS.includes(id);
+
/**
* Normalize a base URL to the `/v1` root an OpenAI-compatible probe needs.
* A scheme is added when missing, because `OLLAMA_HOST` is conventionally a
@@ -270,7 +296,7 @@ export function localRuntimeForProvider(provider) {
// however local its name/id looks. An `LM Studio ` provider pointed
// at a LAN host still matched `lmstudio` by NAME, and the card answered
// "LM Studio installed — `lms` is on PortOS's PATH" and "start it from
- // Settings → Local LLM" about a server PortOS neither runs nor can start.
+ // Models → LLMs" about a server PortOS neither runs nor can start.
// An external API endpoint is assumed to be set up by whoever runs it; the
// only honest report here is none.
if (!isLocalInstanceEndpoint(endpoint)) return null;
diff --git a/server/lib/localProviderRuntime.test.js b/server/lib/localProviderRuntime.test.js
index 5e9ede5bc..3b0cd3cba 100644
--- a/server/lib/localProviderRuntime.test.js
+++ b/server/lib/localProviderRuntime.test.js
@@ -103,7 +103,7 @@ describe('localRuntimeForProvider', () => {
expect(runtime.label).toBe('llama.cpp');
expect(runtime.command).toBe('llama-server');
expect(runtime.endpoint).toBe('http://127.0.0.1:8090/v1');
- expect(runtime.manageUrl).toBe('/settings/local-llm');
+ expect(runtime.manageUrl).toBe('/models/llms');
});
it('falls back to the provider endpoint when the stored OpenCode config is unparseable', () => {
@@ -149,7 +149,7 @@ describe('localRuntimeForProvider', () => {
it('returns null for an API provider whose endpoint lives on ANOTHER machine', () => {
// The name matches `lmstudio`, so the card used to report THIS host's
// install state — "`lms` is on PortOS's PATH", "start LM Studio from
- // Settings → Local LLM" — for a server PortOS neither runs nor can start.
+ // Models → LLMs" — for a server PortOS neither runs nor can start.
expect(localRuntimeForProvider({
type: 'api',
id: 'lmstudio-peer',
diff --git a/server/lib/mediaValidation.js b/server/lib/mediaValidation.js
index 3e0b91f0b..e6e945f2d 100644
--- a/server/lib/mediaValidation.js
+++ b/server/lib/mediaValidation.js
@@ -10,6 +10,7 @@
*/
import { z } from 'zod';
import { PORTS } from './ports.js';
+import { ASSESSABLE_RUNTIMES } from './localProviderRuntime.js';
// OpenWorld snapshot pipeline (issue #877): how often to capture a city-state
// frame and how many to retain. Validated as a settings slice on PUT /api/settings;
@@ -178,6 +179,17 @@ export const localLlmLlamaServerStartSchema = z.object({
ctxSize: z.coerce.number().int().min(512).max(1048576).optional().default(32768),
nGpuLayers: z.coerce.number().int().min(0).max(999).optional().default(99),
alias: z.string().trim().max(100).optional().default('dflash'),
+ // Tuning flags. Every one defaults to null = NOT SET, so the flag is left off
+ // the launch line and llama.cpp applies its own default — a numeric default
+ // here would silently pin a value the user never chose. Ranges mirror
+ // `lib/localModelTuning.js`; keep them in lockstep.
+ batchSize: z.coerce.number().int().min(1).max(8192).optional().nullable().default(null),
+ ubatchSize: z.coerce.number().int().min(1).max(8192).optional().nullable().default(null),
+ threads: z.coerce.number().int().min(1).max(256).optional().nullable().default(null),
+ flashAttn: z.boolean().optional().default(false),
+ cacheTypeK: z.enum(['f16', 'q8_0', 'q4_0']).optional().nullable().default(null),
+ cacheTypeV: z.enum(['f16', 'q8_0', 'q4_0']).optional().nullable().default(null),
+ draftMax: z.coerce.number().int().min(0).max(64).optional().nullable().default(null),
});
// Speculative-decoding weight download: which curated preset, and which half of
// the pair. Both are enum-ish server-owned ids — no path or repo ever arrives
@@ -203,19 +215,41 @@ export const localLlmTestSchema = localLlmPlaygroundOptionsSchema.extend({
modelId: localLlmModelIdSchema,
prompt: z.string().trim().min(1).max(50000),
});
+// Assessments reach EVERY local runtime PortOS can talk to, not just the two it
+// installs models for — llama.cpp, MTPLX, and vLLM are bare OpenAI-compatible
+// daemons with no PortOS-side catalog. `localLlmBackendSchema` stays narrow
+// because install/delete/migrate genuinely only work on the managed pair.
+export const localLlmRuntimeSchema = z.enum(ASSESSABLE_RUNTIMES);
+// Tuning knobs are validated for SHAPE only (a flat map of scalars). Which keys
+// a runtime accepts, and their ranges, live in `lib/localModelTuning.js` — one
+// catalog, applied by `normalizeTuning`, rather than a Zod copy that would drift
+// from it. Unknown keys are dropped there, so a bogus key cannot reach a launch
+// line.
+export const localLlmTuningSchema = z.record(
+ z.string().max(64),
+ z.union([z.number(), z.boolean(), z.string().max(64)])
+).optional();
// Measured local-model assessment (server/services/localModelAssessments.js). One
// request runs ONE model across up to 5 nominal context sizes; the cap keeps a
// single user click from turning into an unbounded, minutes-long provider job.
// 131072 is the largest context any shipped local model advertises.
export const localLlmAssessmentRunSchema = z.object({
- backend: localLlmBackendSchema,
+ backend: localLlmRuntimeSchema,
modelId: localLlmModelIdSchema,
contextTokens: z.array(z.coerce.number().int().min(64).max(131072)).min(1).max(5).optional(),
+ tuning: localLlmTuningSchema,
});
export const localLlmAssessmentIntentSchema = z.object({
intent: z.enum(['balanced', 'smartest', 'fastest', 'lightweight']).optional().default('balanced'),
});
-export const localLlmAssessmentDeleteSchema = localLlmInstallSchema;
+// `tuningKey` identifies WHICH measurement of a model to drop — several can now
+// coexist, one per tuning. Absent/'' targets the backend-defaults record, which
+// is exactly what a pre-tuning client sends.
+export const localLlmAssessmentDeleteSchema = z.object({
+ backend: localLlmRuntimeSchema,
+ modelId: localLlmModelIdSchema,
+ tuningKey: z.string().max(500).optional().default(''),
+});
export const localLlmCompareSchema = z.object({
mode: z.enum(['round-robin', 'parallel']).optional().default('round-robin'),
diff --git a/server/lib/navManifest.js b/server/lib/navManifest.js
index 1b750a57d..9855d2a07 100644
--- a/server/lib/navManifest.js
+++ b/server/lib/navManifest.js
@@ -264,8 +264,13 @@ export const NAV_COMMANDS = [
{ id: 'nav.settings.database', path: '/settings/database', label: 'Database', section: 'Settings', aliases: ['settings-database', 'database'] },
{ id: 'nav.settings.embeddings', path: '/settings/embeddings', label: 'Embeddings', section: 'Settings', aliases: ['settings-embeddings', 'embeddings', 'embedding'], keywords: ['vector', 'pgvector', 'semantic search', 'nomic', 'ollama', 'lm studio'] },
{ id: 'nav.settings.general', path: '/settings/general', label: 'General', section: 'Settings', aliases: ['settings', 'settings-general', 'general'] },
- { id: 'nav.settings.local-llm', path: '/settings/local-llm', label: 'Local LLMs', section: 'Settings', aliases: ['local-llm', 'local-llms', 'ollama', 'lm-studio', 'lmstudio'], keywords: ['ollama', 'lm studio', 'local model', 'local llm', 'gguf', 'pull model', 'install model', 'migrate', 'switch backend'] },
- { id: 'nav.settings.local-llm-playground', path: '/local-llm/playground', label: 'Local LLM Playground', section: 'Settings', aliases: ['llm-playground', 'playground', 'model-playground', 'compare-models'], keywords: ['ollama', 'lm studio', 'compare', 'benchmark', 'chat', 'test model', 'ttft', 'tokens per second', 'local llm'] },
+ // Local-model management is its own top-level section (#4736). The ids keep
+ // their `nav.settings.*` prefix — they are opaque and stored in palette
+ // history, so renaming them would orphan those entries.
+ { id: 'nav.settings.local-llm', path: '/models/llms', label: 'LLMs', section: 'Models', aliases: ['local-llm', 'local-llms', 'llms', 'models-llms', 'ollama', 'lm-studio', 'lmstudio'], keywords: ['ollama', 'lm studio', 'local model', 'local llm', 'gguf', 'pull model', 'install model', 'migrate', 'switch backend', 'llama.cpp'] },
+ { id: 'nav.models.performance', path: '/models/performance', label: 'Model Performance', section: 'Models', aliases: ['model-performance', 'performance', 'assessments', 'model-assessments', 'benchmark-models', 'tuning'], keywords: ['measure', 'assessment', 'benchmark', 'throughput', 'chars per second', 'ttft', 'context', 'tuning', 'llama.cpp', 'mtplx', 'vllm', 'which model', 'fastest model'] },
+ { id: 'nav.models.status', path: '/models/status', label: 'Model Status', section: 'Models', aliases: ['model-status', 'models-status', 'memory-management', 'resident-models'], keywords: ['memory', 'resident', 'loaded', 'unload', 'ram', 'vram', 'free memory', 'what is loaded'] },
+ { id: 'nav.settings.local-llm-playground', path: '/local-llm/playground', label: 'Local LLM Playground', section: 'Models', aliases: ['llm-playground', 'playground', 'model-playground', 'compare-models'], keywords: ['ollama', 'lm studio', 'compare', 'benchmark', 'chat', 'test model', 'ttft', 'tokens per second', 'local llm'] },
{ id: 'nav.settings.mortalloom', path: '/settings/mortalloom', label: 'MortalLoom', section: 'Settings', aliases: ['settings-mortalloom', 'mortalloom'] },
{ id: 'nav.settings.openclaw', path: '/openclaw', label: 'OpenClaw', section: 'Settings', aliases: ['openclaw', 'settings-openclaw'], keywords: ['operator', 'chat', 'agent', 'runtime', 'sessions', 'streaming'] },
{ id: 'nav.settings.security', path: '/settings/security', label: 'Security', section: 'Settings', aliases: ['settings-security', 'login-password', 'auth-password', 'password-settings'], keywords: ['password', 'login', 'auth', 'sign-in', 'lock', 'tailnet', 'sidecar'] },
diff --git a/server/lib/navManifest.test.js b/server/lib/navManifest.test.js
index 77811ad6a..6b1182a5f 100644
--- a/server/lib/navManifest.test.js
+++ b/server/lib/navManifest.test.js
@@ -131,7 +131,7 @@ describe('navManifest — shape invariants', () => {
it('every section is one of the approved sidebar group labels', () => {
const ALLOWED_SECTIONS = new Set([
'Main', 'Apps', 'Brain', 'Calendar', 'Chief of Staff', 'Comms', 'Create',
- 'Dev Tools', 'Goals', 'Health', 'Settings', 'Identity', 'POST',
+ 'Dev Tools', 'Goals', 'Health', 'Models', 'Settings', 'Identity', 'POST',
]);
const bad = NAV_COMMANDS.filter((c) => !ALLOWED_SECTIONS.has(c.section));
expect(bad.map((c) => `${c.id}:${c.section}`)).toEqual([]);
diff --git a/server/lib/openAiChatStream.js b/server/lib/openAiChatStream.js
new file mode 100644
index 000000000..2a310622e
--- /dev/null
+++ b/server/lib/openAiChatStream.js
@@ -0,0 +1,177 @@
+/**
+ * One streaming `POST {base}/chat/completions` against an OpenAI-compatible
+ * endpoint.
+ *
+ * Sibling of `openAiModelsProbe.js`: that module answers "is anything serving
+ * here, and what does it serve?", this one answers "generate against it and tell
+ * me what streamed". Both exist because PortOS talks to five local daemons
+ * (llama.cpp, Ollama, LM Studio, MTPLX, vLLM) that share exactly one wire
+ * protocol and nothing else.
+ *
+ * Two callers, deliberately:
+ * - `services/localLlmPlayground.js` — a provider-backed run with a `/runs`
+ * record, for the backends PortOS configures as providers.
+ * - `services/localModelAssessments.js` — a measurement against a bare
+ * loopback daemon that has no provider record at all.
+ *
+ * Keeping the SSE read loop here means the reasoning-channel handling, the
+ * skip-a-malformed-frame rule, and the partial-output-on-abort behavior are one
+ * decision rather than two copies that drift.
+ */
+
+import { readResponseJson } from './readResponseJson.js';
+
+/**
+ * Parse one OpenAI-style SSE `data:` line into its content/reasoning delta.
+ * Returns null for non-data lines, the `[DONE]`/`✅` sentinels, or a malformed
+ * frame: a single bad frame must SKIP, not abort the stream — one non-JSON
+ * keep-alive would otherwise throw out of the read loop and discard every
+ * token already received.
+ */
+export function extractStreamDelta(rawLine) {
+ const line = rawLine.trim();
+ if (!line.startsWith('data: ')) return null;
+ const data = line.slice(6).trim();
+ if (!data || data === '[DONE]' || data === '✅') return null;
+ let parsed;
+ try {
+ parsed = JSON.parse(data);
+ } catch {
+ return null;
+ }
+ const delta = parsed?.choices?.[0]?.delta;
+ return { content: delta?.content || '', reasoning: delta?.reasoning || '' };
+}
+
+/**
+ * Resolve the text to surface from a (possibly interrupted) stream: prefer the
+ * visible content, fall back to reasoning when no content arrived (some models
+ * emit only a reasoning channel), and `''` when neither did. Used on both the
+ * normal-finish path and the partial-output-on-throw path so a timed-out run
+ * still shows what streamed before the abort.
+ */
+export function resolvePartialOutput({ output = '', reasoning = '' }) {
+ if (output.trim()) return output;
+ if (reasoning.trim()) return reasoning;
+ return '';
+}
+
+export function buildMessages({ systemPrompt, prompt }) {
+ const system = String(systemPrompt || '').trim();
+ return [
+ ...(system ? [{ role: 'system', content: system }] : []),
+ { role: 'user', content: prompt },
+ ];
+}
+
+/**
+ * Stream a chat completion and return the final text.
+ *
+ * @param {object} options
+ * @param {string} options.endpoint OpenAI-compatible base ending in `/v1`
+ * @param {string} [options.apiKey] attached as a bearer token when set
+ * @param {string} options.model
+ * @param {Array<{role:string,content:string}>} options.messages
+ * @param {number} [options.temperature]
+ * @param {number} [options.maxTokens]
+ * @param {object} [options.extraBody] merged into the request body — how a
+ * caller passes a backend-specific knob (Ollama's `num_ctx`) without this
+ * module growing a per-backend branch.
+ * @param {AbortSignal} [options.signal]
+ * @param {(chunk: string, kind: 'content'|'reasoning') => any} [options.onChunk]
+ * awaited, so a consumer's backpressure reaches the upstream read loop.
+ * @returns {Promise} the streamed text. Throws on transport/HTTP
+ * failure; an abort mid-stream throws with `.partialOutput` carrying whatever
+ * had already streamed.
+ */
+export async function streamOpenAiChat({
+ endpoint,
+ apiKey,
+ model,
+ messages,
+ temperature,
+ maxTokens,
+ extraBody = {},
+ signal,
+ onChunk,
+}) {
+ const headers = { 'Content-Type': 'application/json' };
+ if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
+
+ const response = await fetch(`${String(endpoint || '').replace(/\/+$/, '')}/chat/completions`, {
+ method: 'POST',
+ headers,
+ signal,
+ body: JSON.stringify({
+ model,
+ messages,
+ stream: true,
+ temperature,
+ max_tokens: maxTokens,
+ ...extraBody,
+ }),
+ }).catch((err) => ({ ok: false, status: 0, error: err.message }));
+
+ if (!response.ok) {
+ const body = response.text ? await response.text().catch(() => '') : response.error || '';
+ throw new Error(`Provider returned ${response.status || 0}: ${body || response.error || response.statusText || 'request failed'}`);
+ }
+
+ if (!response.body?.getReader) {
+ // A non-streaming 200 (some daemons ignore `stream: true`). Read it whole
+ // rather than reporting an empty generation, which a caller would persist as
+ // a successful run that produced nothing — hence the `null` sentinel on both
+ // a blank and an unparseable body, which throws rather than returning ''.
+ const data = await readResponseJson(response, { fallback: null, emptyValue: null });
+ if (!data) throw new Error(`Provider returned a non-JSON response (${response.status})`);
+ const text = data.choices?.[0]?.message?.content || '';
+ if (text) await onChunk?.(text, 'content');
+ return text;
+ }
+
+ const reader = response.body.getReader();
+ const decoder = new TextDecoder();
+ let buffer = '';
+ let output = '';
+ let reasoning = '';
+
+ const consumeLine = async (rawLine) => {
+ const delta = extractStreamDelta(rawLine);
+ if (!delta) return;
+ if (delta.content) {
+ output += delta.content;
+ await onChunk?.(delta.content, 'content');
+ }
+ // Reasoning streams on its own channel so a reasoning-only model
+ // (deepseek-r1, qwq, …) renders as it arrives instead of sitting on
+ // "waiting for the first token", and so the final content-only text does
+ // not inherit reasoning prose.
+ if (delta.reasoning) {
+ reasoning += delta.reasoning;
+ await onChunk?.(delta.reasoning, 'reasoning');
+ }
+ };
+
+ // Always release the reader (and tear down the socket) on every exit path — a
+ // normal finish, an abort via a timeout signal, or a throw mid-stream. On a
+ // throw, surface the tokens already streamed (attached to the error) instead
+ // of discarding them.
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split('\n');
+ buffer = lines.pop() || '';
+ for (const line of lines) await consumeLine(line);
+ }
+ if (buffer.trim()) await consumeLine(buffer);
+ } catch (err) {
+ err.partialOutput = resolvePartialOutput({ output, reasoning });
+ throw err;
+ } finally {
+ await reader.cancel().catch(() => {});
+ }
+
+ return resolvePartialOutput({ output, reasoning });
+}
diff --git a/server/lib/openAiChatStream.test.js b/server/lib/openAiChatStream.test.js
new file mode 100644
index 000000000..1e61ba7f3
--- /dev/null
+++ b/server/lib/openAiChatStream.test.js
@@ -0,0 +1,70 @@
+/**
+ * The OpenAI-compatible chat-stream helpers, extracted from
+ * `services/localLlmPlayground.js` so a bare loopback daemon (llama.cpp, MTPLX,
+ * vLLM) can be measured without a provider record.
+ *
+ * `streamOpenAiChat` itself is exercised end-to-end through its two callers'
+ * suites; what is worth pinning here are the pure decisions inside the read
+ * loop — a malformed frame must SKIP rather than abort the stream, and a
+ * reasoning-only model must still surface its output.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { buildMessages, extractStreamDelta, resolvePartialOutput } from './openAiChatStream.js';
+
+describe('buildMessages', () => {
+ it('omits the system message when blank', () => {
+ expect(buildMessages({ systemPrompt: ' ', prompt: 'hi' })).toEqual([
+ { role: 'user', content: 'hi' },
+ ]);
+ });
+
+ it('includes a system message when present', () => {
+ expect(buildMessages({ systemPrompt: 'Be terse', prompt: 'hi' })).toEqual([
+ { role: 'system', content: 'Be terse' },
+ { role: 'user', content: 'hi' },
+ ]);
+ });
+});
+
+describe('extractStreamDelta', () => {
+ it('parses an OpenAI-style content delta', () => {
+ const line = 'data: {"choices":[{"delta":{"content":"Hi"}}]}';
+ expect(extractStreamDelta(line)).toEqual({ content: 'Hi', reasoning: '' });
+ });
+
+ it('parses a reasoning delta', () => {
+ const line = 'data: {"choices":[{"delta":{"reasoning":"thinking"}}]}';
+ expect(extractStreamDelta(line)).toEqual({ content: '', reasoning: 'thinking' });
+ });
+
+ it('skips non-data lines and the [DONE]/✅ sentinels', () => {
+ expect(extractStreamDelta(': keep-alive')).toBeNull();
+ expect(extractStreamDelta('data: [DONE]')).toBeNull();
+ expect(extractStreamDelta('data: ✅')).toBeNull();
+ expect(extractStreamDelta('')).toBeNull();
+ });
+
+ it('skips a malformed frame instead of throwing (one bad frame must not abort the stream)', () => {
+ expect(extractStreamDelta('data: {not json')).toBeNull();
+ });
+
+ it('tolerates a frame with no delta', () => {
+ expect(extractStreamDelta('data: {"choices":[{}]}')).toEqual({ content: '', reasoning: '' });
+ });
+});
+
+describe('resolvePartialOutput', () => {
+ it('prefers visible content over reasoning', () => {
+ expect(resolvePartialOutput({ output: 'hello', reasoning: 'thinking' })).toBe('hello');
+ });
+
+ it('falls back to reasoning when no content streamed', () => {
+ expect(resolvePartialOutput({ output: ' ', reasoning: 'partial thought' })).toBe('partial thought');
+ });
+
+ it('returns empty string when neither content nor reasoning streamed', () => {
+ expect(resolvePartialOutput({ output: '', reasoning: '' })).toBe('');
+ expect(resolvePartialOutput({})).toBe('');
+ });
+});
diff --git a/server/routes/localLlm.js b/server/routes/localLlm.js
index b5515bfb9..59ed36ee7 100644
--- a/server/routes/localLlm.js
+++ b/server/routes/localLlm.js
@@ -446,24 +446,24 @@ router.get('/assessments', asyncHandler(async (req, res) => {
// Long-running by nature (one bounded generation per context size), so the
// client's abort signal is threaded through to stop mid-run on disconnect.
router.post('/assessments/run', asyncHandler(async (req, res) => {
- const { backend, modelId, contextTokens } = validateRequest(localLlmAssessmentRunSchema, req.body)
+ const { backend, modelId, contextTokens, tuning } = validateRequest(localLlmAssessmentRunSchema, req.body)
const io = req.app.get('io')
// Same `localLlm:progress` channel the pull/migrate paths use, so one banner
// renders every long local-LLM operation. The extra fields (`scope`, `backend`,
// `modelId`, `sampleIndex`/`sampleCount`) let a listener tell an assessment
// frame from a model pull streaming on the same event.
const onProgress = (frame) => io?.emit('localLlm:progress', frame)
- res.json(await runAssessment({ backend, modelId, contextTokens, signal: abortSignalFromResponse(res), onProgress }))
+ res.json(await runAssessment({ backend, modelId, contextTokens, tuning, signal: abortSignalFromResponse(res), onProgress }))
}))
// POST /api/local-llm/assessments/delete — drop one stale measurement (e.g. after
// a RAM upgrade or a backend update makes the recorded evidence misleading).
// 404s when nothing was removed rather than reporting a phantom success.
router.post('/assessments/delete', asyncHandler(async (req, res) => {
- const { backend, modelId } = validateRequest(localLlmAssessmentDeleteSchema, req.body)
- const result = await deleteAssessment(backend, modelId)
- if (!result.deleted) throw new ServerError('No assessment recorded for that model', { status: 404, context: { backend, modelId } })
- res.json({ success: true, backend, modelId })
+ const { backend, modelId, tuningKey } = validateRequest(localLlmAssessmentDeleteSchema, req.body)
+ const result = await deleteAssessment(backend, modelId, tuningKey)
+ if (!result.deleted) throw new ServerError('No assessment recorded for that model', { status: 404, context: { backend, modelId, tuningKey } })
+ res.json({ success: true, backend, modelId, tuningKey })
}))
// === llama-server (DFlash 2 / Speculative Decoding) ==========================
diff --git a/server/services/huggingFaceCatalog.test.js b/server/services/huggingFaceCatalog.test.js
index 2a0218c85..a8e019da8 100644
--- a/server/services/huggingFaceCatalog.test.js
+++ b/server/services/huggingFaceCatalog.test.js
@@ -6,7 +6,7 @@ import { __resetOllamaRegistryCache } from './ollamaRegistryCatalog.js'
// consults it before the network — so without this mock these tests read the
// developer's live `data/cache/huggingface-repos.json`. The fixtures below use
// real repo ids (bartowski/…, facebook/musicgen-small, nomic-ai/…) that this very
-// feature caches the moment anyone opens Settings → Local LLM, so a cached record
+// feature caches the moment anyone opens Models → LLMs, so a cached record
// would bypass the `fetch` mock entirely and fail assertions locally while CI —
// with no cache file — stayed green. Mocking also keeps the debounced writer from
// ever persisting these fabricated records (a 13 GB `Qwen3.6-35B`, `burst-pub/…`)
diff --git a/server/services/llamaServerManager.js b/server/services/llamaServerManager.js
index 6b27f5813..1d9ab7291 100644
--- a/server/services/llamaServerManager.js
+++ b/server/services/llamaServerManager.js
@@ -24,6 +24,14 @@ export const LLAMA_APP = 'portos-llama-server';
const MAX_LOG_LINES = 100;
const PROBE_TIMEOUT_MS = 1500;
const STARTUP_WAIT_TIMEOUT_MS = 4000;
+// How long a relaunch waits for the kernel to release the old listener.
+const PORT_RELEASE_TIMEOUT_MS = 5000;
+// How long a relaunch waits for the new process to answer. `startLlamaServer`
+// polls for only STARTUP_WAIT_TIMEOUT_MS, which a large GGUF routinely exceeds
+// while loading — so a relaunch must not read "not ready yet" as "wedged".
+// Mutable only through the test seam below: a suite asserting the give-up path
+// cannot sit through two real minutes of polling.
+let relaunchReadyTimeoutMs = 120000;
let currentConfig = null;
let recentLogs = [];
@@ -105,9 +113,49 @@ function parseConfigFromArgs(args) {
ctxSize,
nGpuLayers,
alias,
+ // Tuning flags. `null` means the flag was NOT on the launch line, so
+ // llama.cpp's own default applied — distinct from a value we chose. A
+ // caller re-launching with this config must leave a null off the line
+ // rather than substituting a number llama.cpp never saw.
+ batchSize: getArg('-b') !== null ? Number(getArg('-b')) : null,
+ ubatchSize: getArg('-ub') !== null ? Number(getArg('-ub')) : null,
+ threads: getArg('-t') !== null ? Number(getArg('-t')) : null,
+ flashAttn: list.includes('--flash-attn') || list.includes('-fa'),
+ cacheTypeK: getArg('--cache-type-k'),
+ cacheTypeV: getArg('--cache-type-v'),
+ draftMax: getArg('--draft-max') !== null ? Number(getArg('--draft-max')) : null,
};
}
+// The endpoint the current (or last-known) configuration serves on. Split out so
+// the two callers below can't drift on how host/port are defaulted.
+const endpointFor = (config) =>
+ `http://${config?.host || '127.0.0.1'}:${config?.port ?? PORTS.LLAMA_SERVER}/v1`;
+
+/**
+ * Just the base URL llama-server is serving on — no endpoint probe, no PM2 log
+ * fetch.
+ *
+ * `getLlamaServerStatus` answers a much bigger question and pays for it with a
+ * network probe AND an `execPm2 logs` subprocess. A caller that only needs
+ * "which port is it on?" (the assessments read path, which runs on every
+ * Performance page load) must not spawn a process to find out and then discard
+ * the logs it paid for.
+ *
+ * Reads the same recovered-config path as the status call, so a PortOS restart
+ * that left the PM2 process online still resolves the real port rather than the
+ * default.
+ */
+export async function getLlamaServerEndpoint() {
+ if (!currentConfig) {
+ const pm2Status = await getAppStatusStrict(LLAMA_APP);
+ if (pm2Status?.status === 'online' && pm2Status.args) {
+ currentConfig = parseConfigFromArgs(pm2Status.args);
+ }
+ }
+ return endpointFor(currentConfig);
+}
+
/**
* Returns current status of llama-server (binary availability, running state, config, logs).
*/
@@ -125,7 +173,7 @@ export async function getLlamaServerStatus() {
const host = currentConfig?.host || '127.0.0.1';
const port = currentConfig?.port ?? PORTS.LLAMA_SERVER;
- const endpoint = `http://${host}:${port}/v1`;
+ const endpoint = endpointFor(currentConfig);
const reachable = await probeEndpoint(endpoint);
@@ -192,6 +240,17 @@ export async function startLlamaServer(options = {}) {
ctxSize = 32768,
nGpuLayers = 99,
alias = 'dflash',
+ // Tuning knobs (`lib/localModelTuning.js`). Every one defaults to `null` =
+ // NOT SET: the flag is left off the launch line entirely so llama.cpp
+ // applies its own default. Substituting a number here would silently pin a
+ // value the user never chose and make two "default" runs incomparable.
+ batchSize = null,
+ ubatchSize = null,
+ threads = null,
+ flashAttn = false,
+ cacheTypeK = null,
+ cacheTypeV = null,
+ draftMax = null,
} = options;
if (!model || typeof model !== 'string') {
@@ -251,6 +310,15 @@ export async function startLlamaServer(options = {}) {
if (host) args.push('--host', host);
if (ctxSize) args.push('--ctx-size', String(ctxSize));
if (nGpuLayers !== undefined && nGpuLayers !== null) args.push('-ngl', String(nGpuLayers));
+ if (Number.isFinite(batchSize)) args.push('-b', String(batchSize));
+ if (Number.isFinite(ubatchSize)) args.push('-ub', String(ubatchSize));
+ if (Number.isFinite(threads)) args.push('-t', String(threads));
+ if (flashAttn) args.push('--flash-attn');
+ if (cacheTypeK) args.push('--cache-type-k', String(cacheTypeK));
+ if (cacheTypeV) args.push('--cache-type-v', String(cacheTypeV));
+ // Only meaningful alongside a drafter — passing it without one makes
+ // llama-server reject the launch line outright.
+ if (Number.isFinite(draftMax) && draftPath) args.push('--draft-max', String(draftMax));
if (alias) args.push('--alias', alias);
lastExitError = null;
@@ -278,6 +346,13 @@ export async function startLlamaServer(options = {}) {
ctxSize,
nGpuLayers,
alias,
+ batchSize,
+ ubatchSize,
+ threads,
+ flashAttn,
+ cacheTypeK,
+ cacheTypeV,
+ draftMax,
};
// Delete stale PM2 entry so our own previous instance doesn't count as a collision
@@ -378,6 +453,127 @@ export async function stopLlamaServer() {
return { success: true, message: 'llama-server stopped' };
}
+/**
+ * Block until nothing is listening on `port`, or the timeout elapses.
+ *
+ * `startLlamaServer` refuses when the port is still bound, and PM2's delete
+ * returns before the kernel has released the listener — without this a relaunch
+ * loses a race with itself and reports "port already in use" for the server it
+ * just stopped.
+ */
+async function waitForPortRelease(port) {
+ const deadline = Date.now() + PORT_RELEASE_TIMEOUT_MS;
+ while (Date.now() < deadline && await isPortInUse(port)) await sleep(200);
+}
+
+/**
+ * Block until the endpoint answers, or the readiness budget elapses.
+ * `false` means it never answered — which is a wedged process, not a slow one.
+ */
+async function waitForEndpoint(endpoint) {
+ const deadline = Date.now() + relaunchReadyTimeoutMs;
+ while (Date.now() < deadline) {
+ if (await probeEndpoint(endpoint)) return true;
+ await sleep(1000);
+ }
+ return false;
+}
+
+/**
+ * Relaunch llama-server with a different tuning, keeping the model/drafter it is
+ * already serving.
+ *
+ * This is the "evaluate tuning parameters for launching these" half of the
+ * measured-assessment feature: a sweep across micro-batch sizes or KV-cache
+ * types is only possible if something can put those flags on the launch line
+ * between runs.
+ *
+ * It refuses rather than guesses in the two cases where it cannot know what to
+ * relaunch:
+ * - nothing is running, so there is no model path to reuse;
+ * - something IS listening but PortOS did not start it (an externally-launched
+ * llama-server), so stopping it would kill a process the user owns.
+ *
+ * Every one of those returns `{ applied: false, reason }` instead of throwing:
+ * the caller (an assessment run) can still measure whatever is actually serving
+ * and record that the requested tuning was NOT applied, which is far more useful
+ * than failing the whole run. A launch line llama-server rejects, and a relaunch
+ * that never answers on its port, land on the same shape — and the rejected case
+ * puts the PREVIOUS configuration back, because a tuning sweep is expected to
+ * produce launch lines that don't work and must not leave the daemon down.
+ *
+ * @param {object} tuning launch knobs from `lib/localModelTuning.js`
+ * @returns {Promise<{applied: boolean, reason: string|null, config: object|null}>}
+ */
+export async function relaunchLlamaServerWithTuning(tuning = {}) {
+ const knobs = Object.entries(tuning).filter(([, v]) => v !== null && v !== undefined);
+ if (knobs.length === 0) {
+ return { applied: false, reason: 'no launch knobs were requested', config: currentConfig };
+ }
+
+ const status = await getLlamaServerStatus();
+ if (!status.running) {
+ return { applied: false, reason: 'llama-server is not running, so PortOS has no model path to relaunch with', config: null };
+ }
+ if (!status.managed || !status.config?.model) {
+ return {
+ applied: false,
+ reason: 'llama-server was started outside PortOS — start it from the LLMs page to let PortOS apply tuning',
+ config: status.config || null,
+ };
+ }
+
+ const previous = status.config;
+ const next = { ...previous, ...tuning };
+ console.log(`🦙 llama-server: relaunching to apply tuning (${knobs.map(([k, v]) => `${k}=${v}`).join(', ')})`);
+ await stopLlamaServer();
+ await waitForPortRelease(next.port ?? PORTS.LLAMA_SERVER);
+
+ // A tuning sweep EXPECTS launch lines that don't work — `--flash-attn` on a
+ // build without it, a `--cache-type-k` this build lacks, a `-ub` past what the
+ // GPU can hold. llama-server exits immediately and `startLlamaServer` throws.
+ // Leaving it down would be far worse than not applying the tuning: this daemon
+ // fronts the `llama` provider for the whole install, so every later request
+ // would fail too. Put the previous configuration back before reporting.
+ const started = await startLlamaServer(next).catch(async (err) => {
+ console.error(`❌ llama-server: tuning launch failed (${err.message}) — restoring the previous configuration`);
+ await waitForPortRelease(previous.port ?? PORTS.LLAMA_SERVER);
+ const restored = await startLlamaServer(previous).catch((restoreErr) => {
+ console.error(`❌ llama-server: could not restore the previous configuration: ${restoreErr.message}`);
+ return null;
+ });
+ return { failure: err.message, config: restored?.config || null };
+ });
+ if (started.failure) {
+ return { applied: false, reason: `llama-server rejected that tuning: ${started.failure}`, config: started.config };
+ }
+
+ // PM2 reporting `online` is not the same as the server answering. But
+ // `startLlamaServer` only polls for four seconds, and a large GGUF routinely
+ // takes longer than that to load — so `online: false` is "not ready YET",
+ // not "wedged". Give it a real readiness budget before judging.
+ const ready = started.online || await waitForEndpoint(started.endpoint);
+ if (!ready) {
+ // Still silent. Treat it exactly like a rejected launch line: put the
+ // previous configuration back, so the install's llama provider is not left
+ // pointing at a process that never serves. Without this the caller would go
+ // on to measure a dead endpoint and record the timeouts as evidence.
+ console.error('❌ llama-server: relaunched process never answered — restoring the previous configuration');
+ await stopLlamaServer().catch(() => {});
+ await waitForPortRelease(previous.port ?? PORTS.LLAMA_SERVER);
+ const restored = await startLlamaServer(previous).catch((err) => {
+ console.error(`❌ llama-server: could not restore the previous configuration: ${err.message}`);
+ return null;
+ });
+ return {
+ applied: false,
+ reason: 'llama-server relaunched but never answered on its port',
+ config: restored?.config || null,
+ };
+ }
+ return { applied: true, reason: null, config: started.config };
+}
+
/**
* Runs `brew link --overwrite llama.cpp`, resolving `{ linked, output }` on exit
* rather than rejecting — a failed link attempt should fall through to the
@@ -493,9 +689,11 @@ export async function installLlamaServer({ onProgress = () => {} } = {}) {
/**
* Clears in-memory test state (used by test suites).
*/
-export function _resetLlamaServerStateForTests() {
+export function _resetLlamaServerStateForTests({ relaunchReadyTimeout } = {}) {
currentConfig = null;
recentLogs = [];
lastExitError = null;
+ // Restored to the production budget unless a suite asks for a shorter one.
+ relaunchReadyTimeoutMs = Number.isFinite(relaunchReadyTimeout) ? relaunchReadyTimeout : 120000;
}
diff --git a/server/services/llamaServerManager.test.js b/server/services/llamaServerManager.test.js
index 0b451d39a..f55c99f82 100644
--- a/server/services/llamaServerManager.test.js
+++ b/server/services/llamaServerManager.test.js
@@ -3,6 +3,7 @@ import {
getLlamaServerStatus,
startLlamaServer,
stopLlamaServer,
+ relaunchLlamaServerWithTuning,
installLlamaServer,
_resetLlamaServerStateForTests,
LLAMA_APP,
@@ -453,4 +454,121 @@ describe('llamaServerManager', () => {
await expect(installLlamaServer()).rejects.toThrow(/brew link --overwrite llama\.cpp/i);
});
+
+ // ---- tuning relaunch ----------------------------------------------------
+ // The sweep half of measured assessments: put new flags on the launch line
+ // between runs. Every failure mode here has to leave the daemon USABLE — it
+ // fronts the `llama` provider for the whole install.
+ describe('relaunchLlamaServerWithTuning', () => {
+ // The default harness pins the endpoint unreachable so lifecycle tests don't
+ // collide with a developer's real llama-server. These tests need a FAITHFUL
+ // probe instead: "reachable" has to track the fake PM2 process, or the
+ // relaunch can never observe the server it just started answering — and the
+ // new `online` check would report every success as not-applied.
+ const started = async () => {
+ vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
+ vi.spyOn(openAiModelsProbe, 'probeOpenAiModels')
+ .mockImplementation(async () => ({ reachable: pm2State?.status === 'online' }));
+ await startLlamaServer({ model: modelPath, port: PORTS.LLAMA_SERVER });
+ };
+
+ it('refuses when nothing is running — there is no model path to reuse', async () => {
+ vi.spyOn(processEnv, 'findCommandOnPath').mockReturnValue('/usr/local/bin/llama-server');
+ const result = await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
+ expect(result.applied).toBe(false);
+ expect(result.reason).toMatch(/not running/);
+ });
+
+ it('refuses an empty tuning rather than restarting for no reason', async () => {
+ await started();
+ execPm2Calls = [];
+ const result = await relaunchLlamaServerWithTuning({});
+ expect(result.applied).toBe(false);
+ expect(execPm2Calls).toEqual([]);
+ });
+
+ it('puts the knobs on the new launch line, keeping the model it was serving', async () => {
+ await started();
+ execPm2Calls = [];
+ const result = await relaunchLlamaServerWithTuning({ ubatchSize: 512, flashAttn: true });
+ expect(result.applied).toBe(true);
+ const start = execPm2Calls.find((c) => c[0] === 'start');
+ expect(start).toContain('-ub');
+ expect(start[start.indexOf('-ub') + 1]).toBe('512');
+ expect(start).toContain('--flash-attn');
+ expect(start[start.indexOf('-m') + 1]).toBe(modelPath);
+ });
+
+ // A sweep is EXPECTED to produce launch lines llama.cpp rejects. Leaving the
+ // daemon down would break every later request, not just this measurement.
+ it('restores the previous configuration when the tuned launch line exits', async () => {
+ await started();
+ // The harness's fake, captured as a raw function — re-spying and calling
+ // `pm2Module.execPm2` would re-enter this wrapper and blow the stack.
+ const fakeExec = pm2Module.execPm2.getMockImplementation();
+ let starts = 0;
+ vi.spyOn(pm2Module, 'execPm2').mockImplementation(async (args) => {
+ // Fail only the FIRST start after the relaunch (the tuned one); the
+ // restore that follows must succeed.
+ if (args[0] === 'start' && starts++ === 0) {
+ pm2State = { name: LLAMA_APP, status: 'errored', pid: null, args: [] };
+ execPm2Calls.push(args);
+ return { stdout: '', stderr: '' };
+ }
+ return fakeExec(args);
+ });
+ execPm2Calls = [];
+
+ const result = await relaunchLlamaServerWithTuning({ ubatchSize: 999999 });
+ expect(result.applied).toBe(false);
+ expect(result.reason).toMatch(/rejected that tuning/);
+ // The restore ran, and it carried the ORIGINAL model with no `-ub`.
+ const restore = execPm2Calls.filter((c) => c[0] === 'start').at(-1);
+ expect(restore).not.toContain('-ub');
+ expect(restore[restore.indexOf('-m') + 1]).toBe(modelPath);
+ });
+
+ // PM2 reporting `online` is not the same as the server answering. A daemon
+ // that never opened its port has not had the tuning applied in any sense a
+ // measurement could rest on.
+ // `startLlamaServer` polls for only four seconds, and a large GGUF routinely
+ // takes longer than that to load. Treating "not ready yet" as "wedged" would
+ // tear down a launch that was about to succeed.
+ it('waits past the start probe for a slow load rather than calling it wedged', async () => {
+ await started();
+ let answerAfter = 3;
+ vi.spyOn(openAiModelsProbe, 'probeOpenAiModels').mockImplementation(async () => {
+ if (pm2State?.status !== 'online') return { reachable: false };
+ return { reachable: answerAfter-- <= 0 };
+ });
+ execPm2Calls = [];
+ const result = await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
+ expect(result.applied).toBe(true);
+ // One start only — the slow load was waited out, not restarted.
+ expect(execPm2Calls.filter((c) => c[0] === 'start')).toHaveLength(1);
+ });
+
+ it('reports not-applied when the relaunched server never answers', async () => {
+ await started();
+ // PM2 keeps reporting `online` while the endpoint stays silent — the exact
+ // split the check exists for.
+ vi.spyOn(openAiModelsProbe, 'probeOpenAiModels').mockResolvedValue({ reachable: false });
+ // Shrink the readiness budget: the give-up path is what's under test, and
+ // the production two minutes would just be two minutes of sleeping.
+ _resetLlamaServerStateForTests({ relaunchReadyTimeout: 1500 });
+ execPm2Calls = [];
+ const result = await relaunchLlamaServerWithTuning({ ubatchSize: 512 });
+ expect(result.applied).toBe(false);
+ expect(result.reason).toMatch(/never answered/);
+ // A silent process must not be LEFT running: this daemon fronts the llama
+ // provider for the whole install, so the previous configuration goes back
+ // exactly as it does for a launch line llama.cpp rejects outright.
+ const restore = execPm2Calls.filter((c) => c[0] === 'start').at(-1);
+ expect(restore).not.toContain('-ub');
+ expect(restore[restore.indexOf('-m') + 1]).toBe(modelPath);
+ // Two full start cycles (each polling `STARTUP_WAIT_TIMEOUT_MS` against a
+ // deliberately-silent probe) plus the readiness budget — slow by design,
+ // not by accident, so this one test buys the room rather than the suite.
+ }, 30000);
+ });
});
diff --git a/server/services/localLlmPlayground.js b/server/services/localLlmPlayground.js
index 8db576566..f1f98565e 100644
--- a/server/services/localLlmPlayground.js
+++ b/server/services/localLlmPlayground.js
@@ -4,8 +4,10 @@ import { ensureBackendProvider } from './localLlm.js';
import { getProviderById } from './providers.js';
import { markProviderAvailable } from './providerStatus.js';
import { ensureProviderReady as ensureOllamaProviderReady } from './ollamaManager.js';
-import { readResponseJson } from '../lib/readResponseJson.js';
import { anyAbortSignal } from '../lib/requestAbort.js';
+// The SSE read loop lives in `lib/openAiChatStream.js` so the assessments
+// service can measure a bare loopback daemon that has no provider record.
+import { buildMessages, streamOpenAiChat } from '../lib/openAiChatStream.js';
import { assertSecretEndpoint } from '../lib/aiToolkit/internal/endpointGuard.js';
const PROVIDER_BY_BACKEND = { ollama: 'ollama', lmstudio: 'lmstudio' };
@@ -37,45 +39,6 @@ export function summarizeTimings({ startedAt, firstChunkAt, endedAt, text }) {
};
}
-// Parse one OpenAI-style SSE `data:` line into its content/reasoning delta.
-// Returns null for non-data lines, the [DONE]/✅ sentinels, or a malformed
-// frame: a single bad frame must SKIP, not abort the stream — one non-JSON
-// keep-alive would otherwise throw out of the read loop and discard every
-// token already received.
-export function extractStreamDelta(rawLine) {
- const line = rawLine.trim();
- if (!line.startsWith('data: ')) return null;
- const data = line.slice(6).trim();
- if (!data || data === '[DONE]' || data === '✅') return null;
- let parsed;
- try {
- parsed = JSON.parse(data);
- } catch {
- return null;
- }
- const delta = parsed?.choices?.[0]?.delta;
- return { content: delta?.content || '', reasoning: delta?.reasoning || '' };
-}
-
-export function buildMessages({ systemPrompt, prompt }) {
- const system = String(systemPrompt || '').trim();
- return [
- ...(system ? [{ role: 'system', content: system }] : []),
- { role: 'user', content: prompt },
- ];
-}
-
-// Resolve the text to surface from a (possibly interrupted) stream: prefer the
-// visible content, fall back to reasoning when no content arrived (some models
-// emit only a reasoning channel), and '' when neither did. Used on both the
-// normal-finish path and the partial-output-on-throw path so a timed-out run
-// still shows what streamed before the abort.
-export function resolvePartialOutput({ output = '', reasoning = '' }) {
- if (output.trim()) return output;
- if (reasoning.trim()) return reasoning;
- return '';
-}
-
async function resolveLocalProvider(backend) {
const providerId = PROVIDER_BY_BACKEND[backend];
if (!providerId) {
@@ -94,7 +57,7 @@ async function resolveLocalProvider(backend) {
return provider;
}
-async function streamChatCompletion({ provider, backend, modelId, prompt, systemPrompt, temperature, maxTokens, signal, onChunk }) {
+async function streamChatCompletion({ provider, backend, modelId, prompt, systemPrompt, temperature, maxTokens, extraBody = {}, signal, onChunk }) {
if (backend === 'ollama') {
const ready = await ensureOllamaProviderReady(provider).catch((err) => ({ success: false, error: err.message }));
if (!ready.success) {
@@ -110,92 +73,19 @@ async function streamChatCompletion({ provider, backend, modelId, prompt, system
allowCustomEndpoint: provider.allowCustomEndpoint === true,
});
- const headers = { 'Content-Type': 'application/json' };
- if (provider.apiKey) headers.Authorization = `Bearer ${provider.apiKey}`;
-
- const response = await fetch(`${provider.endpoint}/chat/completions`, {
- method: 'POST',
- headers,
+ return streamOpenAiChat({
+ endpoint: provider.endpoint,
+ apiKey: provider.apiKey,
+ model: modelId,
+ messages: buildMessages({ systemPrompt, prompt }),
+ temperature,
+ maxTokens,
+ // The caller's knobs win over the provider default: an assessment measuring
+ // a specific `num_ctx` must not silently be run at the provider's.
+ extraBody: { ...(Number(provider.numCtx) > 0 ? { num_ctx: Number(provider.numCtx) } : {}), ...extraBody },
signal,
- body: JSON.stringify({
- model: modelId,
- messages: buildMessages({ systemPrompt, prompt }),
- stream: true,
- temperature,
- max_tokens: maxTokens,
- ...(Number(provider.numCtx) > 0 ? { num_ctx: Number(provider.numCtx) } : {}),
- }),
- }).catch((err) => ({ ok: false, status: 0, error: err.message }));
-
- if (!response.ok) {
- const body = response.text ? await response.text().catch(() => '') : response.error || '';
- throw new Error(`Provider returned ${response.status || 0}: ${body || response.error || response.statusText || 'request failed'}`);
- }
-
- if (!response.body?.getReader) {
- // Sentinel fallback: a non-JSON/blank 200 body must throw (caught by
- // runLocalLlmTest → finalizeRunRecord success:false), not return '' — which
- // line 205 would persist as a successful empty run. Mirrors the pre-helper
- // response.json() throw; a valid body with empty content is unchanged.
- const data = await readResponseJson(response, { fallback: null, emptyValue: null });
- if (!data) {
- throw new Error(`Provider returned a non-JSON response (${response.status})`);
- }
- const text = data.choices?.[0]?.message?.content || '';
- if (text) await onChunk(text, 'content');
- return text;
- }
-
- const reader = response.body.getReader();
- const decoder = new TextDecoder();
- let buffer = '';
- let output = '';
- let reasoning = '';
-
- const consumeLine = async (rawLine) => {
- const delta = extractStreamDelta(rawLine);
- if (!delta) return;
- if (delta.content) {
- output += delta.content;
- await onChunk(delta.content, 'content');
- }
- // Stream reasoning live too, on its own channel, so a reasoning-only model
- // (deepseek-r1, qwq, …) renders its chain-of-thought as it arrives instead
- // of sitting on "Waiting for the first token…" until the whole run lands.
- // Kept distinct from content so the live panel can label it and so the
- // final content-only `text` doesn't inherit reasoning prose.
- if (delta.reasoning) {
- reasoning += delta.reasoning;
- await onChunk(delta.reasoning, 'reasoning');
- }
- };
-
- // Always release the reader (and tear down the socket) on every exit path —
- // a normal finish, an abort via the timeout signal, or a throw mid-stream.
- // On a throw (e.g. an AbortError from the timeout) surface the tokens already
- // streamed by attaching them to the error, so runLocalLlmTest can render the
- // partial output alongside the timeout message instead of discarding it.
- try {
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
- buffer += decoder.decode(value, { stream: true });
- const lines = buffer.split('\n');
- buffer = lines.pop() || '';
- for (const line of lines) await consumeLine(line);
- }
- if (buffer.trim()) await consumeLine(buffer);
- } catch (err) {
- err.partialOutput = resolvePartialOutput({ output, reasoning });
- throw err;
- } finally {
- await reader.cancel().catch(() => {});
- }
-
- // Both channels already streamed live via onChunk, so there's no end-of-stream
- // re-emit here — re-pushing `resolved` would double the reasoning-only output
- // the client just received. We only resolve the final text for the result record.
- return resolvePartialOutput({ output, reasoning });
+ onChunk,
+ });
}
export async function runLocalLlmTest({
@@ -206,6 +96,9 @@ export async function runLocalLlmTest({
temperature = 0.3,
maxTokens = 1000,
timeoutMs = 300000,
+ // Backend-specific request knobs merged into the chat-completions body (see
+ // `lib/localModelTuning.js#requestBody`). Empty for a plain playground run.
+ extraBody = {},
signal: clientSignal,
// Optional per-token callback `onToken(delta, kind)` where kind is 'content'
// or 'reasoning'. When provided (streaming route), each delta is forwarded as
@@ -249,6 +142,7 @@ export async function runLocalLlmTest({
systemPrompt,
temperature,
maxTokens,
+ extraBody,
signal,
onChunk: (chunk, kind = 'content') => {
// First token of EITHER channel marks TTFT: for a reasoning model the
@@ -307,6 +201,82 @@ export async function runLocalLlmTest({
}
}
+/**
+ * Measure one generation against a bare OpenAI-compatible loopback daemon —
+ * llama.cpp, MTPLX, or vLLM — that PortOS does NOT hold a provider record for.
+ *
+ * Returns the same shape as `runLocalLlmTest` (text / error / timings) so the
+ * assessment sampler treats every runtime identically. What it deliberately
+ * does NOT do is create a `/runs` record: `createRun` resolves a configured
+ * provider, and inventing one for a daemon the user started outside PortOS
+ * would put a phantom provider in the runs history.
+ *
+ * @param {object} options
+ * @param {string} options.runtime runtime id, echoed back on the result
+ * @param {string} options.endpoint OpenAI-compatible base ending in `/v1`
+ */
+export async function runEndpointLlmTest({
+ runtime,
+ endpoint,
+ // Empty for the usual unauthenticated loopback daemon; set for a vLLM
+ // container started behind `VLLM_API_KEY`, which 401s without it.
+ apiKey = '',
+ modelId,
+ prompt,
+ systemPrompt = '',
+ temperature = 0.3,
+ maxTokens = 1000,
+ timeoutMs = 300000,
+ extraBody = {},
+ signal: clientSignal,
+ onToken,
+}) {
+ const startedAt = Date.now();
+ let firstChunkAt = null;
+ const timeoutController = new AbortController();
+ const timeoutHandle = setTimeout(() => timeoutController.abort(), timeoutMs);
+ const signal = anyAbortSignal([clientSignal, timeoutController.signal]);
+
+ try {
+ const text = await streamOpenAiChat({
+ endpoint,
+ apiKey,
+ model: modelId,
+ messages: buildMessages({ systemPrompt, prompt }),
+ temperature,
+ maxTokens,
+ extraBody,
+ signal,
+ onChunk: (chunk, kind = 'content') => {
+ if (!firstChunkAt && chunk) firstChunkAt = Date.now();
+ if (chunk) return onToken?.(chunk, kind);
+ return undefined;
+ },
+ }).finally(() => clearTimeout(timeoutHandle));
+ return {
+ backend: runtime,
+ modelId,
+ endpoint,
+ text,
+ timings: summarizeTimings({ startedAt, firstChunkAt, endedAt: Date.now(), text }),
+ options: { temperature, maxTokens, timeoutMs },
+ };
+ } catch (err) {
+ clearTimeout(timeoutHandle);
+ const error = err?.name === 'AbortError' ? `Timed out after ${timeoutMs}ms` : err?.message || 'Local LLM test failed';
+ const partialText = typeof err?.partialOutput === 'string' ? err.partialOutput : '';
+ return {
+ backend: runtime,
+ modelId,
+ endpoint,
+ error,
+ text: partialText,
+ timings: summarizeTimings({ startedAt, firstChunkAt, endedAt: Date.now(), text: partialText }),
+ options: { temperature, maxTokens, timeoutMs },
+ };
+ }
+}
+
export async function compareLocalLlmModels({ targets, prompt, mode = 'round-robin', options = {}, signal }) {
const runOne = (target) => runLocalLlmTest({ ...options, ...target, prompt, signal });
const results = [];
diff --git a/server/services/localLlmPlayground.test.js b/server/services/localLlmPlayground.test.js
index 2c12fb1a2..b62f1e070 100644
--- a/server/services/localLlmPlayground.test.js
+++ b/server/services/localLlmPlayground.test.js
@@ -6,7 +6,7 @@ vi.mock('./providers.js', () => ({ getProviderById: vi.fn() }));
vi.mock('./providerStatus.js', () => ({ markProviderAvailable: vi.fn(() => Promise.resolve()) }));
vi.mock('./ollamaManager.js', () => ({ ensureProviderReady: vi.fn(() => Promise.resolve({ success: true })) }));
-import { buildPrompt, buildMessages, summarizeTimings, extractStreamDelta, resolvePartialOutput, runLocalLlmTest } from './localLlmPlayground.js';
+import { buildPrompt, summarizeTimings, runLocalLlmTest } from './localLlmPlayground.js';
import { createRun, finalizeRunRecord } from './runner.js';
import { getProviderById } from './providers.js';
@@ -22,21 +22,6 @@ describe('buildPrompt', () => {
});
});
-describe('buildMessages', () => {
- it('omits the system message when blank', () => {
- expect(buildMessages({ systemPrompt: ' ', prompt: 'hi' })).toEqual([
- { role: 'user', content: 'hi' },
- ]);
- });
-
- it('includes a system message when present', () => {
- expect(buildMessages({ systemPrompt: 'Be terse', prompt: 'hi' })).toEqual([
- { role: 'system', content: 'Be terse' },
- { role: 'user', content: 'hi' },
- ]);
- });
-});
-
describe('summarizeTimings', () => {
it('computes ttft, total, and rate', () => {
const t = summarizeTimings({ startedAt: 1000, firstChunkAt: 1200, endedAt: 3000, text: 'abcdefghij' });
@@ -58,48 +43,6 @@ describe('summarizeTimings', () => {
});
});
-describe('extractStreamDelta', () => {
- it('parses an OpenAI-style content delta', () => {
- const line = 'data: {"choices":[{"delta":{"content":"Hi"}}]}';
- expect(extractStreamDelta(line)).toEqual({ content: 'Hi', reasoning: '' });
- });
-
- it('parses a reasoning delta', () => {
- const line = 'data: {"choices":[{"delta":{"reasoning":"thinking"}}]}';
- expect(extractStreamDelta(line)).toEqual({ content: '', reasoning: 'thinking' });
- });
-
- it('skips non-data lines and the [DONE]/✅ sentinels', () => {
- expect(extractStreamDelta(': keep-alive')).toBeNull();
- expect(extractStreamDelta('data: [DONE]')).toBeNull();
- expect(extractStreamDelta('data: ✅')).toBeNull();
- expect(extractStreamDelta('')).toBeNull();
- });
-
- it('skips a malformed frame instead of throwing (one bad frame must not abort the stream)', () => {
- expect(extractStreamDelta('data: {not json')).toBeNull();
- });
-
- it('tolerates a frame with no delta', () => {
- expect(extractStreamDelta('data: {"choices":[{}]}')).toEqual({ content: '', reasoning: '' });
- });
-});
-
-describe('resolvePartialOutput', () => {
- it('prefers visible content over reasoning', () => {
- expect(resolvePartialOutput({ output: 'hello', reasoning: 'thinking' })).toBe('hello');
- });
-
- it('falls back to reasoning when no content streamed', () => {
- expect(resolvePartialOutput({ output: ' ', reasoning: 'partial thought' })).toBe('partial thought');
- });
-
- it('returns empty string when neither content nor reasoning streamed', () => {
- expect(resolvePartialOutput({ output: '', reasoning: '' })).toBe('');
- expect(resolvePartialOutput({})).toBe('');
- });
-});
-
// Build a fake stream reader: yields each SSE line as a chunk, then either
// finishes cleanly (done) or throws an AbortError to simulate a timeout.
function makeReader(lines, { abort = false } = {}) {
diff --git a/server/services/localModelAssessmentStore.js b/server/services/localModelAssessmentStore.js
index ebebb0124..ad96c22b3 100644
--- a/server/services/localModelAssessmentStore.js
+++ b/server/services/localModelAssessmentStore.js
@@ -34,6 +34,7 @@ import os from 'os';
import { PATHS, atomicWrite, ensureDir, tryReadFile, safeJSONParse } from '../lib/fileUtils.js';
import { getAvailableMemoryGb } from '../lib/localMemory.js';
import { compareEnvironments, describeStaleness, measuredFitVerdict } from '../lib/localModelAssessment.js';
+import { ASSESSABLE_RUNTIMES } from '../lib/localProviderRuntime.js';
import { getVersion as getOllamaVersion } from './ollamaManager.js';
// Resolved lazily, not at import time: `PATHS.data` is patched by suites that
@@ -132,7 +133,7 @@ export function __resetBackendVersionCache() {
* @param {{ backends?: string[] }} [options]
* @returns {Promise>} keyed by backend
*/
-export async function captureLiveEnvironments({ backends = ['ollama', 'lmstudio'] } = {}) {
+export async function captureLiveEnvironments({ backends = ASSESSABLE_RUNTIMES } = {}) {
const durable = captureDurableEnvironment();
const entries = await Promise.all(backends.map(async (backend) => [
backend,
@@ -143,7 +144,19 @@ export async function captureLiveEnvironments({ backends = ['ollama', 'lmstudio'
// ---- store ------------------------------------------------------------------
-export const assessmentKey = (backend, modelId) => `${backend}:${modelId}`;
+/**
+ * Identity of one stored measurement.
+ *
+ * `tuningKey` is the stable signature from `lib/localModelTuning.js`. An UNTUNED
+ * measurement passes `''` and keys exactly as it did before tuning existed — so
+ * every record already on disk keeps resolving with no migration, and a tuned
+ * run of the same model lands beside it instead of overwriting it.
+ */
+export const assessmentKey = (backend, modelId, tuningKey = '') =>
+ (tuningKey ? `${backend}:${modelId}@${tuningKey}` : `${backend}:${modelId}`);
+
+/** The key of a stored record, from whichever fields it carries. */
+export const keyOfAssessment = (a) => assessmentKey(a?.backend, a?.modelId, a?.tuningKey || '');
// Move an unparseable store aside so a fresh one can be written without losing
// whatever the old file held. Best-effort: if the rename fails there is nothing
@@ -189,11 +202,13 @@ export async function saveAssessment(assessment) {
// the unreadable file instead: nothing is lost, and the feature keeps working
// rather than wedging on a file the user has no way to repair from the UI.
if (readError) await quarantineStore(readError);
- const key = assessmentKey(assessment.backend, assessment.modelId);
- // One record per (backend, model): the newest measurement supersedes the old
- // one. History is not kept — a stale reading from a different memory state is
- // worse than no reading, and the run is cheap to repeat.
- const next = assessments.filter((a) => assessmentKey(a?.backend, a?.modelId) !== key);
+ const key = keyOfAssessment(assessment);
+ // One record per (backend, model, TUNING): the newest measurement supersedes
+ // the old one for that exact configuration. History within one tuning is not
+ // kept — a stale reading from a different memory state is worse than no
+ // reading, and the run is cheap to repeat. Two DIFFERENT tunings are two
+ // different answers to two different questions, so they both survive.
+ const next = assessments.filter((a) => keyOfAssessment(a) !== key);
next.push(assessment);
await ensureDir(assessmentsDir());
await atomicWrite(assessmentsFile(), { schemaVersion: STORE_SCHEMA_VERSION, assessments: next });
@@ -204,17 +219,17 @@ export async function saveAssessment(assessment) {
* Drop one recorded assessment. Returns whether a record was actually removed,
* so the caller can 404 rather than reporting a phantom success.
*/
-export async function deleteAssessment(backend, modelId) {
+export async function deleteAssessment(backend, modelId, tuningKey = '') {
const { assessments, readError } = await loadStore();
// Same hazard as saveAssessment: rewriting from an empty in-memory list would
// wipe the file. A delete against an unreadable store has nothing to remove.
if (readError) return { deleted: false };
- const key = assessmentKey(backend, modelId);
- const next = assessments.filter((a) => assessmentKey(a?.backend, a?.modelId) !== key);
+ const key = assessmentKey(backend, modelId, tuningKey);
+ const next = assessments.filter((a) => keyOfAssessment(a) !== key);
if (next.length === assessments.length) return { deleted: false };
await ensureDir(assessmentsDir());
await atomicWrite(assessmentsFile(), { schemaVersion: STORE_SCHEMA_VERSION, assessments: next });
- console.log(`🧹 Local LLM: dropped assessment for ${backend}/${modelId}`);
+ console.log(`🧹 Local LLM: dropped assessment for ${backend}/${modelId}${tuningKey ? ` (${tuningKey})` : ''}`);
return { deleted: true };
}
@@ -250,8 +265,17 @@ export async function getMeasuredFits(backend) {
const { assessments } = await loadStore();
const live = { ...captureDurableEnvironment(), backendVersion: await liveBackendVersion(backend) };
const out = {};
- for (const assessment of assessments) {
+ // A model can now hold several measurements (one per tuning). The badge has
+ // room for ONE, so the newest wins: it describes the configuration the user
+ // most recently cared about, and every tuning is visible in full on the
+ // Performance page. Records with no timestamp sort last rather than
+ // masquerading as the newest.
+ const newestFirst = [...assessments].sort(
+ (a, b) => String(b?.assessedAt || '').localeCompare(String(a?.assessedAt || ''))
+ );
+ for (const assessment of newestFirst) {
if (assessment?.backend !== backend || !assessment?.modelId) continue;
+ if (out[assessment.modelId]) continue;
const staleness = compareEnvironments(assessment.environment, live);
out[assessment.modelId] = {
fit: measuredFitVerdict(assessment),
@@ -271,6 +295,10 @@ export async function getMeasuredFits(backend) {
// field (or the backend reported none), and a consumer must then decline
// to match a quantized variant rather than guess.
quantization: assessment.quantization ?? null,
+ // Which launch configuration produced this reading. `null` = the record
+ // predates tuning (or ran on backend defaults), which is a real answer —
+ // not an unknown.
+ tuningLabel: assessment.tuningLabel ?? null,
};
}
return out;
diff --git a/server/services/localModelAssessments.js b/server/services/localModelAssessments.js
index b6ceb0f3c..1ba31a6e0 100644
--- a/server/services/localModelAssessments.js
+++ b/server/services/localModelAssessments.js
@@ -57,7 +57,26 @@ import {
saveAssessment,
withStaleness,
} from './localModelAssessmentStore.js';
-import { runLocalLlmTest } from './localLlmPlayground.js';
+import {
+ ASSESSABLE_RUNTIMES,
+ LOCAL_RUNTIMES,
+ MANAGED_ASSESSMENT_BACKENDS,
+ isEndpointRuntime,
+ localRuntimeKind,
+} from '../lib/localProviderRuntime.js';
+import {
+ compareTunings,
+ describeTuning,
+ launchTuning,
+ normalizeTuning,
+ requestBody,
+ tuningSignature,
+ tuningSpecsFor,
+} from '../lib/localModelTuning.js';
+import { probeOpenAiModels } from '../lib/openAiModelsProbe.js';
+import { getAllProviders } from './providers.js';
+import { runEndpointLlmTest, runLocalLlmTest } from './localLlmPlayground.js';
+import { getLlamaServerEndpoint, relaunchLlamaServerWithTuning } from './llamaServerManager.js';
import { listModels } from './localLlm.js';
import {
getLoadedModels as getLoadedOllamaModels,
@@ -118,6 +137,83 @@ export function buildSamplePrompt(contextTokens) {
return `${filler}\nIgnoring every reference item above, what is 2 + 2? Answer with the number only.`;
}
+// ---- runtimes ---------------------------------------------------------------
+
+/**
+ * Where this runtime's OpenAI-compatible API actually is right now.
+ *
+ * llama.cpp is the one that moves: PortOS starts it, and a user who picked a
+ * different port under Advanced options is serving somewhere the default no
+ * longer names. Ask the manager rather than re-deriving the port here — probing
+ * the stale default would report a working server as unreachable.
+ */
+export async function runtimeEndpoint(runtime) {
+ if (runtime === 'llama') {
+ // The endpoint-only accessor, NOT `getLlamaServerStatus` — that one pays for
+ // a network probe and an `execPm2 logs` subprocess, and this path runs on
+ // every Performance page load only to learn a port number.
+ const endpoint = await getLlamaServerEndpoint().catch(() => null);
+ if (endpoint) return endpoint;
+ }
+ return LOCAL_RUNTIMES[runtime]?.defaultBaseUrl || null;
+}
+
+/**
+ * The API key a bare endpoint runtime is served behind, or `''` for the usual
+ * unauthenticated loopback daemon.
+ *
+ * A vLLM container started from the shipped compose stack sets `VLLM_API_KEY`
+ * and answers 401 to an unauthenticated request — which `probeOpenAiModels`
+ * correctly reports as "reachable, listing unreadable" and a measurement would
+ * hit on every sample. `providerReadiness.js` already solves this by reading the
+ * key off the matching provider record; this resolves it the same way, keyed on
+ * the same `localRuntimeKind` classifier so the two can't disagree about which
+ * provider backs which runtime.
+ */
+export async function runtimeApiKey(runtime) {
+ const providers = await getAllProviders().catch(() => []);
+ const match = (Array.isArray(providers) ? providers : [])
+ .find((p) => localRuntimeKind(p) === runtime && typeof p?.apiKey === 'string' && p.apiKey !== '');
+ return match?.apiKey || '';
+}
+
+/**
+ * Models this runtime can be measured against, plus why the list failed when it
+ * did.
+ *
+ * The two paths differ in what "installed" even means. A managed backend has a
+ * durable catalog on disk (`listModels`), so its list survives the daemon being
+ * down. An endpoint runtime has no catalog at all — its models are whatever the
+ * running process reports from `GET /v1/models`, so a stopped daemon means "no
+ * models listable", which is an ERROR, never an empty catalog. Collapsing those
+ * would silently hide every model behind a daemon the user just needs to start.
+ *
+ * @returns {Promise<{models: Array|null, error: string|null}>}
+ * `models: null` means the list could not be read; `[]` means it was read and
+ * is genuinely empty.
+ */
+export async function listRuntimeModels(runtime) {
+ if (MANAGED_ASSESSMENT_BACKENDS.includes(runtime)) {
+ const models = await listModels(runtime).catch((err) => ({ error: err?.message || 'model list failed' }));
+ if (!Array.isArray(models)) return { models: null, error: models.error };
+ // Both managers cache an EMPTY list on a failed read rather than throwing,
+ // so `[]` alone cannot distinguish "no models" from "the list could not be
+ // read". Each manager's own error getter is the authoritative signal.
+ const error = runtime === 'ollama' ? getOllamaListError() : getLmStudioListError();
+ return { models, error: error || null };
+ }
+
+ const endpoint = await runtimeEndpoint(runtime);
+ if (!endpoint) return { models: null, error: 'no endpoint is configured for this runtime' };
+ const probe = await probeOpenAiModels(endpoint, { timeoutMs: 2500, apiKey: await runtimeApiKey(runtime) });
+ if (!probe.reachable) return { models: null, error: `not reachable at ${endpoint} (${probe.error})` };
+ if (!probe.models) return { models: null, error: probe.error || 'model listing was not readable' };
+ // An endpoint runtime reports ids only — no params, no quantization. `null`
+ // there is honest: the capability axis simply goes unscored rather than being
+ // guessed from the id.
+ return { models: probe.models.map((id) => ({ id, params: null, quantization: null })), error: null };
+}
+
// ---- measurement ------------------------------------------------------------
/**
@@ -178,9 +274,14 @@ function describeVerdict(verdict, samples) {
* failure stops immediately for the same reason.
*
* @param {object} options
- * @param {'ollama'|'lmstudio'} options.backend
+ * @param {'ollama'|'lmstudio'|'llama'|'mtplx'|'vllm'} options.backend
* @param {string} options.modelId
* @param {number[]} [options.contextTokens] nominal context sizes to sample
+ * @param {object} [options.tuning] launch/runtime knobs (`lib/localModelTuning.js`).
+ * Launch knobs are applied where PortOS starts the daemon (llama.cpp); the
+ * rest are recorded so two readings of one model stay comparable. The tuning
+ * is part of the record's identity, so a second tuning of the same model is a
+ * NEW record rather than an overwrite.
* @param {AbortSignal} [options.signal] client disconnect
* @param {(frame: object) => void} [options.onProgress] per-sample progress.
* A run is minutes long on a large model, so the caller (the route) forwards
@@ -189,7 +290,7 @@ function describeVerdict(verdict, samples) {
* THIS run apart from an unrelated model install streaming on the same event.
* @returns {Promise} the persisted assessment record
*/
-export async function runAssessment({ backend, modelId, contextTokens = DEFAULT_CONTEXT_TOKENS, signal, onProgress } = {}) {
+export async function runAssessment({ backend, modelId, contextTokens = DEFAULT_CONTEXT_TOKENS, tuning, signal, onProgress } = {}) {
const contexts = [...new Set(contextTokens)].filter((n) => Number.isFinite(n) && n > 0).sort((a, b) => a - b);
// The listener runs outside the request lifecycle's error path in some callers
// (a socket emit can throw on a closed io), and a broken progress consumer must
@@ -199,11 +300,38 @@ export async function runAssessment({ backend, modelId, contextTokens = DEFAULT_
try { onProgress({ scope: 'assessment', backend, modelId, ...frame }); }
catch (err) { console.error(`❌ Local LLM: assessment progress listener failed: ${err.message}`); }
};
+ const normalizedTuning = normalizeTuning(backend, tuning);
+ const tuningKey = tuningSignature(normalizedTuning);
+ const tuningLabel = describeTuning(backend, normalizedTuning);
+
+ // Launch knobs go on the daemon's command line BEFORE the first sample, or
+ // the measurement would describe the previous configuration while claiming
+ // the new one. `applied: false` is recorded rather than swallowed — a reading
+ // taken under a tuning PortOS could not apply must not be filed as evidence
+ // for that tuning.
+ const launch = launchTuning(backend, normalizedTuning);
+ const applicable = { ...launch, ...requestBody(backend, normalizedTuning) };
+ const tuningApplication = Object.keys(launch).length > 0
+ ? await relaunchLlamaServerWithTuning(launch).catch((err) => ({ applied: false, reason: err?.message || 'relaunch failed', config: null }))
+ // `null`, NOT `true`, when nothing here is settable: an all-`record` tuning
+ // (LM Studio, MTPLX, vLLM) was never applied by PortOS in any sense, and
+ // reporting it as applied is the exact claim `lib/localModelTuning.js`
+ // forbids. `true` is reserved for knobs that actually reached the daemon.
+ : { applied: Object.keys(applicable).length > 0 ? true : null, reason: null, config: null };
+
+ const endpoint = isEndpointRuntime(backend) ? await runtimeEndpoint(backend) : null;
+ if (isEndpointRuntime(backend) && !endpoint) {
+ throw new Error(`No endpoint is configured for the ${backend} runtime`);
+ }
+ // Same key the listing probe used — a key-gated vLLM 401s every sample
+ // otherwise, and the run would record "does-not-fit" for an auth failure.
+ const apiKey = endpoint ? await runtimeApiKey(backend) : '';
+
const environment = await captureEnvironment({ backend });
- const installed = await listModels(backend).catch(() => []);
- const card = installed.find((m) => m?.id === modelId) || null;
+ const { models: installed } = await listRuntimeModels(backend);
+ const card = (installed || []).find((m) => m?.id === modelId) || null;
- console.log(`📏 Local LLM: assessing ${backend}/${modelId} across ${contexts.length} context sizes`);
+ console.log(`📏 Local LLM: assessing ${backend}/${modelId}${tuningKey ? ` [${tuningKey}]` : ''} across ${contexts.length} context sizes`);
emit({
event: 'start',
sampleIndex: 0,
@@ -225,16 +353,23 @@ export async function runAssessment({ backend, modelId, contextTokens = DEFAULT_
// still throw before the stream opens — an unconfigured provider. Catch that
// into the same result shape so one bad backend records a failed sample
// instead of aborting the whole assessment with no evidence at all.
- const result = await runLocalLlmTest({
- backend,
+ // Both runners take the same shape, including the request-applied knobs —
+ // the ONLY difference is whether the model is reached through a configured
+ // provider or straight at a loopback endpoint.
+ const shared = {
modelId,
prompt: buildSamplePrompt(context),
systemPrompt: SAMPLE_SYSTEM_PROMPT,
temperature: 0,
maxTokens: SAMPLE_MAX_TOKENS,
timeoutMs: SAMPLE_TIMEOUT_MS,
+ extraBody: requestBody(backend, normalizedTuning),
signal,
- }).catch((err) => ({ backend, modelId, text: '', error: err?.message || 'assessment run failed' }));
+ };
+ const result = await (endpoint
+ ? runEndpointLlmTest({ ...shared, runtime: backend, endpoint, apiKey })
+ : runLocalLlmTest({ ...shared, backend })
+ ).catch((err) => ({ backend, modelId, text: '', error: err?.message || 'assessment run failed' }));
const sample = toSample(context, result);
samples.push(sample);
@@ -258,6 +393,20 @@ export async function runAssessment({ backend, modelId, contextTokens = DEFAULT_
const assessment = {
backend,
modelId,
+ // The configuration this reading describes. `{}` / `''` / `null` mean
+ // "backend defaults", which is a real answer — the daemon ran with whatever
+ // it ships with, and a later default-run compares against it directly.
+ tuning: normalizedTuning,
+ tuningKey,
+ tuningLabel,
+ // Whether the tuning actually reached the daemon. `false` with a reason
+ // means the numbers below describe SOME OTHER configuration, and the UI has
+ // to say so rather than filing them under the requested tuning. `null` means
+ // there was nothing for PortOS to apply — backend defaults, or a tuning made
+ // entirely of knobs the user set outside PortOS.
+ tuningApplied: tuningApplication.applied,
+ tuningNotApplied: tuningApplication.applied === false ? tuningApplication.reason : null,
+ endpoint,
params: card?.params ?? null,
// LM Studio serves one quant per install but reports a repo-level id, so the
// quant has to be recorded separately for a catalog badge to know WHICH
@@ -315,51 +464,75 @@ export async function getAssessmentReport({ intent = 'balanced' } = {}) {
// taken before a RAM upgrade or a backend update describes hardware that no
// longer exists, and nothing else on this page would ever say so — the user
// would have to remember. This path can afford the backend-version probe (it
- // already lists models from both backends); the catalog badge path cannot, and
+ // already lists models from every runtime); the catalog badge path cannot, and
// uses the free durable-fields comparison instead.
const liveEnvironments = await captureLiveEnvironments();
const assessments = stored.map((a) => withStaleness(a, liveEnvironments[a?.backend] || null));
- // Both managers cache an EMPTY list on a failed read rather than throwing, so
- // `[]` alone cannot distinguish "this backend has no models" from "the list
- // could not be read" — and presenting the second as the first would silently
- // hide every assessable model plus the reason. Each manager's own list-error
- // getter is the authoritative signal; a `.catch` here is only the backstop.
+ // One listing per assessable runtime. `models: null` is a FAILED read, never
+ // an empty catalog — see `listRuntimeModels`.
const listed = Object.fromEntries(await Promise.all(
- ['ollama', 'lmstudio'].map(async (backend) => {
- const models = await listModels(backend).catch((err) => ({ error: err?.message || 'model list failed' }));
- if (!Array.isArray(models)) return [backend, { models: null, error: models.error }];
- const error = backend === 'ollama' ? getOllamaListError() : getLmStudioListError();
- return [backend, { models, error: error || null }];
- })
+ ASSESSABLE_RUNTIMES.map(async (runtime) => [runtime, await listRuntimeModels(runtime)])
));
- const listErrors = Object.entries(listed).filter(([, r]) => r.error).map(([backend]) => backend);
+ const runtimes = ASSESSABLE_RUNTIMES.map((id) => ({
+ id,
+ label: LOCAL_RUNTIMES[id]?.label || id,
+ managed: MANAGED_ASSESSMENT_BACKENDS.includes(id),
+ // `null` = the listing failed, so the count is unknown. `0` = it was read
+ // and this runtime genuinely serves nothing.
+ modelCount: Array.isArray(listed[id].models) ? listed[id].models.length : null,
+ error: listed[id].error,
+ tuningSpecs: tuningSpecsFor(id),
+ }));
+
+ const listErrors = ASSESSABLE_RUNTIMES.filter((id) => listed[id].error);
const installedKeys = new Set(
- Object.entries(listed).flatMap(([backend, r]) => (r.models || []).map((m) => assessmentKey(backend, m?.id)))
+ ASSESSABLE_RUNTIMES.flatMap((runtime) => (listed[runtime].models || []).map((m) => assessmentKey(runtime, m?.id)))
);
// A model the user has since deleted must not keep showing up as a
- // recommendation — it cannot run. But only drop it when the backend's list is
+ // recommendation — it cannot run. But only drop it when the runtime's list is
// TRUSTWORTHY: an unreadable list would otherwise wipe every recommendation
- // for that backend, which is the same "failed read read as empty" mistake.
- const trusted = new Set(Object.entries(listed).filter(([, r]) => Array.isArray(r.models) && !r.error).map(([backend]) => backend));
+ // for that runtime, which is the same "failed read read as empty" mistake.
+ const trusted = new Set(ASSESSABLE_RUNTIMES.filter((id) => Array.isArray(listed[id].models) && !listed[id].error));
const isStillInstalled = (a) =>
!trusted.has(a?.backend) || installedKeys.has(assessmentKey(a?.backend, a?.modelId));
const stillInstalled = assessments.filter(isStillInstalled);
const uninstalled = assessments
.filter((a) => !isStillInstalled(a))
- .map((a) => ({ backend: a?.backend || null, modelId: a?.modelId || null }));
+ .map((a) => ({ backend: a?.backend || null, modelId: a?.modelId || null, tuningLabel: a?.tuningLabel || null }));
- const { ranked, excluded } = rankByIntent(stillInstalled, resolvedIntent);
+ // A reading taken under a tuning PortOS could NOT apply describes some other
+ // configuration entirely. It is kept on disk (the run cost real minutes, and
+ // the failed attempt plus its reason is what tells the user why) but it must
+ // never be scored AS that tuning: ranking it would recommend a configuration
+ // nobody measured, and comparing it would credit the previous config's
+ // throughput to the knobs that never reached the daemon.
+ const unappliedTuning = stillInstalled.filter((a) => a?.tuningApplied === false);
+ const scorable = stillInstalled.filter((a) => a?.tuningApplied !== false);
+
+ const { ranked, excluded } = rankByIntent(scorable, resolvedIntent);
+ for (const a of unappliedTuning) {
+ excluded.push({
+ backend: a?.backend || null,
+ modelId: a?.modelId || null,
+ tuningKey: a?.tuningKey || '',
+ tuningLabel: a?.tuningLabel || null,
+ verdict: a?.verdict || 'unknown',
+ reason: `measured, but the requested tuning was not applied — ${a?.tuningNotApplied || 'reason not recorded'}`,
+ });
+ }
- const assessedKeys = new Set(assessments.map((a) => assessmentKey(a?.backend, a?.modelId)));
+ // "Not yet measured" is keyed on the model, NOT on the model+tuning: once one
+ // tuning has been measured the model is no longer an unanswered question, and
+ // listing it again under every un-run tuning would make the section unbounded.
+ const assessedModels = new Set(assessments.map((a) => assessmentKey(a?.backend, a?.modelId)));
const unassessed = [];
- for (const [backend, { models }] of Object.entries(listed)) {
- if (!Array.isArray(models)) continue;
- for (const model of models) {
- if (model?.id && !assessedKeys.has(assessmentKey(backend, model.id))) {
- unassessed.push({ backend, modelId: model.id, params: model.params ?? null });
+ for (const runtime of ASSESSABLE_RUNTIMES) {
+ for (const model of listed[runtime].models || []) {
+ if (model?.id && !assessedModels.has(assessmentKey(runtime, model.id))) {
+ unassessed.push({ backend: runtime, modelId: model.id, params: model.params ?? null });
}
}
}
@@ -370,7 +543,12 @@ export async function getAssessmentReport({ intent = 'balanced' } = {}) {
defaultContextTokens: DEFAULT_CONTEXT_TOKENS,
assessments,
unassessed,
- // Backends whose model list could not be trusted — distinct from "listed,
+ // Every assessable runtime with its label, reachability, and knob catalog —
+ // so the UI renders one source of truth instead of a hardcoded backend list.
+ runtimes,
+ // Which tuning won, per model, for models measured under two or more.
+ tuningComparison: compareTunings(scorable),
+ // Runtimes whose model list could not be trusted — distinct from "listed,
// and legitimately empty".
listErrors,
// Measurements for models that are no longer installed. Kept on disk (a
@@ -380,7 +558,7 @@ export async function getAssessmentReport({ intent = 'balanced' } = {}) {
ranked,
excluded,
// The machine as it is now, so the panel can name the difference rather than
- // just flagging "stale". Keyed by backend because the backend version is
+ // just flagging "stale". Keyed by runtime because the backend version is
// part of what makes a reading stale.
liveEnvironments,
};
diff --git a/server/services/localModelAssessments.test.js b/server/services/localModelAssessments.test.js
index e916dc649..7ab570a44 100644
--- a/server/services/localModelAssessments.test.js
+++ b/server/services/localModelAssessments.test.js
@@ -17,7 +17,31 @@ vi.mock('../lib/fileUtils.js', async (importOriginal) =>
makePathsProxy(await importOriginal(), { dataRoot: tempRoot }));
const runLocalLlmTest = vi.fn();
-vi.mock('./localLlmPlayground.js', () => ({ runLocalLlmTest: (...args) => runLocalLlmTest(...args) }));
+const runEndpointLlmTest = vi.fn();
+vi.mock('./localLlmPlayground.js', () => ({
+ runLocalLlmTest: (...args) => runLocalLlmTest(...args),
+ runEndpointLlmTest: (...args) => runEndpointLlmTest(...args),
+}));
+
+// The endpoint runtimes (llama.cpp / MTPLX / vLLM) are reached over loopback
+// HTTP. Left unmocked these tests probe whatever the DEVELOPER's machine happens
+// to be serving — a live llama-server on this box put its real model id into the
+// unassessed list. Default them to "up, serving nothing", which is the only
+// state that keeps the managed-backend assertions about what they are about.
+const probeOpenAiModels = vi.fn();
+vi.mock('../lib/openAiModelsProbe.js', () => ({ probeOpenAiModels: (...args) => probeOpenAiModels(...args) }));
+
+// `runtimeApiKey` reads the provider registry to authenticate a key-gated
+// runtime (a vLLM container started behind VLLM_API_KEY).
+const getAllProviders = vi.fn();
+vi.mock('./providers.js', () => ({ getAllProviders: (...args) => getAllProviders(...args) }));
+
+const getLlamaServerEndpoint = vi.fn();
+const relaunchLlamaServerWithTuning = vi.fn();
+vi.mock('./llamaServerManager.js', () => ({
+ getLlamaServerEndpoint: (...args) => getLlamaServerEndpoint(...args),
+ relaunchLlamaServerWithTuning: (...args) => relaunchLlamaServerWithTuning(...args),
+}));
const listModels = vi.fn();
vi.mock('./localLlm.js', () => ({ listModels: (...args) => listModels(...args) }));
@@ -60,6 +84,11 @@ beforeEach(() => {
getLoadedModels.mockReset().mockResolvedValue([{ id: 'example-model:7b', name: 'example-model:7b', size: 5 * 2 ** 30 }]);
getOllamaListError.mockReset().mockReturnValue(null);
getLmStudioListError.mockReset().mockReturnValue(null);
+ runEndpointLlmTest.mockReset();
+ probeOpenAiModels.mockReset().mockResolvedValue({ reachable: true, models: [], error: null });
+ getLlamaServerEndpoint.mockReset().mockResolvedValue('http://127.0.0.1:5568/v1');
+ getAllProviders.mockReset().mockResolvedValue([]);
+ relaunchLlamaServerWithTuning.mockReset().mockResolvedValue({ applied: true, reason: null, config: null });
});
describe('buildSamplePrompt', () => {
@@ -276,7 +305,7 @@ describe('uninstalled models', () => {
// It can no longer run, so it must not be ranked — but the measurement stays
// on disk so a re-install doesn't cost another run.
expect(report.ranked).toEqual([]);
- expect(report.uninstalled).toContainEqual({ backend: 'ollama', modelId: 'example-model:7b' });
+ expect(report.uninstalled).toContainEqual({ backend: 'ollama', modelId: 'example-model:7b', tuningLabel: null });
expect(report.assessments).toHaveLength(1);
});
@@ -519,3 +548,259 @@ describe('backend-version staleness on the read paths', () => {
expect(ollamaVersion).not.toHaveBeenCalled();
});
});
+
+// ---------------------------------------------------------------------------
+// Endpoint runtimes (llama.cpp / MTPLX / vLLM) and launch tuning
+// ---------------------------------------------------------------------------
+
+describe('endpoint runtimes', () => {
+ it('lists a bare daemon\'s models from GET /v1/models, with no params to guess from', async () => {
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
+ const result = await svc.listRuntimeModels('mtplx');
+ expect(result).toEqual({ models: [{ id: 'dflash', params: null, quantization: null }], error: null });
+ });
+
+ // A stopped daemon has no catalog on disk to fall back to, so "unreachable"
+ // must surface as an ERROR. Reported as an empty list it would silently hide
+ // every model behind a daemon the user only needs to start.
+ it('reports an unreachable daemon as an error, never as an empty catalog', async () => {
+ probeOpenAiModels.mockResolvedValue({ reachable: false, models: null, error: 'ECONNREFUSED' });
+ const result = await svc.listRuntimeModels('vllm');
+ expect(result.models).toBeNull();
+ expect(result.error).toMatch(/not reachable/);
+ });
+
+ it('reads llama.cpp\'s endpoint from the running server, not from the default port', async () => {
+ getLlamaServerEndpoint.mockResolvedValue('http://127.0.0.1:9999/v1');
+ expect(await svc.runtimeEndpoint('llama')).toBe('http://127.0.0.1:9999/v1');
+ });
+
+ it('falls back to the canonical base URL when llama-server cannot be reached', async () => {
+ getLlamaServerEndpoint.mockRejectedValue(new Error('pm2 is not installed'));
+ expect(await svc.runtimeEndpoint('llama')).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/v1$/);
+ });
+
+ // The read path runs on every Performance page load. `getLlamaServerStatus`
+ // costs a network probe plus an `execPm2 logs` subprocess whose output this
+ // caller would throw away.
+ it('resolves the endpoint without paying for a status probe', async () => {
+ await svc.getAssessmentReport();
+ expect(getLlamaServerEndpoint).toHaveBeenCalled();
+ });
+
+ it('measures an endpoint runtime directly instead of through the provider path', async () => {
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
+ runEndpointLlmTest.mockResolvedValue(okRun());
+ const result = await svc.runAssessment({ backend: 'mtplx', modelId: 'dflash', contextTokens: [512] });
+ expect(runLocalLlmTest).not.toHaveBeenCalled();
+ expect(runEndpointLlmTest).toHaveBeenCalledWith(expect.objectContaining({ runtime: 'mtplx', modelId: 'dflash' }));
+ expect(result.verdict).toBe('fits');
+ expect(result.endpoint).toBeTruthy();
+ });
+
+ // Only Ollama's /api/ps reports a footprint. Copying a weight-file size in
+ // would re-introduce the estimate this whole feature exists to replace.
+ it('records no resident footprint for a runtime that reports none', async () => {
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
+ runEndpointLlmTest.mockResolvedValue(okRun());
+ const result = await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512] });
+ expect(result.residentGb).toBeNull();
+ });
+});
+
+describe('tuning', () => {
+ it('keeps two tunings of one model as two records rather than overwriting', async () => {
+ runLocalLlmTest.mockResolvedValue(okRun(40, 120));
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
+ const stored = await svc.loadAssessments();
+ expect(stored).toHaveLength(2);
+ expect(stored.map((a) => a.tuningKey).sort()).toEqual(['', 'numCtx=8192']);
+ });
+
+ it('re-running the SAME tuning replaces that record, not the untuned one', async () => {
+ runLocalLlmTest.mockResolvedValue(okRun(40, 120));
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
+ expect(await svc.loadAssessments()).toHaveLength(2);
+ });
+
+ it('sends a request-applied knob with the measurement', async () => {
+ runLocalLlmTest.mockResolvedValue(okRun());
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
+ expect(runLocalLlmTest).toHaveBeenCalledWith(expect.objectContaining({ extraBody: { num_ctx: 8192 } }));
+ });
+
+ it('puts llama.cpp launch knobs on the command line before the first sample', async () => {
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
+ runEndpointLlmTest.mockResolvedValue(okRun());
+ const result = await svc.runAssessment({
+ backend: 'llama', modelId: 'dflash', contextTokens: [512], tuning: { ubatchSize: 512 },
+ });
+ expect(relaunchLlamaServerWithTuning).toHaveBeenCalledWith({ ubatchSize: 512 });
+ expect(result.tuningApplied).toBe(true);
+ expect(result.tuningLabel).toBe('Micro-batch size 512');
+ });
+
+ // A reading taken under a tuning PortOS could NOT apply describes some other
+ // configuration. Recording it as evidence for the requested tuning would be
+ // the same lie as a fabricated measurement.
+ it('records that a launch tuning was not applied, with the reason', async () => {
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
+ runEndpointLlmTest.mockResolvedValue(okRun());
+ relaunchLlamaServerWithTuning.mockResolvedValue({ applied: false, reason: 'llama-server is not running', config: null });
+ const result = await svc.runAssessment({
+ backend: 'llama', modelId: 'dflash', contextTokens: [512], tuning: { ubatchSize: 512 },
+ });
+ expect(result.tuningApplied).toBe(false);
+ expect(result.tuningNotApplied).toBe('llama-server is not running');
+ });
+
+ // A tuning made entirely of `record` knobs was never applied by PortOS in any
+ // sense — `true` there is the exact claim lib/localModelTuning.js forbids, and
+ // `false` would imply something went wrong. `null` is the honest answer.
+ it('records tuningApplied as null when there is nothing for PortOS to apply', async () => {
+ runLocalLlmTest.mockResolvedValue(okRun());
+ const untuned = await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
+ expect(untuned.tuningApplied).toBeNull();
+ const recordOnly = await svc.runAssessment({
+ backend: 'lmstudio', modelId: 'example-model:7b', contextTokens: [512], tuning: { contextLength: 8192 },
+ });
+ expect(recordOnly.tuningApplied).toBeNull();
+ expect(recordOnly.tuningLabel).toBe('Context length 8k');
+ });
+
+ it('never relaunches llama-server for an untuned run', async () => {
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
+ runEndpointLlmTest.mockResolvedValue(okRun());
+ await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512] });
+ expect(relaunchLlamaServerWithTuning).not.toHaveBeenCalled();
+ });
+
+ it('deletes one tuning of a model and leaves the others', async () => {
+ runLocalLlmTest.mockResolvedValue(okRun());
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
+ expect(await svc.deleteAssessment('ollama', 'example-model:7b', 'numCtx=8192')).toEqual({ deleted: true });
+ const stored = await svc.loadAssessments();
+ expect(stored.map((a) => a.tuningKey)).toEqual(['']);
+ });
+
+ it('reports which tuning won once a model has two measurements', async () => {
+ runLocalLlmTest.mockResolvedValueOnce(okRun(40, 90)).mockResolvedValueOnce(okRun(40, 150));
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
+ const report = await svc.getAssessmentReport();
+ expect(report.tuningComparison).toHaveLength(1);
+ expect(report.tuningComparison[0].best.label).toBe('Context size 8k');
+ });
+
+ // Once ONE tuning is measured the model is no longer an unanswered question.
+ // Re-listing it under every un-run tuning would make the section unbounded.
+ it('drops a model from "not yet measured" as soon as any tuning is recorded', async () => {
+ runLocalLlmTest.mockResolvedValue(okRun());
+ await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
+ const report = await svc.getAssessmentReport();
+ expect(report.unassessed.some((u) => u.backend === 'ollama')).toBe(false);
+ });
+});
+
+describe('runtime roster', () => {
+ it('reports every assessable runtime with its knob catalog', async () => {
+ const report = await svc.getAssessmentReport();
+ expect(report.runtimes.map((r) => r.id)).toEqual(['ollama', 'lmstudio', 'llama', 'mtplx', 'vllm']);
+ expect(report.runtimes.find((r) => r.id === 'llama').tuningSpecs.some((s) => s.id === 'ubatchSize')).toBe(true);
+ });
+
+ // `null` = the listing failed so the count is UNKNOWN; `0` = read, and this
+ // runtime genuinely serves nothing. A UI must be able to tell them apart.
+ it('reports an unknown model count as null and an empty one as 0', async () => {
+ probeOpenAiModels.mockResolvedValue({ reachable: false, models: null, error: 'ECONNREFUSED' });
+ const report = await svc.getAssessmentReport();
+ expect(report.runtimes.find((r) => r.id === 'mtplx').modelCount).toBeNull();
+ expect(report.runtimes.find((r) => r.id === 'ollama').modelCount).toBe(1);
+ });
+});
+
+
+// ---------------------------------------------------------------------------
+// Key-gated runtimes, and evidence hygiene for a tuning that never landed
+// ---------------------------------------------------------------------------
+
+describe('key-gated runtimes', () => {
+ // `vllmBacked` is the structural marker `localRuntimeKind` keys on — it
+ // deliberately does NOT derive the backend from an editable name or endpoint.
+ const vllmProvider = { id: 'vllm', name: 'vLLM', vllmBacked: true, endpoint: 'http://127.0.0.1:18020/v1', apiKey: 'secret-key' };
+
+ // A vLLM container from the shipped compose stack sets VLLM_API_KEY and 401s
+ // an unauthenticated request. Without the key the listing reads as
+ // "unreadable" and every sample fails auth — recorded as a fit verdict.
+ it('authenticates the model listing with the provider record\'s key', async () => {
+ getAllProviders.mockResolvedValue([vllmProvider]);
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['qwen'], error: null });
+ await svc.listRuntimeModels('vllm');
+ expect(probeOpenAiModels).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ apiKey: 'secret-key' }));
+ });
+
+ it('authenticates the measurement with the same key', async () => {
+ getAllProviders.mockResolvedValue([vllmProvider]);
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['qwen'], error: null });
+ runEndpointLlmTest.mockResolvedValue(okRun());
+ await svc.runAssessment({ backend: 'vllm', modelId: 'qwen', contextTokens: [512] });
+ expect(runEndpointLlmTest).toHaveBeenCalledWith(expect.objectContaining({ apiKey: 'secret-key' }));
+ });
+
+ // The usual loopback daemon is unauthenticated; attaching a key from an
+ // unrelated provider would be worse than sending none.
+ it('sends no key when no provider for that runtime carries one', async () => {
+ getAllProviders.mockResolvedValue([{ id: 'ollama', ollamaBacked: true, endpoint: 'http://localhost:11434/v1', apiKey: 'not-mine' }]);
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: [], error: null });
+ await svc.listRuntimeModels('mtplx');
+ expect(probeOpenAiModels).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ apiKey: '' }));
+ });
+});
+
+describe('unapplied tuning is not evidence', () => {
+ const measureWithUnappliedTuning = async () => {
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
+ runEndpointLlmTest.mockResolvedValue(okRun(40, 150));
+ relaunchLlamaServerWithTuning.mockResolvedValue({ applied: false, reason: 'llama-server is not running', config: null });
+ await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512], tuning: { ubatchSize: 512 } });
+ };
+
+ // The numbers describe the configuration that was ACTUALLY running. Ranking
+ // them would recommend a tuning nobody measured.
+ it('keeps the record but never ranks it', async () => {
+ await measureWithUnappliedTuning();
+ const report = await svc.getAssessmentReport();
+ expect(report.assessments).toHaveLength(1);
+ expect(report.ranked).toEqual([]);
+ expect(report.excluded[0].reason).toMatch(/tuning was not applied/);
+ expect(report.excluded[0].tuningKey).toBe('ubatchSize=512');
+ });
+
+ // Comparing it would credit the previous config's throughput to knobs that
+ // never reached the daemon.
+ it('never lets it win a tuning comparison', async () => {
+ // A real, applied backend-defaults reading first — llama is an endpoint
+ // runtime, so it measures through runEndpointLlmTest either way.
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
+ runEndpointLlmTest.mockResolvedValue(okRun(40, 90));
+ await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512] });
+ // …then a faster one whose tuning never reached the daemon. Two records
+ // exist, so the only thing stopping a comparison is the exclusion itself.
+ await measureWithUnappliedTuning();
+ expect(await svc.loadAssessments()).toHaveLength(2);
+ const report = await svc.getAssessmentReport();
+ expect(report.tuningComparison).toEqual([]);
+ });
+
+ it('still ranks a tuning that WAS applied', async () => {
+ probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
+ runEndpointLlmTest.mockResolvedValue(okRun(40, 150));
+ await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512], tuning: { ubatchSize: 512 } });
+ const report = await svc.getAssessmentReport();
+ expect(report.ranked.map((r) => r.tuningKey)).toEqual(['ubatchSize=512']);
+ });
+});
diff --git a/server/services/localRuntimeSetup.js b/server/services/localRuntimeSetup.js
index 8564d7f35..e849fb394 100644
--- a/server/services/localRuntimeSetup.js
+++ b/server/services/localRuntimeSetup.js
@@ -3,7 +3,7 @@
*
* `providerReadiness.js` answers WHAT is missing (the daemon isn't installed,
* isn't running, isn't serving the right model). Until this module existed, the
- * answer to "so fix it" was a link — to the Local LLM settings tab for the two
+ * answer to "so fix it" was a link — to the Models → LLMs page for the two
* backends PortOS manages, and to the vendor's README for MTPLX, which is a
* dead end inside PortOS: the user leaves the app, reads a setup doc, runs two
* commands in a terminal, comes back and reloads. This module makes the
@@ -20,7 +20,7 @@
*
* - **Weights are never downloaded.** llama.cpp cannot be started without a
* GGUF path the user chooses, and no runtime's *model* check is auto-fixed
- * — a multi-gigabyte download is a decision, and the Local LLM tab already
+ * — a multi-gigabyte download is a decision, and the Models → LLMs page already
* owns that flow with a picker. MTPLX is started on a checkpoint ALREADY in
* its cache (`lib/mtplxModels.js`); an empty cache is reported with the
* `mtplx pull` command that fixes it, never fetched.
@@ -324,12 +324,12 @@ const SETUP_ROWS = Object.freeze({
const result = await installLlamaServer({ onProgress: (p) => { if (p?.message) emit(p.message); } })
.catch((err) => ({ success: false, error: err.message }));
return result.success
- ? { success: true, note: 'Choose a GGUF model on Settings → Local LLM to start llama-server — PortOS does not pick weights for you.' }
+ ? { success: true, note: 'Choose a GGUF model on Models → LLMs to start llama-server — PortOS does not pick weights for you.' }
: result;
},
// llama-server takes a required model path, and the weights are a separate
// multi-gigabyte download. Starting it unattended would mean guessing which
- // checkpoint the user meant, so the Local LLM tab keeps that step.
+ // checkpoint the user meant, so the Models → LLMs page keeps that step.
start: null,
}),
});
@@ -423,7 +423,7 @@ export async function runLocalRuntimeSetup(kind, { endpoint, emit = () => {}, is
}
if (!row.start) {
- return { success: true, message: `${runtime.label} is installed. Pick a model on Settings → Local LLM to start it.` };
+ return { success: true, message: `${runtime.label} is installed. Pick a model on Models → LLMs to start it.` };
}
if (isCancelled()) return { success: false, error: 'Cancelled after the install — nothing was started.' };
diff --git a/server/services/localRuntimeSetup.test.js b/server/services/localRuntimeSetup.test.js
index 4fb0127cb..0dcb2ba60 100644
--- a/server/services/localRuntimeSetup.test.js
+++ b/server/services/localRuntimeSetup.test.js
@@ -237,7 +237,7 @@ describe('runLocalRuntimeSetup', () => {
const result = await runLocalRuntimeSetup('llama', { endpoint: 'http://127.0.0.1:8080/v1' });
expect(llama.installLlamaServer).toHaveBeenCalled();
- expect(result).toMatchObject({ success: true, message: expect.stringMatching(/Local LLM/) });
+ expect(result).toMatchObject({ success: true, message: expect.stringMatching(/Models → LLMs/) });
});
it('stops after the install when the modal was closed', async () => {
diff --git a/server/services/loraDatasetCaption.js b/server/services/loraDatasetCaption.js
index dd79c3ede..9636d97ec 100644
--- a/server/services/loraDatasetCaption.js
+++ b/server/services/loraDatasetCaption.js
@@ -162,7 +162,7 @@ export async function resolveCaptionModel({
: null) || visionModels[0];
if (!pick) {
throw new ServerError(
- 'No vision-capable model is installed for captioning. Install one (e.g. Qwen2.5-VL, LLaVA, or Llama 3.2 Vision) from Settings → Local LLM, then pick it on the dataset.',
+ 'No vision-capable model is installed for captioning. Install one (e.g. Qwen2.5-VL, LLaVA, or Llama 3.2 Vision) from Models → LLMs, then pick it on the dataset.',
{ status: 409, code: 'LORA_CAPTION_NO_VISION_MODEL' },
);
}
diff --git a/server/services/providerReadiness.js b/server/services/providerReadiness.js
index c4f0d9cfd..0a24f7302 100644
--- a/server/services/providerReadiness.js
+++ b/server/services/providerReadiness.js
@@ -157,7 +157,7 @@ function runtimeCheck(runtime, { onPath, appInstalled, installed, reachable, set
: `\`${runtime.command}\` was not found on PortOS's PATH.`;
const fixHint = installed ? null
: setupHint(setup, 'install')
- || (runtime.manageUrl ? `Install ${runtime.label} from Settings → Local LLM.`
+ || (runtime.manageUrl ? `Install ${runtime.label} from Models → LLMs.`
: `Use the setup button below to install ${runtime.label}.`);
return { id: 'runtime', label: `${runtime.label} installed`, ok: installed, detail, fixHint };
}
@@ -173,7 +173,7 @@ function serverCheck(runtime, { installed, result, setup }) {
fixHint: null,
};
}
- const start = `Start ${runtime.label}${runtime.manageUrl ? ' from Settings → Local LLM' : ''}.`;
+ const start = `Start ${runtime.label}${runtime.manageUrl ? ' from Models → LLMs' : ''}.`;
const fallback = installed
? `${start} ${runtime.modelsHint}`
: `Install ${runtime.label} first, then start it. ${runtime.modelsHint}`;
@@ -215,9 +215,9 @@ function modelCheck(runtime, wanted, served, probeError = null) {
: `${runtime.label} is serving ${listed}${served.length > 3 ? ` +${served.length - 3} more` : ''}.`;
const fixHint = served.length === 0
? (runtime.manageUrl
- ? 'No model is loaded. Start a preset from Settings → Local LLM.'
+ ? 'No model is loaded. Start a preset from Models → LLMs.'
: 'No model is loaded. Use the setup controls on this card to load one.')
- : `This provider will send \`${wanted}\`, but the running server only accepts ${listed}. Use the button below to match them${runtime.manageUrl ? ', or change the loaded weights in Local LLM settings' : ''}.`;
+ : `This provider will send \`${wanted}\`, but the running server only accepts ${listed}. Use the button below to match them${runtime.manageUrl ? ', or change the loaded weights on the Models → LLMs page' : ''}.`;
return {
id: 'model',
label,
@@ -256,7 +256,7 @@ export async function getProviderReadiness(provider, deps = {}) {
const onPath = Boolean(runtime.command && findCommand(runtime.command));
// LM Studio ships as a macOS app bundle whose `lms` shim the user opts into
// separately, so PATH alone says "not installed" for a perfectly installed
- // copy. The Local LLM tab already counts the bundle (`localLlm.getStatus`);
+ // copy. The Models → LLMs page already counts the bundle (`localLlm.getStatus`);
// without the same signal here the card would render "LM Studio installed"
// and "install LM Studio" two lines apart, and send the user after the wrong
// fix — the real one is "start its server".
diff --git a/server/services/providerReadiness.test.js b/server/services/providerReadiness.test.js
index 8da65efee..669d83d4a 100644
--- a/server/services/providerReadiness.test.js
+++ b/server/services/providerReadiness.test.js
@@ -35,7 +35,7 @@ describe('getProviderReadiness', () => {
it('reports nothing — and probes nothing — for an API provider on another machine', async () => {
// `LM Studio ` matches the `lmstudio` runtime by NAME, so the card
// used to answer "`lms` is on PortOS's PATH" and "start LM Studio from
- // Settings → Local LLM" about a server running on someone else's box. An
+ // Models → LLMs" about a server running on someone else's box. An
// external endpoint is assumed to be set up by whoever runs it.
let probed = 0;
let pathScans = 0;
@@ -122,7 +122,7 @@ describe('getProviderReadiness', () => {
const model = checkById(readiness, 'model');
expect(model.detail).toMatch(/no model loaded/);
expect(model.servedModels).toEqual([]);
- expect(model.fixHint).toMatch(/Local LLM/);
+ expect(model.fixHint).toMatch(/Models → LLMs/);
expect(model.fixHint).not.toMatch(/button below/);
});
@@ -178,7 +178,7 @@ describe('getProviderReadiness', () => {
});
it('counts an LM Studio app bundle as installed, so the card asks for a START not an install', async () => {
- // The Local LLM tab already treats the macOS app bundle as installed. When
+ // The Models → LLMs page already treats the macOS app bundle as installed. When
// this disagreed, one card rendered 'LM Studio installed' and 'Install LM
// Studio' two lines apart — and the install was the wrong fix.
const readiness = await getProviderReadiness(
@@ -191,7 +191,7 @@ describe('getProviderReadiness', () => {
});
it('offers a one-click install+start for MTPLX instead of a setup-doc dead end', async () => {
- // The whole point of the setup button: MTPLX has no Local LLM tab entry,
+ // The whole point of the setup button: MTPLX has no Models → LLMs page entry,
// so before it existed the only answer here was "go read the vendor docs".
const restore = pinPlatform('darwin');
const readiness = await getProviderReadiness(
diff --git a/server/services/providerRuntimeInstaller.js b/server/services/providerRuntimeInstaller.js
index 5517129da..a311eb3ee 100644
--- a/server/services/providerRuntimeInstaller.js
+++ b/server/services/providerRuntimeInstaller.js
@@ -29,7 +29,7 @@
* offered on Windows, where they ship a separate PowerShell
* script PortOS deliberately does not run for the user.
*
- * Ollama and LM Studio are deliberately absent: the Local LLM settings tab owns
+ * Ollama and LM Studio are deliberately absent: the Models → LLMs page owns
* their install (it also starts the service afterwards, and knows that a macOS
* app bundle with no `lms` shim still counts as installed). The Providers page
* links there for those two instead of re-probing them here.
diff --git a/server/services/systemResources.js b/server/services/systemResources.js
index f30bc8caa..3dc28a3a3 100644
--- a/server/services/systemResources.js
+++ b/server/services/systemResources.js
@@ -225,7 +225,7 @@ function downloadedModelInventory({
loaded: !ollamaResidencyError && ollamaLoadedIds.has(stored.id),
residencyUnknown: Boolean(ollamaResidencyError),
inventoryUnknown: !Array.isArray(ollamaStored) || Boolean(stored.inventoryUnknown),
- managePath: '/settings/local-llm',
+ managePath: '/models/llms',
action: ollamaStatus?.available
? { type: 'local-model', backend: 'ollama', modelId: stored.id }
: null,
@@ -276,7 +276,7 @@ function downloadedModelInventory({
residencyUnknown: Boolean(lmStudioResidencyError),
inventoryUnknown: !Array.isArray(lmStudioStored) || Boolean(stored.inventoryUnknown),
cleanupReason: 'Deleting this entry removes the whole LM Studio model folder, including every downloaded quantization in it.',
- managePath: '/settings/local-llm',
+ managePath: '/models/llms',
action: { type: 'local-model', backend: 'lmstudio', modelId: stored.id },
};
});
@@ -448,13 +448,13 @@ export async function buildSystemResourceReport() {
{
id: 'ollama', label: 'Ollama models', kind: 'model',
sizeBytes: finiteOrNull(ollamaBytes), status: backendState(ollamaBytes),
- managePath: '/settings/local-llm', protected: false,
+ managePath: '/models/llms', protected: false,
note: 'Local language-model manifests and shared blobs.',
},
{
id: 'lmstudio', label: 'LM Studio models', kind: 'model',
sizeBytes: finiteOrNull(lmStudioBytes), status: backendState(lmStudioBytes),
- managePath: '/settings/local-llm', protected: false,
+ managePath: '/models/llms', protected: false,
note: 'Downloaded GGUF or MLX model directories.',
},
{