diff --git a/AGENTS.md b/AGENTS.md index f620f11..c010320 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,7 +54,7 @@ history/ ├── server/ │ ├── index.ts # Express: /api/*, /sse/:id, static+SPA; reads PORT/LLM_*/DATA_DIR │ ├── engine.ts # pure: buildPrompt / parse+validate / applyTurnResult / runTurn / advisor -│ ├── llm.ts # LlmClient (injectable transport) + env config +│ ├── llm.ts # LlmClient (injectable transport, retry+backoff, logging) + env config │ ├── stub.ts # deterministic LLM transport (tests/e2e) + ADVISOR_MARKER │ ├── map.ts # GameState → RegionMap (ISO_A3 → owner + stable color) │ ├── presets.ts # load preset JSON files from disk @@ -137,19 +137,31 @@ history/ | `LLM_API_BASE` | `https://ai.kkhost.pl/v1` | OpenAI-compatible **vLLM** base URL (no API key required) | | `LLM_MODEL` | `qwen3.8-27b` | vLLM **reasoning** model — CoT in `reasoning`, final answer in `content` | | `LLM_TIMEOUT_MS` | `180000` | the 27B reasoning model can be slow | +| `LLM_MAX_TOKENS` | `4000` | max **output** tokens per request; with `reasoning_effort:low` a realistic turn uses ~800–1100, so 4000 is ~4× headroom and stays under the 90s proxy wall | +| `LLM_REASONING_EFFORT` | `low` | sent as top-level `reasoning_effort`; cuts the model's CoT by ~85% (the 502 fix). Set to empty/`off` to not send the field at all | | `LLM_STUB` | _(unset = live)_ | `1` → deterministic stub transport (no model needed) | > **Reasoning model:** `qwen3.8-27b` emits its chain-of-thought in the response's `reasoning` -> field and the final JSON answer in `content`. `server/llm.ts` therefore always sends a generous -> `max_tokens` (`DEFAULT_MAX_TOKENS = 4000`) so the model finishes reasoning before writing -> `content` (a too-small budget returns a null `content`), and reads only `content`, ignoring -> `reasoning`. No auth header is sent — the endpoint requires no key. +> field and the final JSON answer in `content`. `server/llm.ts` therefore sends top-level +> `reasoning_effort: "low"` (`LLM_REASONING_EFFORT`) to cut the CoT by ~85% — small enough to +> fit the `max_tokens` budget (`LLM_MAX_TOKENS`, default 4000) — and reads only `content`, +> ignoring `reasoning`. `chat()` retries transient failures (timeout/abort, 5xx, network +> errors, null/empty `content`) up to 3 attempts with 2s→3s backoff and logs every call +> (`[llm] chat ok` / `[llm] chat failed`). No auth header is sent — the endpoint requires no key. | `PORT` | `4321` | single server port | | `DATA_DIR` | `./data` | where `history.db` lives | | `CLIENT_DIST` | `./dist` | built client to serve (auto-found) | ## Gotchas +- **502s were the reasoning model exhausting its output budget.** The original client sent only + `max_tokens: 4000`; `qwen3.8-27b`'s chain-of-thought alone often exceeded 4000 tokens, so + `content` came back `null`, which the client treated as fatal → HTTP 502 (~half of turns), + with no retry and no logging. Fixed by sending `reasoning_effort: "low"` (cuts the CoT ~85%), + retrying transient failures (timeout/abort, 5xx, network errors, null/empty `content`) up to + 3 attempts with 2s→3s backoff, and logging every LLM call. A realistic turn now takes + ~11–16s; the effective ceiling is the upstream OpenResty ~90s timeout, **not** our 180s + `LLM_TIMEOUT_MS`. - **Live LLM is a runtime dependency.** The live model is `qwen3.8-27b` on `https://ai.kkhost.pl/v1` (a vLLM **reasoning** model, no API key). If that endpoint is down, turns return 502 (and the advisor degrades to a canned list). Set `LLM_STUB=1` for a fully diff --git a/server/llm.test.ts b/server/llm.test.ts index e203e02..a5980e6 100644 --- a/server/llm.test.ts +++ b/server/llm.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { createLlmClient, DEFAULT_MAX_TOKENS } from './llm'; function json(body: unknown) { @@ -10,42 +10,88 @@ function json(body: unknown) { } as unknown as Response; } +function httpError(status: number, text = 'upstream down') { + return { + ok: false, + status, + text: async () => text, + json: async () => ({}), + } as unknown as Response; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + describe('createLlmClient (injectable transport)', () => { it('chat posts model+messages to {base}/chat/completions and returns content', async () => { - const calls: { url: string; init?: RequestInit }[] = []; + const sent: { url: string; init?: RequestInit }[] = []; const transport = async (url: string, init?: RequestInit) => { - calls.push({ url, init }); + sent.push({ url, init }); return json({ choices: [{ message: { content: 'hello back' } }] }); }; const llm = createLlmClient({ transport, apiBase: 'http://x/v1/', model: 'm1' }); const out = await llm.chat([{ role: 'user', content: 'hi' }], { maxTokens: 10, temperature: 0.5 }); expect(out).toBe('hello back'); - expect(calls[0].url).toBe('http://x/v1/chat/completions'); // trailing slash normalized - const body = JSON.parse(calls[0].init!.body as string); + expect(sent[0].url).toBe('http://x/v1/chat/completions'); // trailing slash normalized + const body = JSON.parse(sent[0].init!.body as string); // The wire key is `max_tokens` (snake_case), never camelCase `maxTokens`. expect(body).toEqual({ model: 'm1', messages: [{ role: 'user', content: 'hi' }], temperature: 0.5, max_tokens: 10, + reasoning_effort: 'low', }); // No auth header: the live vLLM endpoint requires no API key. - expect(calls[0].init!.headers).toEqual({ 'content-type': 'application/json' }); + expect(sent[0].init!.headers).toEqual({ 'content-type': 'application/json' }); }); it('chat uses a generous default max_tokens when the caller omits it', async () => { - const calls: { url: string; init?: RequestInit }[] = []; + const sent: { url: string; init?: RequestInit }[] = []; const transport = async (url: string, init?: RequestInit) => { - calls.push({ url, init }); + sent.push({ url, init }); return json({ choices: [{ message: { content: 'ok' } }] }); }; const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm1' }); await llm.chat([{ role: 'user', content: 'hi' }]); - const body = JSON.parse(calls[0].init!.body as string); + const body = JSON.parse(sent[0].init!.body as string); expect(body.max_tokens).toBe(DEFAULT_MAX_TOKENS); expect(body.max_tokens).toBe(4000); }); + it('chat sends reasoning_effort "low" by default', async () => { + const sent: { url: string; init?: RequestInit }[] = []; + const transport = async (url: string, init?: RequestInit) => { + sent.push({ url, init }); + return json({ choices: [{ message: { content: 'ok' } }] }); + }; + const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm1' }); + await llm.chat([{ role: 'user', content: 'hi' }]); + const body = JSON.parse(sent[0].init!.body as string); + expect(body.reasoning_effort).toBe('low'); + }); + + it('chat omits reasoning_effort when it is "off" or empty', async () => { + const sent: { url: string; init?: RequestInit }[] = []; + const transport = async (url: string, init?: RequestInit) => { + sent.push({ url, init }); + return json({ choices: [{ message: { content: 'ok' } }] }); + }; + for (const effort of ['off', 'OFF', '']) { + sent.length = 0; + const llm = createLlmClient({ + transport, + apiBase: 'http://x/v1', + model: 'm1', + reasoningEffort: effort, + }); + await llm.chat([{ role: 'user', content: 'hi' }]); + const body = JSON.parse(sent[0].init!.body as string); + expect(body.reasoning_effort, `effort=${JSON.stringify(effort)}`).toBeUndefined(); + } + }); + it('listModels returns data[].id from GET {base}/models', async () => { const transport = async (url: string) => { expect(url).toBe('http://x/v1/models'); @@ -55,21 +101,110 @@ describe('createLlmClient (injectable transport)', () => { expect(await llm.listModels()).toEqual(['a', 'b']); }); - it('chat throws on non-2xx with the HTTP status in the message', async () => { - const transport = async () => - ({ - ok: false, - status: 503, - text: async () => 'upstream down', - json: async () => ({}), - } as unknown as Response); - const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm' }); - await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 503/); + it('chat retries 5xx and succeeds once the transport recovers (500, 500 → 200)', async () => { + const sent: { url: string; init?: RequestInit }[] = []; + const transport = async (url: string, init?: RequestInit) => { + sent.push({ url, init }); + if (sent.length < 3) return httpError(500); + return json({ choices: [{ message: { content: 'recovered' } }] }); + }; + const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 }); + const out = await llm.chat([{ role: 'user', content: 'hi' }]); + expect(out).toBe('recovered'); + expect(sent).toHaveLength(3); // two failures, then success on the third attempt }); - it('chat throws when the response has no content', async () => { - const transport = async () => json({ choices: [] }); - const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm' }); + it('chat gives up after exactly 3 attempts when the transport always returns 5xx', async () => { + const sent: { url: string; init?: RequestInit }[] = []; + const transport = async (url: string, init?: RequestInit) => { + sent.push({ url, init }); + return httpError(500, 'still down'); + }; + const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 }); + await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 500/); + expect(sent).toHaveLength(3); + }); + + it('chat does not retry on 4xx (deterministic client error → first attempt only)', async () => { + const sent: { url: string; init?: RequestInit }[] = []; + const transport = async (url: string, init?: RequestInit) => { + sent.push({ url, init }); + return httpError(400, 'bad request'); + }; + const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 }); + await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 400/); + expect(sent).toHaveLength(1); + }); + + it('chat retries a null content (truncation) and then succeeds (null → content)', async () => { + const sent: { url: string; init?: RequestInit }[] = []; + const transport = async (url: string, init?: RequestInit) => { + sent.push({ url, init }); + if (sent.length === 1) return json({ choices: [{ message: { content: null } }] }); + return json({ choices: [{ message: { content: 'real content' } }] }); + }; + const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 }); + const out = await llm.chat([{ role: 'user', content: 'hi' }]); + expect(out).toBe('real content'); + expect(sent).toHaveLength(2); + }); + + it('chat rejects with the HTTP status after 3 attempts when 5xx never recovers', async () => { + const sent: { url: string; init?: RequestInit }[] = []; + const transport = async (url: string, init?: RequestInit) => { + sent.push({ url, init }); + return httpError(503); + }; + const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 }); + await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 503/); + expect(sent).toHaveLength(3); + }); + + it('chat rejects with a content error after 3 attempts when content is never present', async () => { + const sent: { url: string; init?: RequestInit }[] = []; + const transport = async (url: string, init?: RequestInit) => { + sent.push({ url, init }); + return json({ choices: [] }); + }; + const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 }); await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/no choices/); + expect(sent).toHaveLength(3); + }); + + it('chat logs one line on success and one on final failure', async () => { + const logSpy = vi.spyOn(console, 'log'); + const errorSpy = vi.spyOn(console, 'error'); + + const okTransport = async () => + json({ choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }] }); + const okLlm = createLlmClient({ transport: okTransport, apiBase: 'http://x/v1', model: 'm' }); + await okLlm.chat([{ role: 'user', content: 'hi' }]); + expect(logSpy).toHaveBeenCalledWith( + '[llm] chat ok', + expect.objectContaining({ ms: expect.any(Number), finishReason: 'stop' }), + ); + + const failSent: { url: string; init?: RequestInit }[] = []; + const failTransport = async (url: string, init?: RequestInit) => { + failSent.push({ url, init }); + return httpError(502, 'bad gateway'); + }; + const failLlm = createLlmClient({ + transport: failTransport, + apiBase: 'http://x/v1', + model: 'm', + retryBaseDelayMs: 1, + }); + await expect(failLlm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 502/); + expect(failSent).toHaveLength(3); + expect(errorSpy).toHaveBeenCalledWith( + '[llm] chat failed', + expect.objectContaining({ + attempts: 3, + lastStatus: 502, + message: expect.stringContaining('HTTP 502'), + totalMs: expect.any(Number), + }), + ); }); }); diff --git a/server/llm.ts b/server/llm.ts index 7aa9c33..acc1b46 100644 --- a/server/llm.ts +++ b/server/llm.ts @@ -1,18 +1,28 @@ // Thin OpenAI chat-completions client over native fetch (mirrors mesh-bot/src/llm). // The transport is injectable so unit tests and the deterministic e2e stub can // replace the network call. Config is read from env, never hardcoded: -// LLM_API_BASE (default https://ai.kkhost.pl/v1 — a vLLM endpoint, no API key) -// LLM_MODEL (default qwen3.8-27b — a vLLM *reasoning* model) -// LLM_TIMEOUT_MS (default 180000 — the 27B reasoning model can be slow) +// LLM_API_BASE (default https://ai.kkhost.pl/v1 — a vLLM endpoint, no API key) +// LLM_MODEL (default qwen3.8-27b — a vLLM *reasoning* model) +// LLM_TIMEOUT_MS (default 180000 — the 27B reasoning model can be slow) +// LLM_MAX_TOKENS (default 4000 — max *output* tokens per request) +// LLM_REASONING_EFFORT (default "low" — sent as top-level `reasoning_effort`; +// empty or "off" omits the field entirely) // // The live model is a REASONING model: it emits its chain-of-thought in the -// response's `reasoning` field and the final answer in `content`. It therefore -// needs a generous `max_tokens` budget to finish thinking before writing -// `content` — a too-small budget makes it run out mid-reasoning and return a -// null `content`. We always send an explicit, generous `max_tokens` (the -// OpenAI/vLLM wire key — NOT camelCase `maxTokens`, which the server ignores) -// and read only `choices[0].message.content`, ignoring `reasoning`. No auth -// header is added (the endpoint requires no key). +// response's `reasoning` field and the final answer in `content`. Sending +// top-level `reasoning_effort: "low"` cuts the CoT by ~85% (~4000 → +// ~400–600 tokens), so it fits comfortably inside the `max_tokens` budget and +// `content` comes back populated. (Other knobs misbehave on this build: +// "minimal" produced MORE CoT than "low", `chat_template_kwargs: +// {enable_thinking:false}` intermittently emits invalid JSON, and `/no_think` +// is silently ignored.) We read only `choices[0].message.content`, ignoring +// `reasoning`. No auth header is added (the endpoint requires no key). +// +// Reliability: `chat()` retries transient failures — timeout/abort, HTTP 5xx, +// network errors, and null/empty `content` (truncation safety net) — with up +// to 3 total attempts and 2s→3s backoff (±20% jitter). HTTP 4xx is a +// deterministic client error and throws immediately, with no retry. Every +// call is logged: one line on success, one on final failure. export interface ChatMessage { role: 'system' | 'user' | 'assistant'; @@ -36,14 +46,29 @@ export const DEFAULT_BASE = 'https://ai.kkhost.pl/v1'; export const DEFAULT_MODEL = 'qwen3.8-27b'; /** - * Generous token budget for every chat request. The live qwen3.8-27b is a + * Default max *output* tokens per chat request. The live qwen3.8-27b is a * reasoning model: it spends `reasoning_tokens` on its chain-of-thought before - * writing the final answer into `content`. A small budget makes it run out - * mid-reasoning and return a null `content`, so we give it room to finish both. - * `max_tokens` is always set in the request body, even when a caller omits it. + * writing the final answer into `content`. With `reasoning_effort: "low"` a + * realistic turn uses ~800–1100 output tokens, so 4000 is ~4× headroom while + * staying under the upstream ~90s proxy wall. `max_tokens` is always set in the + * request body (the vLLM/OpenAI wire key — NOT camelCase `maxTokens`, which the + * server ignores), even when a caller omits it. */ export const DEFAULT_MAX_TOKENS = 4000; +/** + * Default for `LLM_REASONING_EFFORT`. Sending top-level `reasoning_effort: + * "low"` to the vLLM model cuts its chain-of-thought by ~85%, which is what + * keeps `content` from being truncated to null. + */ +export const DEFAULT_REASONING_EFFORT = 'low'; + +/** Total attempts `chat()` makes before giving up (1 initial + 2 retries). */ +export const MAX_ATTEMPTS = 3; + +/** Base retry backoff: retry 1 waits 2000ms, retry 2 waits 3000ms, each ±20% jitter. */ +const RETRY_BASE_DELAY_MS = 2000; + const fetchTransport: Transport = (url, init) => fetch(url, init); export interface LlmClientDeps { @@ -51,6 +76,91 @@ export interface LlmClientDeps { apiBase?: string; model?: string; timeoutMs?: number; + maxTokens?: number; + reasoningEffort?: string; + /** + * Base delay (ms) for the retry backoff, overriding RETRY_BASE_DELAY_MS. + * Mostly useful in tests to keep the suite fast. + */ + retryBaseDelayMs?: number; +} + +/** parseInt with a fallback for unset/empty/malformed values. */ +function envInt(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const n = parseInt(raw, 10); + return Number.isFinite(n) && n > 0 ? n : fallback; +} + +/** Trimmed env value, or undefined when unset/empty. */ +function envStr(name: string): string | undefined { + const raw = process.env[name]?.trim(); + return raw ? raw : undefined; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Backoff before retry `n` (1-based): `2000 * 1.5^(n-1)`, ±20% jitter. */ +function backoffDelay(retry: number, baseMs: number): number { + const base = baseMs * 1.5 ** (retry - 1); + return base * (0.8 + Math.random() * 0.4); +} + +/** Network-level error codes that mark a transient failure. */ +const RETRYABLE_NET_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ECONNABORTED', + 'EPIPE', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'ETIMEDOUT', + 'EAI_AGAIN', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_SOCKET', + 'UND_ERR_REQ_SOCKET', + 'UND_ERR_REQ_RETRY', +]); + +/** True for a timeout/abort (an AbortSignal that fired, a TimeoutError, or AbortError). */ +function isTimeoutOrAbort(err: unknown): boolean { + let e: unknown = err; + for (let i = 0; i < 3 && e; i++) { + if (e instanceof Error && (e.name === 'TimeoutError' || e.name === 'AbortError')) return true; + e = (e as { cause?: unknown })?.cause; + } + return false; +} + +/** True for a transient network failure (ECONNRESET / ECONNREFUSED / socket closed / …). */ +function isNetworkError(err: unknown): boolean { + let e: unknown = err; + let msg = ''; + for (let i = 0; i < 3 && e; i++) { + if (e instanceof Error) { + msg += ` ${e.message}`; + const code = (e as Error & { code?: string }).code; + if (code && RETRYABLE_NET_CODES.has(code)) return true; + } else { + msg += ` ${String(e)}`; + } + e = (e as { cause?: unknown })?.cause; + } + return /socket closed|socket hang up|connection reset|connection refused|other side closed|fetch failed/i.test( + msg, + ); +} + +/** One transport attempt, classified as success / retryable / fatal. */ +interface Attempt { + status?: number; + error?: Error; + retryable: boolean; + content?: string; + finishReason?: string | null; } export function createLlmClient(deps: LlmClientDeps = {}): LlmClient { @@ -61,36 +171,111 @@ export function createLlmClient(deps: LlmClientDeps = {}): LlmClient { deps.timeoutMs ? String(deps.timeoutMs) : (process.env.LLM_TIMEOUT_MS ?? '180000'), 10, ); + const maxTokens = deps.maxTokens ?? envInt('LLM_MAX_TOKENS', DEFAULT_MAX_TOKENS); + const effort = deps.reasoningEffort ?? envStr('LLM_REASONING_EFFORT') ?? DEFAULT_REASONING_EFFORT; + // "off" (case-insensitive) or empty disables the thinking knob entirely. + const sendEffort = effort !== '' && effort.toLowerCase() !== 'off'; + const retryBaseMs = deps.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS; + + /** + * One POST to /chat/completions. Classifies the outcome: + * - success → content + finish_reason + * - retryable → timeout/abort, HTTP 5xx, network error, null/empty content + * - fatal → HTTP 4xx (deterministic client error) or an unexpected throw + */ + async function attemptOnce(payload: string, signal: AbortSignal): Promise { + let res: Response; + try { + res = await transport(`${apiBase}/chat/completions`, { + method: 'POST', + // No auth header: the ai.kkhost.pl vLLM endpoint requires no API key. + headers: { 'content-type': 'application/json' }, + body: payload, + signal, + }); + } catch (err) { + const e = err instanceof Error ? err : new Error(String(err)); + // The transport threw: our AbortSignal fired (timeout/abort) or a + // network-level failure (ECONNRESET / ECONNREFUSED / socket closed). + return { error: e, retryable: signal.aborted || isTimeoutOrAbort(e) || isNetworkError(e) }; + } - async function chat(messages: ChatMessage[], opts: ChatOptions = {}): Promise { - const payload: Record = { model, messages }; - if (opts.temperature != null) payload.temperature = opts.temperature; - // Always send an explicit, generous `max_tokens` (the vLLM/OpenAI wire key — - // camelCase `maxTokens` is ignored by the server) so the reasoning model has - // room to finish `reasoning` before emitting `content`. - payload.max_tokens = opts.maxTokens ?? DEFAULT_MAX_TOKENS; - const res = await transport(`${apiBase}/chat/completions`, { - method: 'POST', - // No auth header: the ai.kkhost.pl vLLM endpoint requires no API key. - headers: { 'content-type': 'application/json' }, - body: JSON.stringify(payload), - signal: AbortSignal.timeout(timeoutMs), - }); if (!res.ok) { const text = await res.text().catch(() => ''); - throw new Error(`chat failed: HTTP ${res.status} ${text.slice(0, 200)}`); + return { + status: res.status, + error: new Error(`chat failed: HTTP ${res.status} ${text.slice(0, 200)}`), + // 4xx is a deterministic client error → do not retry; 5xx is retryable. + retryable: res.status >= 500 && res.status < 600, + }; } - // Reasoning models also return their chain-of-thought in a separate - // `reasoning` field. We deliberately ignore it and use only the final - // answer, which lives in `content`. + const body = (await res.json()) as { - choices?: { message?: { content?: string; reasoning?: string | null } }[]; + choices?: { + finish_reason?: string | null; + message?: { content?: string; reasoning?: string | null }; + }[]; }; + // The reasoning model can exhaust its output budget on the CoT and return + // a null `content` — treat that as a retryable truncation, not a success. const content = body.choices?.[0]?.message?.content; - if (typeof content !== 'string') { - throw new Error('chat failed: no choices[0].message.content in response'); + if (typeof content !== 'string' || content === '') { + return { + status: res.status, + error: new Error('chat failed: no choices[0].message.content in response'), + retryable: true, + }; } - return content; + return { + status: res.status, + retryable: false, + content, + finishReason: body.choices?.[0]?.finish_reason ?? null, + }; + } + + async function chat(messages: ChatMessage[], opts: ChatOptions = {}): Promise { + const payload: Record = { + model, + messages, + // The wire key is `max_tokens` (snake_case) — camelCase is ignored by the + // server. With `reasoning_effort: "low"` the CoT fits inside this budget, + // so `content` comes back populated. + max_tokens: opts.maxTokens ?? maxTokens, + }; + if (opts.temperature != null) payload.temperature = opts.temperature; + if (sendEffort) payload.reasoning_effort = effort; + + const totalStart = Date.now(); + let lastStatus: number | undefined; + let lastError: Error = new Error('chat failed: no attempt completed'); + + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + const out = await attemptOnce(JSON.stringify(payload), AbortSignal.timeout(timeoutMs)); + if (out.status !== undefined) lastStatus = out.status; + if (out.error) lastError = out.error; + + if (out.content !== undefined) { + console.log('[llm] chat ok', { + ms: Date.now() - totalStart, + finishReason: out.finishReason ?? null, + }); + return out.content; + } + // Give up on a non-retryable failure, or when the last attempt is spent. + if (!out.retryable || attempt === MAX_ATTEMPTS) { + console.error('[llm] chat failed', { + attempts: attempt, + lastStatus, + message: lastError.message, + totalMs: Date.now() - totalStart, + }); + throw lastError; + } + await sleep(backoffDelay(attempt, retryBaseMs)); + } + // Unreachable: every loop path returns or throws. + throw lastError; } async function listModels(): Promise {