// Thin OpenAI chat-completions client over native fetch (mirrors mesh-bot/src/llm). // The transport is injectable so unit tests and the deterministic e2e stub can // replace the network call. Config is read from env, never hardcoded: // LLM_API_BASE (default https://ai.kkhost.pl/v1 — a vLLM endpoint, no API key) // LLM_MODEL (default qwen3.8-27b — a vLLM *reasoning* model) // LLM_TIMEOUT_MS (default 180000 — the 27B reasoning model can be slow) // LLM_MAX_TOKENS (default 4000 — max *output* tokens per request) // LLM_REASONING_EFFORT (default "low" — sent as top-level `reasoning_effort`; // empty or "off" omits the field entirely) // // The live model is a REASONING model: it emits its chain-of-thought in the // response's `reasoning` field and the final answer in `content`. Sending // top-level `reasoning_effort: "low"` cuts the CoT by ~85% (~4000 → // ~400–600 tokens), so it fits comfortably inside the `max_tokens` budget and // `content` comes back populated. (Other knobs misbehave on this build: // "minimal" produced MORE CoT than "low", `chat_template_kwargs: // {enable_thinking:false}` intermittently emits invalid JSON, and `/no_think` // is silently ignored.) We read only `choices[0].message.content`, ignoring // `reasoning`. No auth header is added (the endpoint requires no key). // // Reliability: `chat()` retries transient failures — timeout/abort, HTTP 5xx, // network errors, and null/empty `content` (truncation safety net) — with up // to 3 total attempts and 2s→3s backoff (±20% jitter). HTTP 4xx is a // deterministic client error and throws immediately, with no retry. Every // call is logged: one line on success, one on final failure. // // Function calling: `chatWithTools()` extends `chat()` with the OpenAI tool // loop. When the model answers with `finish_reason: "tool_calls"`, the client // runs the provided executor for every requested call, appends the assistant // tool-call message plus one `tool` result message per call, and re-sends the // growing history until the model produces a final `content`. The loop is // bounded to DEFAULT_MAX_TOOL_ROUNDS (6) tool rounds; if the budget is spent // without a final answer, one last *no-tools* request forces the model to // commit. The same retry/backoff/reasoning-effort machinery applies per // request, so an injectable transport only has to forward `tools` in the body // and accept `assistant`/`tool` messages in the history. The deterministic // stub answers plain `content` (no tool calls) and therefore bypasses the // loop in a single round. export interface ChatMessage { role: 'system' | 'user' | 'assistant'; content: string; } /** One OpenAI-style function call the model requests (`finish_reason: "tool_calls"`). */ export interface ToolCallSpec { id: string; type: 'function'; function: { name: string; arguments: string }; } /** A function the model may call (one OpenAI `tools` entry). */ export interface ToolDefinition { type: 'function'; function: { name: string; description?: string; parameters?: Record; }; } /** * The full message vocabulary a request can carry: plain chat messages, plus * assistant messages that request tool calls and the `tool` results that * answer them (the OpenAI function-calling wire shape). */ export type WireMessage = | ChatMessage | { role: 'assistant'; content: string | null; tool_calls: ToolCallSpec[] } | { role: 'tool'; tool_call_id: string; content: string }; /** Executes one tool call; the returned string is fed back to the model as the `tool` result. */ export type ToolExecutor = (name: string, args: Record) => string; /** A network call. The default implementation is native fetch. */ export type Transport = (url: string, init?: RequestInit) => Promise; export interface ChatOptions { maxTokens?: number; temperature?: number; } export interface LlmClient { chat(messages: ChatMessage[], opts?: ChatOptions): Promise; /** * The function-calling variant of chat() — see the file header for the loop * semantics (bounded tool rounds, then a forced final answer). Optional on * the interface: minimal test doubles may omit it, and callers (the * engine's runTurn) fall back to chat() when it is absent. */ chatWithTools?( messages: WireMessage[], tools: ToolDefinition[], executeTool: ToolExecutor, opts?: ChatOptions & { maxRounds?: number }, ): Promise; listModels(): Promise; } export const DEFAULT_BASE = 'https://ai.kkhost.pl/v1'; export const DEFAULT_MODEL = 'qwen3.8-27b'; /** * Default max *output* tokens per chat request. The live qwen3.8-27b is a * reasoning model: it spends `reasoning_tokens` on its chain-of-thought before * writing the final answer into `content`. With `reasoning_effort: "low"` a * realistic turn uses ~800–1100 output tokens, so 4000 is ~4× headroom while * staying under the upstream ~90s proxy wall. `max_tokens` is always set in the * request body (the vLLM/OpenAI wire key — NOT camelCase `maxTokens`, which the * server ignores), even when a caller omits it. */ export const DEFAULT_MAX_TOKENS = 4000; /** * Default for `LLM_REASONING_EFFORT`. Sending top-level `reasoning_effort: * "low"` to the vLLM model cuts its chain-of-thought by ~85%, which is what * keeps `content` from being truncated to null. */ export const DEFAULT_REASONING_EFFORT = 'low'; /** Total attempts per request (chat / chatWithTools) before giving up (1 initial + 2 retries). */ export const MAX_ATTEMPTS = 3; /** * Max tool-call rounds (assistant tool_calls → tool results) a chatWithTools * loop may make before a final no-tools request forces the answer. Keeps a * turn bounded even if the model keeps asking for data; a realistic turn uses * 0–2 rounds. */ export const DEFAULT_MAX_TOOL_ROUNDS = 6; /** Base retry backoff: retry 1 waits 2000ms, retry 2 waits 3000ms, each ±20% jitter. */ const RETRY_BASE_DELAY_MS = 2000; const fetchTransport: Transport = (url, init) => fetch(url, init); export interface LlmClientDeps { transport?: Transport; apiBase?: string; model?: string; timeoutMs?: number; maxTokens?: number; reasoningEffort?: string; /** * Base delay (ms) for the retry backoff, overriding RETRY_BASE_DELAY_MS. * Mostly useful in tests to keep the suite fast. */ retryBaseDelayMs?: number; } /** parseInt with a fallback for unset/empty/malformed values. */ function envInt(name: string, fallback: number): number { const raw = process.env[name]?.trim(); if (!raw) return fallback; const n = parseInt(raw, 10); return Number.isFinite(n) && n > 0 ? n : fallback; } /** Trimmed env value, or undefined when unset/empty. */ function envStr(name: string): string | undefined { const raw = process.env[name]?.trim(); return raw ? raw : undefined; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } /** Backoff before retry `n` (1-based): `2000 * 1.5^(n-1)`, ±20% jitter. */ function backoffDelay(retry: number, baseMs: number): number { const base = baseMs * 1.5 ** (retry - 1); return base * (0.8 + Math.random() * 0.4); } /** Network-level error codes that mark a transient failure. */ const RETRYABLE_NET_CODES = new Set([ 'ECONNRESET', 'ECONNREFUSED', 'ECONNABORTED', 'EPIPE', 'EHOSTUNREACH', 'ENETUNREACH', 'ETIMEDOUT', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_SOCKET', 'UND_ERR_REQ_SOCKET', 'UND_ERR_REQ_RETRY', ]); /** True for a timeout/abort (an AbortSignal that fired, a TimeoutError, or AbortError). */ function isTimeoutOrAbort(err: unknown): boolean { let e: unknown = err; for (let i = 0; i < 3 && e; i++) { if (e instanceof Error && (e.name === 'TimeoutError' || e.name === 'AbortError')) return true; e = (e as { cause?: unknown })?.cause; } return false; } /** True for a transient network failure (ECONNRESET / ECONNREFUSED / socket closed / …). */ function isNetworkError(err: unknown): boolean { let e: unknown = err; let msg = ''; for (let i = 0; i < 3 && e; i++) { if (e instanceof Error) { msg += ` ${e.message}`; const code = (e as Error & { code?: string }).code; if (code && RETRYABLE_NET_CODES.has(code)) return true; } else { msg += ` ${String(e)}`; } e = (e as { cause?: unknown })?.cause; } return /socket closed|socket hang up|connection reset|connection refused|other side closed|fetch failed/i.test( msg, ); } /** Parse the wire `tool_calls` array into validated ToolCallSpecs (malformed entries are dropped). */ function parseToolCalls(raw: unknown): ToolCallSpec[] { if (!Array.isArray(raw)) return []; const out: ToolCallSpec[] = []; for (const tc of raw) { if (!tc || typeof tc !== 'object') continue; const t = tc as Record; const fn = t.function as Record | undefined; if (!fn || typeof fn.name !== 'string' || fn.name === '') continue; const argumentsText = typeof fn.arguments === 'string' ? fn.arguments : fn.arguments != null ? JSON.stringify(fn.arguments) : '{}'; out.push({ id: typeof t.id === 'string' && t.id !== '' ? t.id : `call_${out.length}`, type: 'function', function: { name: fn.name, arguments: argumentsText }, }); } return out; } /** One transport attempt, classified as success / retryable / fatal. */ interface Attempt { status?: number; error?: Error; retryable: boolean; content?: string; finishReason?: string | null; toolCalls?: ToolCallSpec[]; } export function createLlmClient(deps: LlmClientDeps = {}): LlmClient { const transport = deps.transport ?? fetchTransport; const apiBase = (deps.apiBase ?? process.env.LLM_API_BASE ?? DEFAULT_BASE).replace(/\/+$/, ''); const model = deps.model ?? process.env.LLM_MODEL ?? DEFAULT_MODEL; const timeoutMs = parseInt( deps.timeoutMs ? String(deps.timeoutMs) : (process.env.LLM_TIMEOUT_MS ?? '180000'), 10, ); const maxTokens = deps.maxTokens ?? envInt('LLM_MAX_TOKENS', DEFAULT_MAX_TOKENS); const effort = deps.reasoningEffort ?? envStr('LLM_REASONING_EFFORT') ?? DEFAULT_REASONING_EFFORT; // "off" (case-insensitive) or empty disables the thinking knob entirely. const sendEffort = effort !== '' && effort.toLowerCase() !== 'off'; const retryBaseMs = deps.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS; /** * One POST to /chat/completions. Classifies the outcome: * - success → content and/or tool_calls * - retryable → timeout/abort, HTTP 5xx, network error, null/empty content * - fatal → HTTP 4xx (deterministic client error) or an unexpected throw */ async function attemptOnce(payload: string, signal: AbortSignal): Promise { let res: Response; try { res = await transport(`${apiBase}/chat/completions`, { method: 'POST', // No auth header: the ai.kkhost.pl vLLM endpoint requires no API key. headers: { 'content-type': 'application/json' }, body: payload, signal, }); } catch (err) { const e = err instanceof Error ? err : new Error(String(err)); // The transport threw: our AbortSignal fired (timeout/abort) or a // network-level failure (ECONNRESET / ECONNREFUSED / socket closed). return { error: e, retryable: signal.aborted || isTimeoutOrAbort(e) || isNetworkError(e) }; } if (!res.ok) { const text = await res.text().catch(() => ''); return { status: res.status, error: new Error(`chat failed: HTTP ${res.status} ${text.slice(0, 200)}`), // 4xx is a deterministic client error → do not retry; 5xx is retryable. retryable: res.status >= 500 && res.status < 600, }; } const body = (await res.json()) as { choices?: { finish_reason?: string | null; message?: { content?: string; reasoning?: string | null; tool_calls?: unknown }; }[]; }; const choice = body.choices?.[0]; const message = choice?.message; // A tool-call round is a SUCCESS even though `content` is null there: the // model is asking for data, not failing (never treat it as truncation). const toolCalls = parseToolCalls(message?.tool_calls); if (toolCalls.length > 0) { return { status: res.status, retryable: false, content: typeof message?.content === 'string' && message.content !== '' ? message.content : undefined, finishReason: choice?.finish_reason ?? null, toolCalls, }; } // The reasoning model can exhaust its output budget on the CoT and return // a null `content` — treat that as a retryable truncation, not a success. const content = message?.content; if (typeof content !== 'string' || content === '') { return { status: res.status, error: new Error('chat failed: no choices[0].message.content in response'), retryable: true, }; } return { status: res.status, retryable: false, content, finishReason: choice?.finish_reason ?? null, }; } /** Build one /chat/completions request body. `tools` is only included when non-empty, so plain chat() keeps its exact legacy shape. */ function buildPayload( messages: WireMessage[], tools: ToolDefinition[] | undefined, opts: ChatOptions, ): Record { const payload: Record = { model, messages, // The wire key is `max_tokens` (snake_case) — camelCase is ignored by the // server. With `reasoning_effort: "low"` the CoT fits inside this budget, // so `content` comes back populated. max_tokens: opts.maxTokens ?? maxTokens, }; if (opts.temperature != null) payload.temperature = opts.temperature; if (sendEffort) payload.reasoning_effort = effort; if (tools && tools.length > 0) payload.tools = tools; return payload; } /** * One request with the standard retry/backoff: up to MAX_ATTEMPTS per * request; resolves with a successful Attempt (content and/or toolCalls) or * throws the last error once the attempts are spent. */ async function requestWithRetry(payload: Record): Promise { const totalStart = Date.now(); let lastStatus: number | undefined; let lastError: Error = new Error('chat failed: no attempt completed'); for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { const out = await attemptOnce(JSON.stringify(payload), AbortSignal.timeout(timeoutMs)); if (out.status !== undefined) lastStatus = out.status; if (out.error) lastError = out.error; if (out.content !== undefined || out.toolCalls !== undefined) { console.log('[llm] chat ok', { ms: Date.now() - totalStart, finishReason: out.finishReason ?? null, toolCalls: out.toolCalls?.length ?? 0, }); return out; } // Give up on a non-retryable failure, or when the last attempt is spent. if (!out.retryable || attempt === MAX_ATTEMPTS) { console.error('[llm] chat failed', { attempts: attempt, lastStatus, message: lastError.message, totalMs: Date.now() - totalStart, }); throw lastError; } await sleep(backoffDelay(attempt, retryBaseMs)); } // Unreachable: every loop path returns or throws. throw lastError; } async function chat(messages: ChatMessage[], opts: ChatOptions = {}): Promise { const out = await requestWithRetry(buildPayload(messages, undefined, opts)); return out.content ?? ''; } /** * The function-calling loop — see the file header. Bounded: at most * `maxRounds` tool rounds, then one final no-tools request forces an answer. * Throws like chat() only on transport-level exhaustion (the engine maps that * to a 502; bad *content* never throws, it is parsed defensively downstream). */ async function chatWithTools( initial: WireMessage[], tools: ToolDefinition[], executeTool: ToolExecutor, opts: ChatOptions & { maxRounds?: number } = {}, ): Promise { const maxRounds = Math.max(1, opts.maxRounds ?? DEFAULT_MAX_TOOL_ROUNDS); const messages: WireMessage[] = [...initial]; for (let round = 0; round < maxRounds; round++) { const out = await requestWithRetry(buildPayload(messages, tools, opts)); // Tool calls win over any partial content: the model asked for data // before it finished. if (out.toolCalls && out.toolCalls.length > 0) { messages.push({ role: 'assistant', content: out.content ?? null, tool_calls: out.toolCalls, }); for (const tc of out.toolCalls) { let result: string; try { const parsed: unknown = tc.function.arguments !== '' ? JSON.parse(tc.function.arguments) : {}; const args = parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record) : {}; result = executeTool(tc.function.name, args); } catch (err) { // Malformed arguments are not fatal: hand the model an error // result so it can re-call with valid arguments or move on. result = JSON.stringify({ error: `invalid tool arguments: ${err instanceof Error ? err.message : String(err)}`, }); } messages.push({ role: 'tool', tool_call_id: tc.id, content: result }); } continue; } if (out.content !== undefined) return out.content; } // The tool budget is spent without a final answer: one last no-tools // request forces the model to commit. messages.push({ role: 'user', content: 'You have reached your tool budget. Answer now with the final result only.', }); const out = await requestWithRetry(buildPayload(messages, undefined, opts)); return out.content ?? ''; } async function listModels(): Promise { const res = await transport(`${apiBase}/models`, { signal: AbortSignal.timeout(timeoutMs), }); if (!res.ok) throw new Error(`listModels failed: HTTP ${res.status}`); const body = (await res.json()) as { data?: { id: string }[] }; return (body.data ?? []).map((m) => m.id); } return { chat, chatWithTools, listModels }; }