// Thin OpenAI chat-completions client over native fetch (mirrors mesh-bot/src/llm). // The transport is injectable so unit tests and the deterministic e2e stub can // replace the network call. Config is read from env, never hardcoded: // LLM_API_BASE (default https://ai.kkhost.pl/v1 — a vLLM endpoint, no API key) // LLM_MODEL (default qwen3.8-27b — a vLLM *reasoning* model) // LLM_TIMEOUT_MS (default 180000 — the 27B reasoning model can be slow) // // The live model is a REASONING model: it emits its chain-of-thought in the // response's `reasoning` field and the final answer in `content`. It therefore // needs a generous `max_tokens` budget to finish thinking before writing // `content` — a too-small budget makes it run out mid-reasoning and return a // null `content`. We always send an explicit, generous `max_tokens` (the // OpenAI/vLLM wire key — NOT camelCase `maxTokens`, which the server ignores) // and read only `choices[0].message.content`, ignoring `reasoning`. No auth // header is added (the endpoint requires no key). export interface ChatMessage { role: 'system' | 'user' | 'assistant'; content: string; } /** A network call. The default implementation is native fetch. */ export type Transport = (url: string, init?: RequestInit) => Promise; export interface ChatOptions { maxTokens?: number; temperature?: number; } export interface LlmClient { chat(messages: ChatMessage[], opts?: ChatOptions): Promise; listModels(): Promise; } export const DEFAULT_BASE = 'https://ai.kkhost.pl/v1'; export const DEFAULT_MODEL = 'qwen3.8-27b'; /** * Generous token budget for every chat request. The live qwen3.8-27b is a * reasoning model: it spends `reasoning_tokens` on its chain-of-thought before * writing the final answer into `content`. A small budget makes it run out * mid-reasoning and return a null `content`, so we give it room to finish both. * `max_tokens` is always set in the request body, even when a caller omits it. */ export const DEFAULT_MAX_TOKENS = 4000; const fetchTransport: Transport = (url, init) => fetch(url, init); export interface LlmClientDeps { transport?: Transport; apiBase?: string; model?: string; timeoutMs?: number; } export function createLlmClient(deps: LlmClientDeps = {}): LlmClient { const transport = deps.transport ?? fetchTransport; const apiBase = (deps.apiBase ?? process.env.LLM_API_BASE ?? DEFAULT_BASE).replace(/\/+$/, ''); const model = deps.model ?? process.env.LLM_MODEL ?? DEFAULT_MODEL; const timeoutMs = parseInt( deps.timeoutMs ? String(deps.timeoutMs) : (process.env.LLM_TIMEOUT_MS ?? '180000'), 10, ); async function chat(messages: ChatMessage[], opts: ChatOptions = {}): Promise { const payload: Record = { model, messages }; if (opts.temperature != null) payload.temperature = opts.temperature; // Always send an explicit, generous `max_tokens` (the vLLM/OpenAI wire key — // camelCase `maxTokens` is ignored by the server) so the reasoning model has // room to finish `reasoning` before emitting `content`. payload.max_tokens = opts.maxTokens ?? DEFAULT_MAX_TOKENS; const res = await transport(`${apiBase}/chat/completions`, { method: 'POST', // No auth header: the ai.kkhost.pl vLLM endpoint requires no API key. headers: { 'content-type': 'application/json' }, body: JSON.stringify(payload), signal: AbortSignal.timeout(timeoutMs), }); if (!res.ok) { const text = await res.text().catch(() => ''); throw new Error(`chat failed: HTTP ${res.status} ${text.slice(0, 200)}`); } // Reasoning models also return their chain-of-thought in a separate // `reasoning` field. We deliberately ignore it and use only the final // answer, which lives in `content`. const body = (await res.json()) as { choices?: { message?: { content?: string; reasoning?: string | null } }[]; }; const content = body.choices?.[0]?.message?.content; if (typeof content !== 'string') { throw new Error('chat failed: no choices[0].message.content in response'); } return content; } async function listModels(): Promise { const res = await transport(`${apiBase}/models`, { signal: AbortSignal.timeout(timeoutMs), }); if (!res.ok) throw new Error(`listModels failed: HTTP ${res.status}`); const body = (await res.json()) as { data?: { id: string }[] }; return (body.data ?? []).map((m) => m.id); } return { chat, listModels }; }