// Thin OpenAI chat-completions client over native fetch (mirrors mesh-bot/src/llm). // The transport is injectable so unit tests and the deterministic e2e stub can // replace the network call. Config is read from env, never hardcoded: // LLM_API_BASE (default https://ai.kkhost.pl/v1 — a vLLM endpoint, no API key) // LLM_MODEL (default qwen3.8-27b — a vLLM *reasoning* model) // LLM_TIMEOUT_MS (default 180000 — the 27B reasoning model can be slow) // LLM_MAX_TOKENS (default 4000 — max *output* tokens per request) // LLM_REASONING_EFFORT (default "low" — sent as top-level `reasoning_effort`; // empty or "off" omits the field entirely) // // The live model is a REASONING model: it emits its chain-of-thought in the // response's `reasoning` field and the final answer in `content`. Sending // top-level `reasoning_effort: "low"` cuts the CoT by ~85% (~4000 → // ~400–600 tokens), so it fits comfortably inside the `max_tokens` budget and // `content` comes back populated. (Other knobs misbehave on this build: // "minimal" produced MORE CoT than "low", `chat_template_kwargs: // {enable_thinking:false}` intermittently emits invalid JSON, and `/no_think` // is silently ignored.) We read only `choices[0].message.content`, ignoring // `reasoning`. No auth header is added (the endpoint requires no key). // // Reliability: `chat()` retries transient failures — timeout/abort, HTTP 5xx, // network errors, and null/empty `content` (truncation safety net) — with up // to 3 total attempts and 2s→3s backoff (±20% jitter). HTTP 4xx is a // deterministic client error and throws immediately, with no retry. Every // call is logged: one line on success, one on final failure. export interface ChatMessage { role: 'system' | 'user' | 'assistant'; content: string; } /** A network call. The default implementation is native fetch. */ export type Transport = (url: string, init?: RequestInit) => Promise; export interface ChatOptions { maxTokens?: number; temperature?: number; } export interface LlmClient { chat(messages: ChatMessage[], opts?: ChatOptions): Promise; listModels(): Promise; } export const DEFAULT_BASE = 'https://ai.kkhost.pl/v1'; export const DEFAULT_MODEL = 'qwen3.8-27b'; /** * Default max *output* tokens per chat request. The live qwen3.8-27b is a * reasoning model: it spends `reasoning_tokens` on its chain-of-thought before * writing the final answer into `content`. With `reasoning_effort: "low"` a * realistic turn uses ~800–1100 output tokens, so 4000 is ~4× headroom while * staying under the upstream ~90s proxy wall. `max_tokens` is always set in the * request body (the vLLM/OpenAI wire key — NOT camelCase `maxTokens`, which the * server ignores), even when a caller omits it. */ export const DEFAULT_MAX_TOKENS = 4000; /** * Default for `LLM_REASONING_EFFORT`. Sending top-level `reasoning_effort: * "low"` to the vLLM model cuts its chain-of-thought by ~85%, which is what * keeps `content` from being truncated to null. */ export const DEFAULT_REASONING_EFFORT = 'low'; /** Total attempts `chat()` makes before giving up (1 initial + 2 retries). */ export const MAX_ATTEMPTS = 3; /** Base retry backoff: retry 1 waits 2000ms, retry 2 waits 3000ms, each ±20% jitter. */ const RETRY_BASE_DELAY_MS = 2000; const fetchTransport: Transport = (url, init) => fetch(url, init); export interface LlmClientDeps { transport?: Transport; apiBase?: string; model?: string; timeoutMs?: number; maxTokens?: number; reasoningEffort?: string; /** * Base delay (ms) for the retry backoff, overriding RETRY_BASE_DELAY_MS. * Mostly useful in tests to keep the suite fast. */ retryBaseDelayMs?: number; } /** parseInt with a fallback for unset/empty/malformed values. */ function envInt(name: string, fallback: number): number { const raw = process.env[name]?.trim(); if (!raw) return fallback; const n = parseInt(raw, 10); return Number.isFinite(n) && n > 0 ? n : fallback; } /** Trimmed env value, or undefined when unset/empty. */ function envStr(name: string): string | undefined { const raw = process.env[name]?.trim(); return raw ? raw : undefined; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } /** Backoff before retry `n` (1-based): `2000 * 1.5^(n-1)`, ±20% jitter. */ function backoffDelay(retry: number, baseMs: number): number { const base = baseMs * 1.5 ** (retry - 1); return base * (0.8 + Math.random() * 0.4); } /** Network-level error codes that mark a transient failure. */ const RETRYABLE_NET_CODES = new Set([ 'ECONNRESET', 'ECONNREFUSED', 'ECONNABORTED', 'EPIPE', 'EHOSTUNREACH', 'ENETUNREACH', 'ETIMEDOUT', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_SOCKET', 'UND_ERR_REQ_SOCKET', 'UND_ERR_REQ_RETRY', ]); /** True for a timeout/abort (an AbortSignal that fired, a TimeoutError, or AbortError). */ function isTimeoutOrAbort(err: unknown): boolean { let e: unknown = err; for (let i = 0; i < 3 && e; i++) { if (e instanceof Error && (e.name === 'TimeoutError' || e.name === 'AbortError')) return true; e = (e as { cause?: unknown })?.cause; } return false; } /** True for a transient network failure (ECONNRESET / ECONNREFUSED / socket closed / …). */ function isNetworkError(err: unknown): boolean { let e: unknown = err; let msg = ''; for (let i = 0; i < 3 && e; i++) { if (e instanceof Error) { msg += ` ${e.message}`; const code = (e as Error & { code?: string }).code; if (code && RETRYABLE_NET_CODES.has(code)) return true; } else { msg += ` ${String(e)}`; } e = (e as { cause?: unknown })?.cause; } return /socket closed|socket hang up|connection reset|connection refused|other side closed|fetch failed/i.test( msg, ); } /** One transport attempt, classified as success / retryable / fatal. */ interface Attempt { status?: number; error?: Error; retryable: boolean; content?: string; finishReason?: string | null; } export function createLlmClient(deps: LlmClientDeps = {}): LlmClient { const transport = deps.transport ?? fetchTransport; const apiBase = (deps.apiBase ?? process.env.LLM_API_BASE ?? DEFAULT_BASE).replace(/\/+$/, ''); const model = deps.model ?? process.env.LLM_MODEL ?? DEFAULT_MODEL; const timeoutMs = parseInt( deps.timeoutMs ? String(deps.timeoutMs) : (process.env.LLM_TIMEOUT_MS ?? '180000'), 10, ); const maxTokens = deps.maxTokens ?? envInt('LLM_MAX_TOKENS', DEFAULT_MAX_TOKENS); const effort = deps.reasoningEffort ?? envStr('LLM_REASONING_EFFORT') ?? DEFAULT_REASONING_EFFORT; // "off" (case-insensitive) or empty disables the thinking knob entirely. const sendEffort = effort !== '' && effort.toLowerCase() !== 'off'; const retryBaseMs = deps.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS; /** * One POST to /chat/completions. Classifies the outcome: * - success → content + finish_reason * - retryable → timeout/abort, HTTP 5xx, network error, null/empty content * - fatal → HTTP 4xx (deterministic client error) or an unexpected throw */ async function attemptOnce(payload: string, signal: AbortSignal): Promise { let res: Response; try { res = await transport(`${apiBase}/chat/completions`, { method: 'POST', // No auth header: the ai.kkhost.pl vLLM endpoint requires no API key. headers: { 'content-type': 'application/json' }, body: payload, signal, }); } catch (err) { const e = err instanceof Error ? err : new Error(String(err)); // The transport threw: our AbortSignal fired (timeout/abort) or a // network-level failure (ECONNRESET / ECONNREFUSED / socket closed). return { error: e, retryable: signal.aborted || isTimeoutOrAbort(e) || isNetworkError(e) }; } if (!res.ok) { const text = await res.text().catch(() => ''); return { status: res.status, error: new Error(`chat failed: HTTP ${res.status} ${text.slice(0, 200)}`), // 4xx is a deterministic client error → do not retry; 5xx is retryable. retryable: res.status >= 500 && res.status < 600, }; } const body = (await res.json()) as { choices?: { finish_reason?: string | null; message?: { content?: string; reasoning?: string | null }; }[]; }; // The reasoning model can exhaust its output budget on the CoT and return // a null `content` — treat that as a retryable truncation, not a success. const content = body.choices?.[0]?.message?.content; if (typeof content !== 'string' || content === '') { return { status: res.status, error: new Error('chat failed: no choices[0].message.content in response'), retryable: true, }; } return { status: res.status, retryable: false, content, finishReason: body.choices?.[0]?.finish_reason ?? null, }; } async function chat(messages: ChatMessage[], opts: ChatOptions = {}): Promise { const payload: Record = { model, messages, // The wire key is `max_tokens` (snake_case) — camelCase is ignored by the // server. With `reasoning_effort: "low"` the CoT fits inside this budget, // so `content` comes back populated. max_tokens: opts.maxTokens ?? maxTokens, }; if (opts.temperature != null) payload.temperature = opts.temperature; if (sendEffort) payload.reasoning_effort = effort; const totalStart = Date.now(); let lastStatus: number | undefined; let lastError: Error = new Error('chat failed: no attempt completed'); for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { const out = await attemptOnce(JSON.stringify(payload), AbortSignal.timeout(timeoutMs)); if (out.status !== undefined) lastStatus = out.status; if (out.error) lastError = out.error; if (out.content !== undefined) { console.log('[llm] chat ok', { ms: Date.now() - totalStart, finishReason: out.finishReason ?? null, }); return out.content; } // Give up on a non-retryable failure, or when the last attempt is spent. if (!out.retryable || attempt === MAX_ATTEMPTS) { console.error('[llm] chat failed', { attempts: attempt, lastStatus, message: lastError.message, totalMs: Date.now() - totalStart, }); throw lastError; } await sleep(backoffDelay(attempt, retryBaseMs)); } // Unreachable: every loop path returns or throws. throw lastError; } async function listModels(): Promise { const res = await transport(`${apiBase}/models`, { signal: AbortSignal.timeout(timeoutMs), }); if (!res.ok) throw new Error(`listModels failed: HTTP ${res.status}`); const body = (await res.json()) as { data?: { id: string }[] }; return (body.data ?? []).map((m) => m.id); } return { chat, listModels }; }