History/server/llm.ts

// Thin OpenAI chat-completions client over native fetch (mirrors mesh-bot/src/llm).
// The transport is injectable so unit tests and the deterministic e2e stub can
// replace the network call. Config is read from env, never hardcoded:
//   LLM_API_BASE   (default https://ai.kkhost.pl/v1 — a vLLM endpoint, no API key)
//   LLM_MODEL      (default qwen3.8-27b — a vLLM *reasoning* model)
//   LLM_TIMEOUT_MS (default 180000 — the 27B reasoning model can be slow)
//
// The live model is a REASONING model: it emits its chain-of-thought in the
// response's `reasoning` field and the final answer in `content`. It therefore
// needs a generous `max_tokens` budget to finish thinking before writing
// `content` — a too-small budget makes it run out mid-reasoning and return a
// null `content`. We always send an explicit, generous `max_tokens` (the
// OpenAI/vLLM wire key — NOT camelCase `maxTokens`, which the server ignores)
// and read only `choices[0].message.content`, ignoring `reasoning`. No auth
// header is added (the endpoint requires no key).

export interface ChatMessage {
  role: 'system' | 'user' | 'assistant';
  content: string;
}

/** A network call. The default implementation is native fetch. */
export type Transport = (url: string, init?: RequestInit) => Promise<Response>;

export interface ChatOptions {
  maxTokens?: number;
  temperature?: number;
}

export interface LlmClient {
  chat(messages: ChatMessage[], opts?: ChatOptions): Promise<string>;
  listModels(): Promise<string[]>;
}

export const DEFAULT_BASE = 'https://ai.kkhost.pl/v1';
export const DEFAULT_MODEL = 'qwen3.8-27b';

/**
 * Generous token budget for every chat request. The live qwen3.8-27b is a
 * reasoning model: it spends `reasoning_tokens` on its chain-of-thought before
 * writing the final answer into `content`. A small budget makes it run out
 * mid-reasoning and return a null `content`, so we give it room to finish both.
 * `max_tokens` is always set in the request body, even when a caller omits it.
 */
export const DEFAULT_MAX_TOKENS = 4000;

const fetchTransport: Transport = (url, init) => fetch(url, init);

export interface LlmClientDeps {
  transport?: Transport;
  apiBase?: string;
  model?: string;
  timeoutMs?: number;
}

export function createLlmClient(deps: LlmClientDeps = {}): LlmClient {
  const transport = deps.transport ?? fetchTransport;
  const apiBase = (deps.apiBase ?? process.env.LLM_API_BASE ?? DEFAULT_BASE).replace(/\/+$/, '');
  const model = deps.model ?? process.env.LLM_MODEL ?? DEFAULT_MODEL;
  const timeoutMs = parseInt(
    deps.timeoutMs ? String(deps.timeoutMs) : (process.env.LLM_TIMEOUT_MS ?? '180000'),
    10,
  );

  async function chat(messages: ChatMessage[], opts: ChatOptions = {}): Promise<string> {
    const payload: Record<string, unknown> = { model, messages };
    if (opts.temperature != null) payload.temperature = opts.temperature;
    // Always send an explicit, generous `max_tokens` (the vLLM/OpenAI wire key —
    // camelCase `maxTokens` is ignored by the server) so the reasoning model has
    // room to finish `reasoning` before emitting `content`.
    payload.max_tokens = opts.maxTokens ?? DEFAULT_MAX_TOKENS;
    const res = await transport(`${apiBase}/chat/completions`, {
      method: 'POST',
      // No auth header: the ai.kkhost.pl vLLM endpoint requires no API key.
      headers: { 'content-type': 'application/json' },
      body: JSON.stringify(payload),
      signal: AbortSignal.timeout(timeoutMs),
    });
    if (!res.ok) {
      const text = await res.text().catch(() => '');
      throw new Error(`chat failed: HTTP ${res.status} ${text.slice(0, 200)}`);
    }
    // Reasoning models also return their chain-of-thought in a separate
    // `reasoning` field. We deliberately ignore it and use only the final
    // answer, which lives in `content`.
    const body = (await res.json()) as {
      choices?: { message?: { content?: string; reasoning?: string | null } }[];
    };
    const content = body.choices?.[0]?.message?.content;
    if (typeof content !== 'string') {
      throw new Error('chat failed: no choices[0].message.content in response');
    }
    return content;
  }

  async function listModels(): Promise<string[]> {
    const res = await transport(`${apiBase}/models`, {
      signal: AbortSignal.timeout(timeoutMs),
    });
    if (!res.ok) throw new Error(`listModels failed: HTTP ${res.status}`);
    const body = (await res.json()) as { data?: { id: string }[] };
    return (body.data ?? []).map((m) => m.id);
  }

  return { chat, listModels };
}