History/server/llm.ts

// Thin OpenAI chat-completions client over native fetch (mirrors mesh-bot/src/llm).
// The transport is injectable so unit tests and the deterministic e2e stub can
// replace the network call. Config is read from env, never hardcoded:
//   LLM_API_BASE         (default https://ai.kkhost.pl/v1 — a vLLM endpoint, no API key)
//   LLM_MODEL            (default qwen3.8-27b — a vLLM *reasoning* model)
//   LLM_TIMEOUT_MS       (default 180000 — the 27B reasoning model can be slow)
//   LLM_MAX_TOKENS       (default 4000 — max *output* tokens per request)
//   LLM_REASONING_EFFORT (default "low" — sent as top-level `reasoning_effort`;
//                         empty or "off" omits the field entirely)
//
// The live model is a REASONING model: it emits its chain-of-thought in the
// response's `reasoning` field and the final answer in `content`. Sending
// top-level `reasoning_effort: "low"` cuts the CoT by ~85% (~4000 →
// ~400–600 tokens), so it fits comfortably inside the `max_tokens` budget and
// `content` comes back populated. (Other knobs misbehave on this build:
// "minimal" produced MORE CoT than "low", `chat_template_kwargs:
// {enable_thinking:false}` intermittently emits invalid JSON, and `/no_think`
// is silently ignored.) We read only `choices[0].message.content`, ignoring
// `reasoning`. No auth header is added (the endpoint requires no key).
//
// Reliability: `chat()` retries transient failures — timeout/abort, HTTP 5xx,
// network errors, and null/empty `content` (truncation safety net) — with up
// to 3 total attempts and 2s→3s backoff (±20% jitter). HTTP 4xx is a
// deterministic client error and throws immediately, with no retry. Every
// call is logged: one line on success, one on final failure.
//
// Function calling: `chatWithTools()` extends `chat()` with the OpenAI tool
// loop. When the model answers with `finish_reason: "tool_calls"`, the client
// runs the provided executor for every requested call, appends the assistant
// tool-call message plus one `tool` result message per call, and re-sends the
// growing history until the model produces a final `content`. The loop is
// bounded to DEFAULT_MAX_TOOL_ROUNDS (6) tool rounds; if the budget is spent
// without a final answer, one last *no-tools* request forces the model to
// commit. The same retry/backoff/reasoning-effort machinery applies per
// request, so an injectable transport only has to forward `tools` in the body
// and accept `assistant`/`tool` messages in the history. The deterministic
// stub answers plain `content` (no tool calls) and therefore bypasses the
// loop in a single round.

export interface ChatMessage {
  role: 'system' | 'user' | 'assistant';
  content: string;
}

/** One OpenAI-style function call the model requests (`finish_reason: "tool_calls"`). */
export interface ToolCallSpec {
  id: string;
  type: 'function';
  function: { name: string; arguments: string };
}

/** A function the model may call (one OpenAI `tools` entry). */
export interface ToolDefinition {
  type: 'function';
  function: {
    name: string;
    description?: string;
    parameters?: Record<string, unknown>;
  };
}

/**
 * The full message vocabulary a request can carry: plain chat messages, plus
 * assistant messages that request tool calls and the `tool` results that
 * answer them (the OpenAI function-calling wire shape).
 */
export type WireMessage =
  | ChatMessage
  | { role: 'assistant'; content: string | null; tool_calls: ToolCallSpec[] }
  | { role: 'tool'; tool_call_id: string; content: string };

/** Executes one tool call; the returned string is fed back to the model as the `tool` result. */
export type ToolExecutor = (name: string, args: Record<string, unknown>) => string;

/** A network call. The default implementation is native fetch. */
export type Transport = (url: string, init?: RequestInit) => Promise<Response>;

export interface ChatOptions {
  maxTokens?: number;
  temperature?: number;
}

export interface LlmClient {
  chat(messages: ChatMessage[], opts?: ChatOptions): Promise<string>;
  /**
   * The function-calling variant of chat() — see the file header for the loop
   * semantics (bounded tool rounds, then a forced final answer). Optional on
   * the interface: minimal test doubles may omit it, and callers (the
   * engine's runTurn) fall back to chat() when it is absent.
   */
  chatWithTools?(
    messages: WireMessage[],
    tools: ToolDefinition[],
    executeTool: ToolExecutor,
    opts?: ChatOptions & { maxRounds?: number },
  ): Promise<string>;
  listModels(): Promise<string[]>;
}

export const DEFAULT_BASE = 'https://ai.kkhost.pl/v1';
export const DEFAULT_MODEL = 'qwen3.8-27b';

/**
 * Default max *output* tokens per chat request. The live qwen3.8-27b is a
 * reasoning model: it spends `reasoning_tokens` on its chain-of-thought before
 * writing the final answer into `content`. With `reasoning_effort: "low"` a
 * realistic turn uses ~800–1100 output tokens, so 4000 is ~4× headroom while
 * staying under the upstream ~90s proxy wall. `max_tokens` is always set in the
 * request body (the vLLM/OpenAI wire key — NOT camelCase `maxTokens`, which the
 * server ignores), even when a caller omits it.
 */
export const DEFAULT_MAX_TOKENS = 4000;

/**
 * Default for `LLM_REASONING_EFFORT`. Sending top-level `reasoning_effort:
 * "low"` to the vLLM model cuts its chain-of-thought by ~85%, which is what
 * keeps `content` from being truncated to null.
 */
export const DEFAULT_REASONING_EFFORT = 'low';

/** Total attempts per request (chat / chatWithTools) before giving up (1 initial + 2 retries). */
export const MAX_ATTEMPTS = 3;

/**
 * Max tool-call rounds (assistant tool_calls → tool results) a chatWithTools
 * loop may make before a final no-tools request forces the answer. Keeps a
 * turn bounded even if the model keeps asking for data; a realistic turn uses
 * 0–2 rounds.
 */
export const DEFAULT_MAX_TOOL_ROUNDS = 6;

/** Base retry backoff: retry 1 waits 2000ms, retry 2 waits 3000ms, each ±20% jitter. */
const RETRY_BASE_DELAY_MS = 2000;

const fetchTransport: Transport = (url, init) => fetch(url, init);

export interface LlmClientDeps {
  transport?: Transport;
  apiBase?: string;
  model?: string;
  timeoutMs?: number;
  maxTokens?: number;
  reasoningEffort?: string;
  /**
   * Base delay (ms) for the retry backoff, overriding RETRY_BASE_DELAY_MS.
   * Mostly useful in tests to keep the suite fast.
   */
  retryBaseDelayMs?: number;
}

/** parseInt with a fallback for unset/empty/malformed values. */
function envInt(name: string, fallback: number): number {
  const raw = process.env[name]?.trim();
  if (!raw) return fallback;
  const n = parseInt(raw, 10);
  return Number.isFinite(n) && n > 0 ? n : fallback;
}

/** Trimmed env value, or undefined when unset/empty. */
function envStr(name: string): string | undefined {
  const raw = process.env[name]?.trim();
  return raw ? raw : undefined;
}

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

/** Backoff before retry `n` (1-based): `2000 * 1.5^(n-1)`, ±20% jitter. */
function backoffDelay(retry: number, baseMs: number): number {
  const base = baseMs * 1.5 ** (retry - 1);
  return base * (0.8 + Math.random() * 0.4);
}

/** Network-level error codes that mark a transient failure. */
const RETRYABLE_NET_CODES = new Set([
  'ECONNRESET',
  'ECONNREFUSED',
  'ECONNABORTED',
  'EPIPE',
  'EHOSTUNREACH',
  'ENETUNREACH',
  'ETIMEDOUT',
  'EAI_AGAIN',
  'UND_ERR_CONNECT_TIMEOUT',
  'UND_ERR_SOCKET',
  'UND_ERR_REQ_SOCKET',
  'UND_ERR_REQ_RETRY',
]);

/** True for a timeout/abort (an AbortSignal that fired, a TimeoutError, or AbortError). */
function isTimeoutOrAbort(err: unknown): boolean {
  let e: unknown = err;
  for (let i = 0; i < 3 && e; i++) {
    if (e instanceof Error && (e.name === 'TimeoutError' || e.name === 'AbortError')) return true;
    e = (e as { cause?: unknown })?.cause;
  }
  return false;
}

/** True for a transient network failure (ECONNRESET / ECONNREFUSED / socket closed / …). */
function isNetworkError(err: unknown): boolean {
  let e: unknown = err;
  let msg = '';
  for (let i = 0; i < 3 && e; i++) {
    if (e instanceof Error) {
      msg += ` ${e.message}`;
      const code = (e as Error & { code?: string }).code;
      if (code && RETRYABLE_NET_CODES.has(code)) return true;
    } else {
      msg += ` ${String(e)}`;
    }
    e = (e as { cause?: unknown })?.cause;
  }
  return /socket closed|socket hang up|connection reset|connection refused|other side closed|fetch failed/i.test(
    msg,
  );
}

/** Parse the wire `tool_calls` array into validated ToolCallSpecs (malformed entries are dropped). */
function parseToolCalls(raw: unknown): ToolCallSpec[] {
  if (!Array.isArray(raw)) return [];
  const out: ToolCallSpec[] = [];
  for (const tc of raw) {
    if (!tc || typeof tc !== 'object') continue;
    const t = tc as Record<string, unknown>;
    const fn = t.function as Record<string, unknown> | undefined;
    if (!fn || typeof fn.name !== 'string' || fn.name === '') continue;
    const argumentsText =
      typeof fn.arguments === 'string'
        ? fn.arguments
        : fn.arguments != null
          ? JSON.stringify(fn.arguments)
          : '{}';
    out.push({
      id: typeof t.id === 'string' && t.id !== '' ? t.id : `call_${out.length}`,
      type: 'function',
      function: { name: fn.name, arguments: argumentsText },
    });
  }
  return out;
}

/** One transport attempt, classified as success / retryable / fatal. */
interface Attempt {
  status?: number;
  error?: Error;
  retryable: boolean;
  content?: string;
  finishReason?: string | null;
  toolCalls?: ToolCallSpec[];
}

export function createLlmClient(deps: LlmClientDeps = {}): LlmClient {
  const transport = deps.transport ?? fetchTransport;
  const apiBase = (deps.apiBase ?? process.env.LLM_API_BASE ?? DEFAULT_BASE).replace(/\/+$/, '');
  const model = deps.model ?? process.env.LLM_MODEL ?? DEFAULT_MODEL;
  const timeoutMs = parseInt(
    deps.timeoutMs ? String(deps.timeoutMs) : (process.env.LLM_TIMEOUT_MS ?? '180000'),
    10,
  );
  const maxTokens = deps.maxTokens ?? envInt('LLM_MAX_TOKENS', DEFAULT_MAX_TOKENS);
  const effort = deps.reasoningEffort ?? envStr('LLM_REASONING_EFFORT') ?? DEFAULT_REASONING_EFFORT;
  // "off" (case-insensitive) or empty disables the thinking knob entirely.
  const sendEffort = effort !== '' && effort.toLowerCase() !== 'off';
  const retryBaseMs = deps.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS;

  /**
   * One POST to /chat/completions. Classifies the outcome:
   *   - success   → content and/or tool_calls
   *   - retryable → timeout/abort, HTTP 5xx, network error, null/empty content
   *   - fatal     → HTTP 4xx (deterministic client error) or an unexpected throw
   */
  async function attemptOnce(payload: string, signal: AbortSignal): Promise<Attempt> {
    let res: Response;
    try {
      res = await transport(`${apiBase}/chat/completions`, {
        method: 'POST',
        // No auth header: the ai.kkhost.pl vLLM endpoint requires no API key.
        headers: { 'content-type': 'application/json' },
        body: payload,
        signal,
      });
    } catch (err) {
      const e = err instanceof Error ? err : new Error(String(err));
      // The transport threw: our AbortSignal fired (timeout/abort) or a
      // network-level failure (ECONNRESET / ECONNREFUSED / socket closed).
      return { error: e, retryable: signal.aborted || isTimeoutOrAbort(e) || isNetworkError(e) };
    }

    if (!res.ok) {
      const text = await res.text().catch(() => '');
      return {
        status: res.status,
        error: new Error(`chat failed: HTTP ${res.status} ${text.slice(0, 200)}`),
        // 4xx is a deterministic client error → do not retry; 5xx is retryable.
        retryable: res.status >= 500 && res.status < 600,
      };
    }

    const body = (await res.json()) as {
      choices?: {
        finish_reason?: string | null;
        message?: { content?: string; reasoning?: string | null; tool_calls?: unknown };
      }[];
    };
    const choice = body.choices?.[0];
    const message = choice?.message;
    // A tool-call round is a SUCCESS even though `content` is null there: the
    // model is asking for data, not failing (never treat it as truncation).
    const toolCalls = parseToolCalls(message?.tool_calls);
    if (toolCalls.length > 0) {
      return {
        status: res.status,
        retryable: false,
        content:
          typeof message?.content === 'string' && message.content !== '' ? message.content : undefined,
        finishReason: choice?.finish_reason ?? null,
        toolCalls,
      };
    }
    // The reasoning model can exhaust its output budget on the CoT and return
    // a null `content` — treat that as a retryable truncation, not a success.
    const content = message?.content;
    if (typeof content !== 'string' || content === '') {
      return {
        status: res.status,
        error: new Error('chat failed: no choices[0].message.content in response'),
        retryable: true,
      };
    }
    return {
      status: res.status,
      retryable: false,
      content,
      finishReason: choice?.finish_reason ?? null,
    };
  }

  /** Build one /chat/completions request body. `tools` is only included when non-empty, so plain chat() keeps its exact legacy shape. */
  function buildPayload(
    messages: WireMessage[],
    tools: ToolDefinition[] | undefined,
    opts: ChatOptions,
  ): Record<string, unknown> {
    const payload: Record<string, unknown> = {
      model,
      messages,
      // The wire key is `max_tokens` (snake_case) — camelCase is ignored by the
      // server. With `reasoning_effort: "low"` the CoT fits inside this budget,
      // so `content` comes back populated.
      max_tokens: opts.maxTokens ?? maxTokens,
    };
    if (opts.temperature != null) payload.temperature = opts.temperature;
    if (sendEffort) payload.reasoning_effort = effort;
    if (tools && tools.length > 0) payload.tools = tools;
    return payload;
  }

  /**
   * One request with the standard retry/backoff: up to MAX_ATTEMPTS per
   * request; resolves with a successful Attempt (content and/or toolCalls) or
   * throws the last error once the attempts are spent.
   */
  async function requestWithRetry(payload: Record<string, unknown>): Promise<Attempt> {
    const totalStart = Date.now();
    let lastStatus: number | undefined;
    let lastError: Error = new Error('chat failed: no attempt completed');

    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
      const out = await attemptOnce(JSON.stringify(payload), AbortSignal.timeout(timeoutMs));
      if (out.status !== undefined) lastStatus = out.status;
      if (out.error) lastError = out.error;

      if (out.content !== undefined || out.toolCalls !== undefined) {
        console.log('[llm] chat ok', {
          ms: Date.now() - totalStart,
          finishReason: out.finishReason ?? null,
          toolCalls: out.toolCalls?.length ?? 0,
        });
        return out;
      }
      // Give up on a non-retryable failure, or when the last attempt is spent.
      if (!out.retryable || attempt === MAX_ATTEMPTS) {
        console.error('[llm] chat failed', {
          attempts: attempt,
          lastStatus,
          message: lastError.message,
          totalMs: Date.now() - totalStart,
        });
        throw lastError;
      }
      await sleep(backoffDelay(attempt, retryBaseMs));
    }
    // Unreachable: every loop path returns or throws.
    throw lastError;
  }

  async function chat(messages: ChatMessage[], opts: ChatOptions = {}): Promise<string> {
    const out = await requestWithRetry(buildPayload(messages, undefined, opts));
    return out.content ?? '';
  }

  /**
   * The function-calling loop — see the file header. Bounded: at most
   * `maxRounds` tool rounds, then one final no-tools request forces an answer.
   * Throws like chat() only on transport-level exhaustion (the engine maps that
   * to a 502; bad *content* never throws, it is parsed defensively downstream).
   */
  async function chatWithTools(
    initial: WireMessage[],
    tools: ToolDefinition[],
    executeTool: ToolExecutor,
    opts: ChatOptions & { maxRounds?: number } = {},
  ): Promise<string> {
    const maxRounds = Math.max(1, opts.maxRounds ?? DEFAULT_MAX_TOOL_ROUNDS);
    const messages: WireMessage[] = [...initial];
    for (let round = 0; round < maxRounds; round++) {
      const out = await requestWithRetry(buildPayload(messages, tools, opts));
      // Tool calls win over any partial content: the model asked for data
      // before it finished.
      if (out.toolCalls && out.toolCalls.length > 0) {
        messages.push({
          role: 'assistant',
          content: out.content ?? null,
          tool_calls: out.toolCalls,
        });
        for (const tc of out.toolCalls) {
          let result: string;
          try {
            const parsed: unknown =
              tc.function.arguments !== '' ? JSON.parse(tc.function.arguments) : {};
            const args =
              parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)
                ? (parsed as Record<string, unknown>)
                : {};
            result = executeTool(tc.function.name, args);
          } catch (err) {
            // Malformed arguments are not fatal: hand the model an error
            // result so it can re-call with valid arguments or move on.
            result = JSON.stringify({
              error: `invalid tool arguments: ${err instanceof Error ? err.message : String(err)}`,
            });
          }
          messages.push({ role: 'tool', tool_call_id: tc.id, content: result });
        }
        continue;
      }
      if (out.content !== undefined) return out.content;
    }
    // The tool budget is spent without a final answer: one last no-tools
    // request forces the model to commit.
    messages.push({
      role: 'user',
      content: 'You have reached your tool budget. Answer now with the final result only.',
    });
    const out = await requestWithRetry(buildPayload(messages, undefined, opts));
    return out.content ?? '';
  }

  async function listModels(): Promise<string[]> {
    const res = await transport(`${apiBase}/models`, {
      signal: AbortSignal.timeout(timeoutMs),
    });
    if (!res.ok) throw new Error(`listModels failed: HTTP ${res.status}`);
    const body = (await res.json()) as { data?: { id: string }[] };
    return (body.data ?? []).map((m) => m.id);
  }

  return { chat, chatWithTools, listModels };
}