History/server/llm.ts

// Thin OpenAI chat-completions client over native fetch (mirrors mesh-bot/src/llm).
// The transport is injectable so unit tests and the deterministic e2e stub can
// replace the network call. Config is read from env, never hardcoded:
//   LLM_API_BASE         (default https://ai.kkhost.pl/v1 — a vLLM endpoint, no API key)
//   LLM_MODEL            (default qwen3.8-27b — a vLLM *reasoning* model)
//   LLM_TIMEOUT_MS       (default 180000 — the 27B reasoning model can be slow)
//   LLM_MAX_TOKENS       (default 4000 — max *output* tokens per request)
//   LLM_REASONING_EFFORT (default "low" — sent as top-level `reasoning_effort`;
//                         empty or "off" omits the field entirely)
//
// The live model is a REASONING model: it emits its chain-of-thought in the
// response's `reasoning` field and the final answer in `content`. Sending
// top-level `reasoning_effort: "low"` cuts the CoT by ~85% (~4000 →
// ~400–600 tokens), so it fits comfortably inside the `max_tokens` budget and
// `content` comes back populated. (Other knobs misbehave on this build:
// "minimal" produced MORE CoT than "low", `chat_template_kwargs:
// {enable_thinking:false}` intermittently emits invalid JSON, and `/no_think`
// is silently ignored.) We read only `choices[0].message.content`, ignoring
// `reasoning`. No auth header is added (the endpoint requires no key).
//
// Reliability: `chat()` retries transient failures — timeout/abort, HTTP 5xx,
// network errors, and null/empty `content` (truncation safety net) — with up
// to 3 total attempts and 2s→3s backoff (±20% jitter). HTTP 4xx is a
// deterministic client error and throws immediately, with no retry. Every
// call is logged: one line on success, one on final failure.

export interface ChatMessage {
  role: 'system' | 'user' | 'assistant';
  content: string;
}

/** A network call. The default implementation is native fetch. */
export type Transport = (url: string, init?: RequestInit) => Promise<Response>;

export interface ChatOptions {
  maxTokens?: number;
  temperature?: number;
}

export interface LlmClient {
  chat(messages: ChatMessage[], opts?: ChatOptions): Promise<string>;
  listModels(): Promise<string[]>;
}

export const DEFAULT_BASE = 'https://ai.kkhost.pl/v1';
export const DEFAULT_MODEL = 'qwen3.8-27b';

/**
 * Default max *output* tokens per chat request. The live qwen3.8-27b is a
 * reasoning model: it spends `reasoning_tokens` on its chain-of-thought before
 * writing the final answer into `content`. With `reasoning_effort: "low"` a
 * realistic turn uses ~800–1100 output tokens, so 4000 is ~4× headroom while
 * staying under the upstream ~90s proxy wall. `max_tokens` is always set in the
 * request body (the vLLM/OpenAI wire key — NOT camelCase `maxTokens`, which the
 * server ignores), even when a caller omits it.
 */
export const DEFAULT_MAX_TOKENS = 4000;

/**
 * Default for `LLM_REASONING_EFFORT`. Sending top-level `reasoning_effort:
 * "low"` to the vLLM model cuts its chain-of-thought by ~85%, which is what
 * keeps `content` from being truncated to null.
 */
export const DEFAULT_REASONING_EFFORT = 'low';

/** Total attempts `chat()` makes before giving up (1 initial + 2 retries). */
export const MAX_ATTEMPTS = 3;

/** Base retry backoff: retry 1 waits 2000ms, retry 2 waits 3000ms, each ±20% jitter. */
const RETRY_BASE_DELAY_MS = 2000;

const fetchTransport: Transport = (url, init) => fetch(url, init);

export interface LlmClientDeps {
  transport?: Transport;
  apiBase?: string;
  model?: string;
  timeoutMs?: number;
  maxTokens?: number;
  reasoningEffort?: string;
  /**
   * Base delay (ms) for the retry backoff, overriding RETRY_BASE_DELAY_MS.
   * Mostly useful in tests to keep the suite fast.
   */
  retryBaseDelayMs?: number;
}

/** parseInt with a fallback for unset/empty/malformed values. */
function envInt(name: string, fallback: number): number {
  const raw = process.env[name]?.trim();
  if (!raw) return fallback;
  const n = parseInt(raw, 10);
  return Number.isFinite(n) && n > 0 ? n : fallback;
}

/** Trimmed env value, or undefined when unset/empty. */
function envStr(name: string): string | undefined {
  const raw = process.env[name]?.trim();
  return raw ? raw : undefined;
}

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

/** Backoff before retry `n` (1-based): `2000 * 1.5^(n-1)`, ±20% jitter. */
function backoffDelay(retry: number, baseMs: number): number {
  const base = baseMs * 1.5 ** (retry - 1);
  return base * (0.8 + Math.random() * 0.4);
}

/** Network-level error codes that mark a transient failure. */
const RETRYABLE_NET_CODES = new Set([
  'ECONNRESET',
  'ECONNREFUSED',
  'ECONNABORTED',
  'EPIPE',
  'EHOSTUNREACH',
  'ENETUNREACH',
  'ETIMEDOUT',
  'EAI_AGAIN',
  'UND_ERR_CONNECT_TIMEOUT',
  'UND_ERR_SOCKET',
  'UND_ERR_REQ_SOCKET',
  'UND_ERR_REQ_RETRY',
]);

/** True for a timeout/abort (an AbortSignal that fired, a TimeoutError, or AbortError). */
function isTimeoutOrAbort(err: unknown): boolean {
  let e: unknown = err;
  for (let i = 0; i < 3 && e; i++) {
    if (e instanceof Error && (e.name === 'TimeoutError' || e.name === 'AbortError')) return true;
    e = (e as { cause?: unknown })?.cause;
  }
  return false;
}

/** True for a transient network failure (ECONNRESET / ECONNREFUSED / socket closed / …). */
function isNetworkError(err: unknown): boolean {
  let e: unknown = err;
  let msg = '';
  for (let i = 0; i < 3 && e; i++) {
    if (e instanceof Error) {
      msg += ` ${e.message}`;
      const code = (e as Error & { code?: string }).code;
      if (code && RETRYABLE_NET_CODES.has(code)) return true;
    } else {
      msg += ` ${String(e)}`;
    }
    e = (e as { cause?: unknown })?.cause;
  }
  return /socket closed|socket hang up|connection reset|connection refused|other side closed|fetch failed/i.test(
    msg,
  );
}

/** One transport attempt, classified as success / retryable / fatal. */
interface Attempt {
  status?: number;
  error?: Error;
  retryable: boolean;
  content?: string;
  finishReason?: string | null;
}

export function createLlmClient(deps: LlmClientDeps = {}): LlmClient {
  const transport = deps.transport ?? fetchTransport;
  const apiBase = (deps.apiBase ?? process.env.LLM_API_BASE ?? DEFAULT_BASE).replace(/\/+$/, '');
  const model = deps.model ?? process.env.LLM_MODEL ?? DEFAULT_MODEL;
  const timeoutMs = parseInt(
    deps.timeoutMs ? String(deps.timeoutMs) : (process.env.LLM_TIMEOUT_MS ?? '180000'),
    10,
  );
  const maxTokens = deps.maxTokens ?? envInt('LLM_MAX_TOKENS', DEFAULT_MAX_TOKENS);
  const effort = deps.reasoningEffort ?? envStr('LLM_REASONING_EFFORT') ?? DEFAULT_REASONING_EFFORT;
  // "off" (case-insensitive) or empty disables the thinking knob entirely.
  const sendEffort = effort !== '' && effort.toLowerCase() !== 'off';
  const retryBaseMs = deps.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS;

  /**
   * One POST to /chat/completions. Classifies the outcome:
   *   - success   → content + finish_reason
   *   - retryable → timeout/abort, HTTP 5xx, network error, null/empty content
   *   - fatal     → HTTP 4xx (deterministic client error) or an unexpected throw
   */
  async function attemptOnce(payload: string, signal: AbortSignal): Promise<Attempt> {
    let res: Response;
    try {
      res = await transport(`${apiBase}/chat/completions`, {
        method: 'POST',
        // No auth header: the ai.kkhost.pl vLLM endpoint requires no API key.
        headers: { 'content-type': 'application/json' },
        body: payload,
        signal,
      });
    } catch (err) {
      const e = err instanceof Error ? err : new Error(String(err));
      // The transport threw: our AbortSignal fired (timeout/abort) or a
      // network-level failure (ECONNRESET / ECONNREFUSED / socket closed).
      return { error: e, retryable: signal.aborted || isTimeoutOrAbort(e) || isNetworkError(e) };
    }

    if (!res.ok) {
      const text = await res.text().catch(() => '');
      return {
        status: res.status,
        error: new Error(`chat failed: HTTP ${res.status} ${text.slice(0, 200)}`),
        // 4xx is a deterministic client error → do not retry; 5xx is retryable.
        retryable: res.status >= 500 && res.status < 600,
      };
    }

    const body = (await res.json()) as {
      choices?: {
        finish_reason?: string | null;
        message?: { content?: string; reasoning?: string | null };
      }[];
    };
    // The reasoning model can exhaust its output budget on the CoT and return
    // a null `content` — treat that as a retryable truncation, not a success.
    const content = body.choices?.[0]?.message?.content;
    if (typeof content !== 'string' || content === '') {
      return {
        status: res.status,
        error: new Error('chat failed: no choices[0].message.content in response'),
        retryable: true,
      };
    }
    return {
      status: res.status,
      retryable: false,
      content,
      finishReason: body.choices?.[0]?.finish_reason ?? null,
    };
  }

  async function chat(messages: ChatMessage[], opts: ChatOptions = {}): Promise<string> {
    const payload: Record<string, unknown> = {
      model,
      messages,
      // The wire key is `max_tokens` (snake_case) — camelCase is ignored by the
      // server. With `reasoning_effort: "low"` the CoT fits inside this budget,
      // so `content` comes back populated.
      max_tokens: opts.maxTokens ?? maxTokens,
    };
    if (opts.temperature != null) payload.temperature = opts.temperature;
    if (sendEffort) payload.reasoning_effort = effort;

    const totalStart = Date.now();
    let lastStatus: number | undefined;
    let lastError: Error = new Error('chat failed: no attempt completed');

    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
      const out = await attemptOnce(JSON.stringify(payload), AbortSignal.timeout(timeoutMs));
      if (out.status !== undefined) lastStatus = out.status;
      if (out.error) lastError = out.error;

      if (out.content !== undefined) {
        console.log('[llm] chat ok', {
          ms: Date.now() - totalStart,
          finishReason: out.finishReason ?? null,
        });
        return out.content;
      }
      // Give up on a non-retryable failure, or when the last attempt is spent.
      if (!out.retryable || attempt === MAX_ATTEMPTS) {
        console.error('[llm] chat failed', {
          attempts: attempt,
          lastStatus,
          message: lastError.message,
          totalMs: Date.now() - totalStart,
        });
        throw lastError;
      }
      await sleep(backoffDelay(attempt, retryBaseMs));
    }
    // Unreachable: every loop path returns or throws.
    throw lastError;
  }

  async function listModels(): Promise<string[]> {
    const res = await transport(`${apiBase}/models`, {
      signal: AbortSignal.timeout(timeoutMs),
    });
    if (!res.ok) throw new Error(`listModels failed: HTTP ${res.status}`);
    const body = (await res.json()) as { data?: { id: string }[] };
    return (body.data ?? []).map((m) => m.id);
  }

  return { chat, listModels };
}