reliability: low reasoning_effort, retry-with-backoff, LLM call logging

- send top-level reasoning_effort: "low" (LLM_REASONING_EFFORT) to cut the qwen3.8-27b CoT ~85%, so content is no longer truncated to null -> the ~50% 502s - new env: LLM_MAX_TOKENS (default 4000, per-call opts.maxTokens still overrides), LLM_REASONING_EFFORT (default "low", off/empty omits the field) - chat() retries up to 3 attempts: timeout/abort, 5xx, network errors (ECONNRESET/ECONNREFUSED/socket closed), null/empty content; backoff 2s->3s (2000*1.5^(n-1), +/-20% jitter); 4xx throws immediately - logging: '[llm] chat ok' {ms, finishReason} on success; '[llm] chat failed' {attempts, lastStatus, message, totalMs} on final failure - tests: 12 in llm.test.ts (retry success/failure, no 4xx retry, null-content retry, logging spies); AGENTS.md env table + gotchas updated
2026-10-11 13:46:18 +00:00
parent 612de2fc40
commit 3297658c81
3 changed files with 395 additions and 63 deletions
@@ -1,18 +1,28 @@
 // Thin OpenAI chat-completions client over native fetch (mirrors mesh-bot/src/llm).
 // The transport is injectable so unit tests and the deterministic e2e stub can
 // replace the network call. Config is read from env, never hardcoded:
-//   LLM_API_BASE   (default https://ai.kkhost.pl/v1 — a vLLM endpoint, no API key)
-//   LLM_MODEL      (default qwen3.8-27b — a vLLM *reasoning* model)
-//   LLM_TIMEOUT_MS (default 180000 — the 27B reasoning model can be slow)
+//   LLM_API_BASE         (default https://ai.kkhost.pl/v1 — a vLLM endpoint, no API key)
+//   LLM_MODEL            (default qwen3.8-27b — a vLLM *reasoning* model)
+//   LLM_TIMEOUT_MS       (default 180000 — the 27B reasoning model can be slow)
+//   LLM_MAX_TOKENS       (default 4000 — max *output* tokens per request)
+//   LLM_REASONING_EFFORT (default "low" — sent as top-level `reasoning_effort`;
+//                         empty or "off" omits the field entirely)
 //
 // The live model is a REASONING model: it emits its chain-of-thought in the
-// response's `reasoning` field and the final answer in `content`. It therefore
-// needs a generous `max_tokens` budget to finish thinking before writing
-// `content` — a too-small budget makes it run out mid-reasoning and return a
-// null `content`. We always send an explicit, generous `max_tokens` (the
-// OpenAI/vLLM wire key — NOT camelCase `maxTokens`, which the server ignores)
-// and read only `choices[0].message.content`, ignoring `reasoning`. No auth
-// header is added (the endpoint requires no key).
+// response's `reasoning` field and the final answer in `content`. Sending
+// top-level `reasoning_effort: "low"` cuts the CoT by ~85% (~4000 →
+// ~400–600 tokens), so it fits comfortably inside the `max_tokens` budget and
+// `content` comes back populated. (Other knobs misbehave on this build:
+// "minimal" produced MORE CoT than "low", `chat_template_kwargs:
+// {enable_thinking:false}` intermittently emits invalid JSON, and `/no_think`
+// is silently ignored.) We read only `choices[0].message.content`, ignoring
+// `reasoning`. No auth header is added (the endpoint requires no key).
+//
+// Reliability: `chat()` retries transient failures — timeout/abort, HTTP 5xx,
+// network errors, and null/empty `content` (truncation safety net) — with up
+// to 3 total attempts and 2s→3s backoff (±20% jitter). HTTP 4xx is a
+// deterministic client error and throws immediately, with no retry. Every
+// call is logged: one line on success, one on final failure.

 export interface ChatMessage {
  role: 'system' | 'user' | 'assistant';
@@ -36,14 +46,29 @@ export const DEFAULT_BASE = 'https://ai.kkhost.pl/v1';
 export const DEFAULT_MODEL = 'qwen3.8-27b';

 /**
- * Generous token budget for every chat request. The live qwen3.8-27b is a
+ * Default max *output* tokens per chat request. The live qwen3.8-27b is a
 * reasoning model: it spends `reasoning_tokens` on its chain-of-thought before
- * writing the final answer into `content`. A small budget makes it run out
- * mid-reasoning and return a null `content`, so we give it room to finish both.
- * `max_tokens` is always set in the request body, even when a caller omits it.
+ * writing the final answer into `content`. With `reasoning_effort: "low"` a
+ * realistic turn uses ~800–1100 output tokens, so 4000 is ~4× headroom while
+ * staying under the upstream ~90s proxy wall. `max_tokens` is always set in the
+ * request body (the vLLM/OpenAI wire key — NOT camelCase `maxTokens`, which the
+ * server ignores), even when a caller omits it.
 */
 export const DEFAULT_MAX_TOKENS = 4000;

+/**
+ * Default for `LLM_REASONING_EFFORT`. Sending top-level `reasoning_effort:
+ * "low"` to the vLLM model cuts its chain-of-thought by ~85%, which is what
+ * keeps `content` from being truncated to null.
+ */
+export const DEFAULT_REASONING_EFFORT = 'low';
+
+/** Total attempts `chat()` makes before giving up (1 initial + 2 retries). */
+export const MAX_ATTEMPTS = 3;
+
+/** Base retry backoff: retry 1 waits 2000ms, retry 2 waits 3000ms, each ±20% jitter. */
+const RETRY_BASE_DELAY_MS = 2000;
+
 const fetchTransport: Transport = (url, init) => fetch(url, init);

 export interface LlmClientDeps {
@@ -51,6 +76,91 @@ export interface LlmClientDeps {
  apiBase?: string;
  model?: string;
  timeoutMs?: number;
+  maxTokens?: number;
+  reasoningEffort?: string;
+  /**
+   * Base delay (ms) for the retry backoff, overriding RETRY_BASE_DELAY_MS.
+   * Mostly useful in tests to keep the suite fast.
+   */
+  retryBaseDelayMs?: number;
+}
+
+/** parseInt with a fallback for unset/empty/malformed values. */
+function envInt(name: string, fallback: number): number {
+  const raw = process.env[name]?.trim();
+  if (!raw) return fallback;
+  const n = parseInt(raw, 10);
+  return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+
+/** Trimmed env value, or undefined when unset/empty. */
+function envStr(name: string): string | undefined {
+  const raw = process.env[name]?.trim();
+  return raw ? raw : undefined;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** Backoff before retry `n` (1-based): `2000 * 1.5^(n-1)`, ±20% jitter. */
+function backoffDelay(retry: number, baseMs: number): number {
+  const base = baseMs * 1.5 ** (retry - 1);
+  return base * (0.8 + Math.random() * 0.4);
+}
+
+/** Network-level error codes that mark a transient failure. */
+const RETRYABLE_NET_CODES = new Set([
+  'ECONNRESET',
+  'ECONNREFUSED',
+  'ECONNABORTED',
+  'EPIPE',
+  'EHOSTUNREACH',
+  'ENETUNREACH',
+  'ETIMEDOUT',
+  'EAI_AGAIN',
+  'UND_ERR_CONNECT_TIMEOUT',
+  'UND_ERR_SOCKET',
+  'UND_ERR_REQ_SOCKET',
+  'UND_ERR_REQ_RETRY',
+]);
+
+/** True for a timeout/abort (an AbortSignal that fired, a TimeoutError, or AbortError). */
+function isTimeoutOrAbort(err: unknown): boolean {
+  let e: unknown = err;
+  for (let i = 0; i < 3 && e; i++) {
+    if (e instanceof Error && (e.name === 'TimeoutError' || e.name === 'AbortError')) return true;
+    e = (e as { cause?: unknown })?.cause;
+  }
+  return false;
+}
+
+/** True for a transient network failure (ECONNRESET / ECONNREFUSED / socket closed / …). */
+function isNetworkError(err: unknown): boolean {
+  let e: unknown = err;
+  let msg = '';
+  for (let i = 0; i < 3 && e; i++) {
+    if (e instanceof Error) {
+      msg += ` ${e.message}`;
+      const code = (e as Error & { code?: string }).code;
+      if (code && RETRYABLE_NET_CODES.has(code)) return true;
+    } else {
+      msg += ` ${String(e)}`;
+    }
+    e = (e as { cause?: unknown })?.cause;
+  }
+  return /socket closed|socket hang up|connection reset|connection refused|other side closed|fetch failed/i.test(
+    msg,
+  );
+}
+
+/** One transport attempt, classified as success / retryable / fatal. */
+interface Attempt {
+  status?: number;
+  error?: Error;
+  retryable: boolean;
+  content?: string;
+  finishReason?: string | null;
 }

 export function createLlmClient(deps: LlmClientDeps = {}): LlmClient {
@@ -61,36 +171,111 @@ export function createLlmClient(deps: LlmClientDeps = {}): LlmClient {
    deps.timeoutMs ? String(deps.timeoutMs) : (process.env.LLM_TIMEOUT_MS ?? '180000'),
    10,
  );
+  const maxTokens = deps.maxTokens ?? envInt('LLM_MAX_TOKENS', DEFAULT_MAX_TOKENS);
+  const effort = deps.reasoningEffort ?? envStr('LLM_REASONING_EFFORT') ?? DEFAULT_REASONING_EFFORT;
+  // "off" (case-insensitive) or empty disables the thinking knob entirely.
+  const sendEffort = effort !== '' && effort.toLowerCase() !== 'off';
+  const retryBaseMs = deps.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS;
+
+  /**
+   * One POST to /chat/completions. Classifies the outcome:
+   *   - success   → content + finish_reason
+   *   - retryable → timeout/abort, HTTP 5xx, network error, null/empty content
+   *   - fatal     → HTTP 4xx (deterministic client error) or an unexpected throw
+   */
+  async function attemptOnce(payload: string, signal: AbortSignal): Promise<Attempt> {
+    let res: Response;
+    try {
+      res = await transport(`${apiBase}/chat/completions`, {
+        method: 'POST',
+        // No auth header: the ai.kkhost.pl vLLM endpoint requires no API key.
+        headers: { 'content-type': 'application/json' },
+        body: payload,
+        signal,
+      });
+    } catch (err) {
+      const e = err instanceof Error ? err : new Error(String(err));
+      // The transport threw: our AbortSignal fired (timeout/abort) or a
+      // network-level failure (ECONNRESET / ECONNREFUSED / socket closed).
+      return { error: e, retryable: signal.aborted || isTimeoutOrAbort(e) || isNetworkError(e) };
+    }

-  async function chat(messages: ChatMessage[], opts: ChatOptions = {}): Promise<string> {
-    const payload: Record<string, unknown> = { model, messages };
-    if (opts.temperature != null) payload.temperature = opts.temperature;
-    // Always send an explicit, generous `max_tokens` (the vLLM/OpenAI wire key —
-    // camelCase `maxTokens` is ignored by the server) so the reasoning model has
-    // room to finish `reasoning` before emitting `content`.
-    payload.max_tokens = opts.maxTokens ?? DEFAULT_MAX_TOKENS;
-    const res = await transport(`${apiBase}/chat/completions`, {
-      method: 'POST',
-      // No auth header: the ai.kkhost.pl vLLM endpoint requires no API key.
-      headers: { 'content-type': 'application/json' },
-      body: JSON.stringify(payload),
-      signal: AbortSignal.timeout(timeoutMs),
-    });
    if (!res.ok) {
      const text = await res.text().catch(() => '');
-      throw new Error(`chat failed: HTTP ${res.status} ${text.slice(0, 200)}`);
+      return {
+        status: res.status,
+        error: new Error(`chat failed: HTTP ${res.status} ${text.slice(0, 200)}`),
+        // 4xx is a deterministic client error → do not retry; 5xx is retryable.
+        retryable: res.status >= 500 && res.status < 600,
+      };
    }
-    // Reasoning models also return their chain-of-thought in a separate
-    // `reasoning` field. We deliberately ignore it and use only the final
-    // answer, which lives in `content`.
+
    const body = (await res.json()) as {
-      choices?: { message?: { content?: string; reasoning?: string | null } }[];
+      choices?: {
+        finish_reason?: string | null;
+        message?: { content?: string; reasoning?: string | null };
+      }[];
    };
+    // The reasoning model can exhaust its output budget on the CoT and return
+    // a null `content` — treat that as a retryable truncation, not a success.
    const content = body.choices?.[0]?.message?.content;
-    if (typeof content !== 'string') {
-      throw new Error('chat failed: no choices[0].message.content in response');
+    if (typeof content !== 'string' || content === '') {
+      return {
+        status: res.status,
+        error: new Error('chat failed: no choices[0].message.content in response'),
+        retryable: true,
+      };
    }
-    return content;
+    return {
+      status: res.status,
+      retryable: false,
+      content,
+      finishReason: body.choices?.[0]?.finish_reason ?? null,
+    };
+  }
+
+  async function chat(messages: ChatMessage[], opts: ChatOptions = {}): Promise<string> {
+    const payload: Record<string, unknown> = {
+      model,
+      messages,
+      // The wire key is `max_tokens` (snake_case) — camelCase is ignored by the
+      // server. With `reasoning_effort: "low"` the CoT fits inside this budget,
+      // so `content` comes back populated.
+      max_tokens: opts.maxTokens ?? maxTokens,
+    };
+    if (opts.temperature != null) payload.temperature = opts.temperature;
+    if (sendEffort) payload.reasoning_effort = effort;
+
+    const totalStart = Date.now();
+    let lastStatus: number | undefined;
+    let lastError: Error = new Error('chat failed: no attempt completed');
+
+    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+      const out = await attemptOnce(JSON.stringify(payload), AbortSignal.timeout(timeoutMs));
+      if (out.status !== undefined) lastStatus = out.status;
+      if (out.error) lastError = out.error;
+
+      if (out.content !== undefined) {
+        console.log('[llm] chat ok', {
+          ms: Date.now() - totalStart,
+          finishReason: out.finishReason ?? null,
+        });
+        return out.content;
+      }
+      // Give up on a non-retryable failure, or when the last attempt is spent.
+      if (!out.retryable || attempt === MAX_ATTEMPTS) {
+        console.error('[llm] chat failed', {
+          attempts: attempt,
+          lastStatus,
+          message: lastError.message,
+          totalMs: Date.now() - totalStart,
+        });
+        throw lastError;
+      }
+      await sleep(backoffDelay(attempt, retryBaseMs));
+    }
+    // Unreachable: every loop path returns or throws.
+    throw lastError;
  }

  async function listModels(): Promise<string[]> {