History/server/llm.test.ts

import { afterEach, describe, expect, it, vi } from 'vitest';
import { createLlmClient, DEFAULT_MAX_TOKENS } from './llm';

function json(body: unknown) {
  return {
    ok: true,
    status: 200,
    json: async () => body,
    text: async () => JSON.stringify(body),
  } as unknown as Response;
}

function httpError(status: number, text = 'upstream down') {
  return {
    ok: false,
    status,
    text: async () => text,
    json: async () => ({}),
  } as unknown as Response;
}

afterEach(() => {
  vi.restoreAllMocks();
});

describe('createLlmClient (injectable transport)', () => {
  it('chat posts model+messages to {base}/chat/completions and returns content', async () => {
    const sent: { url: string; init?: RequestInit }[] = [];
    const transport = async (url: string, init?: RequestInit) => {
      sent.push({ url, init });
      return json({ choices: [{ message: { content: 'hello back' } }] });
    };
    const llm = createLlmClient({ transport, apiBase: 'http://x/v1/', model: 'm1' });
    const out = await llm.chat([{ role: 'user', content: 'hi' }], { maxTokens: 10, temperature: 0.5 });
    expect(out).toBe('hello back');
    expect(sent[0].url).toBe('http://x/v1/chat/completions'); // trailing slash normalized
    const body = JSON.parse(sent[0].init!.body as string);
    // The wire key is `max_tokens` (snake_case), never camelCase `maxTokens`.
    expect(body).toEqual({
      model: 'm1',
      messages: [{ role: 'user', content: 'hi' }],
      temperature: 0.5,
      max_tokens: 10,
      reasoning_effort: 'low',
    });
    // No auth header: the live vLLM endpoint requires no API key.
    expect(sent[0].init!.headers).toEqual({ 'content-type': 'application/json' });
  });

  it('chat uses a generous default max_tokens when the caller omits it', async () => {
    const sent: { url: string; init?: RequestInit }[] = [];
    const transport = async (url: string, init?: RequestInit) => {
      sent.push({ url, init });
      return json({ choices: [{ message: { content: 'ok' } }] });
    };
    const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm1' });
    await llm.chat([{ role: 'user', content: 'hi' }]);
    const body = JSON.parse(sent[0].init!.body as string);
    expect(body.max_tokens).toBe(DEFAULT_MAX_TOKENS);
    expect(body.max_tokens).toBe(4000);
  });

  it('chat sends reasoning_effort "low" by default', async () => {
    const sent: { url: string; init?: RequestInit }[] = [];
    const transport = async (url: string, init?: RequestInit) => {
      sent.push({ url, init });
      return json({ choices: [{ message: { content: 'ok' } }] });
    };
    const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm1' });
    await llm.chat([{ role: 'user', content: 'hi' }]);
    const body = JSON.parse(sent[0].init!.body as string);
    expect(body.reasoning_effort).toBe('low');
  });

  it('chat omits reasoning_effort when it is "off" or empty', async () => {
    const sent: { url: string; init?: RequestInit }[] = [];
    const transport = async (url: string, init?: RequestInit) => {
      sent.push({ url, init });
      return json({ choices: [{ message: { content: 'ok' } }] });
    };
    for (const effort of ['off', 'OFF', '']) {
      sent.length = 0;
      const llm = createLlmClient({
        transport,
        apiBase: 'http://x/v1',
        model: 'm1',
        reasoningEffort: effort,
      });
      await llm.chat([{ role: 'user', content: 'hi' }]);
      const body = JSON.parse(sent[0].init!.body as string);
      expect(body.reasoning_effort, `effort=${JSON.stringify(effort)}`).toBeUndefined();
    }
  });

  it('listModels returns data[].id from GET {base}/models', async () => {
    const transport = async (url: string) => {
      expect(url).toBe('http://x/v1/models');
      return json({ data: [{ id: 'a' }, { id: 'b' }] });
    };
    const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm' });
    expect(await llm.listModels()).toEqual(['a', 'b']);
  });

  it('chat retries 5xx and succeeds once the transport recovers (500, 500 → 200)', async () => {
    const sent: { url: string; init?: RequestInit }[] = [];
    const transport = async (url: string, init?: RequestInit) => {
      sent.push({ url, init });
      if (sent.length < 3) return httpError(500);
      return json({ choices: [{ message: { content: 'recovered' } }] });
    };
    const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
    const out = await llm.chat([{ role: 'user', content: 'hi' }]);
    expect(out).toBe('recovered');
    expect(sent).toHaveLength(3); // two failures, then success on the third attempt
  });

  it('chat gives up after exactly 3 attempts when the transport always returns 5xx', async () => {
    const sent: { url: string; init?: RequestInit }[] = [];
    const transport = async (url: string, init?: RequestInit) => {
      sent.push({ url, init });
      return httpError(500, 'still down');
    };
    const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
    await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 500/);
    expect(sent).toHaveLength(3);
  });

  it('chat does not retry on 4xx (deterministic client error → first attempt only)', async () => {
    const sent: { url: string; init?: RequestInit }[] = [];
    const transport = async (url: string, init?: RequestInit) => {
      sent.push({ url, init });
      return httpError(400, 'bad request');
    };
    const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
    await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 400/);
    expect(sent).toHaveLength(1);
  });

  it('chat retries a null content (truncation) and then succeeds (null → content)', async () => {
    const sent: { url: string; init?: RequestInit }[] = [];
    const transport = async (url: string, init?: RequestInit) => {
      sent.push({ url, init });
      if (sent.length === 1) return json({ choices: [{ message: { content: null } }] });
      return json({ choices: [{ message: { content: 'real content' } }] });
    };
    const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
    const out = await llm.chat([{ role: 'user', content: 'hi' }]);
    expect(out).toBe('real content');
    expect(sent).toHaveLength(2);
  });

  it('chat rejects with the HTTP status after 3 attempts when 5xx never recovers', async () => {
    const sent: { url: string; init?: RequestInit }[] = [];
    const transport = async (url: string, init?: RequestInit) => {
      sent.push({ url, init });
      return httpError(503);
    };
    const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
    await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 503/);
    expect(sent).toHaveLength(3);
  });

  it('chat rejects with a content error after 3 attempts when content is never present', async () => {
    const sent: { url: string; init?: RequestInit }[] = [];
    const transport = async (url: string, init?: RequestInit) => {
      sent.push({ url, init });
      return json({ choices: [] });
    };
    const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
    await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/no choices/);
    expect(sent).toHaveLength(3);
  });

  it('chat logs one line on success and one on final failure', async () => {
    const logSpy = vi.spyOn(console, 'log');
    const errorSpy = vi.spyOn(console, 'error');

    const okTransport = async () =>
      json({ choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }] });
    const okLlm = createLlmClient({ transport: okTransport, apiBase: 'http://x/v1', model: 'm' });
    await okLlm.chat([{ role: 'user', content: 'hi' }]);
    expect(logSpy).toHaveBeenCalledWith(
      '[llm] chat ok',
      expect.objectContaining({ ms: expect.any(Number), finishReason: 'stop' }),
    );

    const failSent: { url: string; init?: RequestInit }[] = [];
    const failTransport = async (url: string, init?: RequestInit) => {
      failSent.push({ url, init });
      return httpError(502, 'bad gateway');
    };
    const failLlm = createLlmClient({
      transport: failTransport,
      apiBase: 'http://x/v1',
      model: 'm',
      retryBaseDelayMs: 1,
    });
    await expect(failLlm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 502/);
    expect(failSent).toHaveLength(3);
    expect(errorSpy).toHaveBeenCalledWith(
      '[llm] chat failed',
      expect.objectContaining({
        attempts: 3,
        lastStatus: 502,
        message: expect.stringContaining('HTTP 502'),
        totalMs: expect.any(Number),
      }),
    );
  });
});