OpenCode/History
2
0
Fork 0
Code Issues 1 Pull Requests Actions Activity

reliability: low reasoning_effort, retry-with-backoff, LLM call logging

Browse Source 
- send top-level reasoning_effort: "low" (LLM_REASONING_EFFORT) to cut the qwen3.8-27b CoT ~85%, so content is no longer truncated to null -> the ~50% 502s
- new env: LLM_MAX_TOKENS (default 4000, per-call opts.maxTokens still overrides), LLM_REASONING_EFFORT (default "low", off/empty omits the field)
- chat() retries up to 3 attempts: timeout/abort, 5xx, network errors (ECONNRESET/ECONNREFUSED/socket closed), null/empty content; backoff 2s->3s (2000*1.5^(n-1), +/-20% jitter); 4xx throws immediately
- logging: '[llm] chat ok' {ms, finishReason} on success; '[llm] chat failed' {attempts, lastStatus, message, totalMs} on final failure
- tests: 12 in llm.test.ts (retry success/failure, no 4xx retry, null-content retry, logging spies); AGENTS.md env table + gotchas updated
This commit is contained in:
kamilkrzywda
2026-10-11 13:46:18 +00:00
parent 612de2fc40
commit 3297658c81
3 changed files with 395 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/llm.test.ts
+157 -22
View File
@@ -1,4 +1,4 @@
import { describe, expect, it } from 'vitest';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { createLlmClient, DEFAULT_MAX_TOKENS } from './llm';
function json(body: unknown) {
@@ -10,42 +10,88 @@ function json(body: unknown) {
} as unknown as Response;
}
function httpError(status: number, text = 'upstream down') {
return {
ok: false,
status,
text: async () => text,
json: async () => ({}),
} as unknown as Response;
}
afterEach(() => {
vi.restoreAllMocks();
});
describe('createLlmClient (injectable transport)', () => {
it('chat posts model+messages to {base}/chat/completions and returns content', async () => {
const calls: { url: string; init?: RequestInit }[] = [];
const sent: { url: string; init?: RequestInit }[] = [];
const transport = async (url: string, init?: RequestInit) => {
calls.push({ url, init });
sent.push({ url, init });
return json({ choices: [{ message: { content: 'hello back' } }] });
};
const llm = createLlmClient({ transport, apiBase: 'http://x/v1/', model: 'm1' });
const out = await llm.chat([{ role: 'user', content: 'hi' }], { maxTokens: 10, temperature: 0.5 });
expect(out).toBe('hello back');
expect(calls[0].url).toBe('http://x/v1/chat/completions'); // trailing slash normalized
const body = JSON.parse(calls[0].init!.body as string);
expect(sent[0].url).toBe('http://x/v1/chat/completions'); // trailing slash normalized
const body = JSON.parse(sent[0].init!.body as string);
// The wire key is `max_tokens` (snake_case), never camelCase `maxTokens`.
expect(body).toEqual({
model: 'm1',
messages: [{ role: 'user', content: 'hi' }],
temperature: 0.5,
max_tokens: 10,
reasoning_effort: 'low',
});
// No auth header: the live vLLM endpoint requires no API key.
expect(calls[0].init!.headers).toEqual({ 'content-type': 'application/json' });
expect(sent[0].init!.headers).toEqual({ 'content-type': 'application/json' });
});
it('chat uses a generous default max_tokens when the caller omits it', async () => {
const calls: { url: string; init?: RequestInit }[] = [];
const sent: { url: string; init?: RequestInit }[] = [];
const transport = async (url: string, init?: RequestInit) => {
calls.push({ url, init });
sent.push({ url, init });
return json({ choices: [{ message: { content: 'ok' } }] });
};
const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm1' });
await llm.chat([{ role: 'user', content: 'hi' }]);
const body = JSON.parse(calls[0].init!.body as string);
const body = JSON.parse(sent[0].init!.body as string);
expect(body.max_tokens).toBe(DEFAULT_MAX_TOKENS);
expect(body.max_tokens).toBe(4000);
});
it('chat sends reasoning_effort "low" by default', async () => {
const sent: { url: string; init?: RequestInit }[] = [];
const transport = async (url: string, init?: RequestInit) => {
sent.push({ url, init });
return json({ choices: [{ message: { content: 'ok' } }] });
};
const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm1' });
await llm.chat([{ role: 'user', content: 'hi' }]);
const body = JSON.parse(sent[0].init!.body as string);
expect(body.reasoning_effort).toBe('low');
});
it('chat omits reasoning_effort when it is "off" or empty', async () => {
const sent: { url: string; init?: RequestInit }[] = [];
const transport = async (url: string, init?: RequestInit) => {
sent.push({ url, init });
return json({ choices: [{ message: { content: 'ok' } }] });
};
for (const effort of ['off', 'OFF', '']) {
sent.length = 0;
const llm = createLlmClient({
transport,
apiBase: 'http://x/v1',
model: 'm1',
reasoningEffort: effort,
});
await llm.chat([{ role: 'user', content: 'hi' }]);
const body = JSON.parse(sent[0].init!.body as string);
expect(body.reasoning_effort, `effort=${JSON.stringify(effort)}`).toBeUndefined();
}
});
it('listModels returns data[].id from GET {base}/models', async () => {
const transport = async (url: string) => {
expect(url).toBe('http://x/v1/models');
@@ -55,21 +101,110 @@ describe('createLlmClient (injectable transport)', () => {
expect(await llm.listModels()).toEqual(['a', 'b']);
});
it('chat throws on non-2xx with the HTTP status in the message', async () => {
const transport = async () =>
({
ok: false,
status: 503,
text: async () => 'upstream down',
json: async () => ({}),
} as unknown as Response);
const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm' });
await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 503/);
it('chat retries 5xx and succeeds once the transport recovers (500, 500 → 200)', async () => {
const sent: { url: string; init?: RequestInit }[] = [];
const transport = async (url: string, init?: RequestInit) => {
sent.push({ url, init });
if (sent.length < 3) return httpError(500);
return json({ choices: [{ message: { content: 'recovered' } }] });
};
const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
const out = await llm.chat([{ role: 'user', content: 'hi' }]);
expect(out).toBe('recovered');
expect(sent).toHaveLength(3); // two failures, then success on the third attempt
});
it('chat throws when the response has no content', async () => {
const transport = async () => json({ choices: [] });
const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm' });
it('chat gives up after exactly 3 attempts when the transport always returns 5xx', async () => {
const sent: { url: string; init?: RequestInit }[] = [];
const transport = async (url: string, init?: RequestInit) => {
sent.push({ url, init });
return httpError(500, 'still down');
};
const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 500/);
expect(sent).toHaveLength(3);
});
it('chat does not retry on 4xx (deterministic client error → first attempt only)', async () => {
const sent: { url: string; init?: RequestInit }[] = [];
const transport = async (url: string, init?: RequestInit) => {
sent.push({ url, init });
return httpError(400, 'bad request');
};
const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 400/);
expect(sent).toHaveLength(1);
});
it('chat retries a null content (truncation) and then succeeds (null → content)', async () => {
const sent: { url: string; init?: RequestInit }[] = [];
const transport = async (url: string, init?: RequestInit) => {
sent.push({ url, init });
if (sent.length === 1) return json({ choices: [{ message: { content: null } }] });
return json({ choices: [{ message: { content: 'real content' } }] });
};
const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
const out = await llm.chat([{ role: 'user', content: 'hi' }]);
expect(out).toBe('real content');
expect(sent).toHaveLength(2);
});
it('chat rejects with the HTTP status after 3 attempts when 5xx never recovers', async () => {
const sent: { url: string; init?: RequestInit }[] = [];
const transport = async (url: string, init?: RequestInit) => {
sent.push({ url, init });
return httpError(503);
};
const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 503/);
expect(sent).toHaveLength(3);
});
it('chat rejects with a content error after 3 attempts when content is never present', async () => {
const sent: { url: string; init?: RequestInit }[] = [];
const transport = async (url: string, init?: RequestInit) => {
sent.push({ url, init });
return json({ choices: [] });
};
const llm = createLlmClient({ transport, apiBase: 'http://x/v1', model: 'm', retryBaseDelayMs: 1 });
await expect(llm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/no choices/);
expect(sent).toHaveLength(3);
});
it('chat logs one line on success and one on final failure', async () => {
const logSpy = vi.spyOn(console, 'log');
const errorSpy = vi.spyOn(console, 'error');
const okTransport = async () =>
json({ choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }] });
const okLlm = createLlmClient({ transport: okTransport, apiBase: 'http://x/v1', model: 'm' });
await okLlm.chat([{ role: 'user', content: 'hi' }]);
expect(logSpy).toHaveBeenCalledWith(
'[llm] chat ok',
expect.objectContaining({ ms: expect.any(Number), finishReason: 'stop' }),
);
const failSent: { url: string; init?: RequestInit }[] = [];
const failTransport = async (url: string, init?: RequestInit) => {
failSent.push({ url, init });
return httpError(502, 'bad gateway');
};
const failLlm = createLlmClient({
transport: failTransport,
apiBase: 'http://x/v1',
model: 'm',
retryBaseDelayMs: 1,
});
await expect(failLlm.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(/HTTP 502/);
expect(failSent).toHaveLength(3);
expect(errorSpy).toHaveBeenCalledWith(
'[llm] chat failed',
expect.objectContaining({
attempts: 3,
lastStatus: 502,
message: expect.stringContaining('HTTP 502'),
totalMs: expect.any(Number),
}),
);
});
});