kamilkrzywda
98b9ec5185
CI / build + unit (pull_request) Failing after 21s
Production incident: '#rss get lublin' at 9:41 AM failed with 'The operation was aborted due to timeout' — the LlmService call hit its 60 s cap while the local OpenAI-compatible server (qwen3.8-27b, uncontrollable reasoning tokens) was slow/busy. The same endpoint answers a trivial prompt in ~1 s when idle, so this is latency variance on the server side, not a dead endpoint. Bump the LLM_TIMEOUT_MS fallback from 60000 to 180000 (3 min headroom for slow reasoning periods); still env-overridable, only the default changed.
69 lines
2.3 KiB
TypeScript
69 lines
2.3 KiB
TypeScript
import { Injectable } from '@nestjs/common';
|
|
|
|
/** A single chat message in OpenAI chat-completions shape. */
|
|
export interface ChatMessage {
|
|
role: string;
|
|
content: string;
|
|
}
|
|
|
|
@Injectable()
|
|
export class LlmService {
|
|
// read at construction, like other services' env config
|
|
private readonly apiBase =
|
|
process.env.LLM_API_BASE ?? 'http://192.168.1.70:1111/v1';
|
|
private readonly model = process.env.LLM_MODEL ?? 'qwen3.8-27b';
|
|
// local 27B model can be slow (uncontrollable reasoning tokens) — generous default: 3 min headroom for slow reasoning periods
|
|
private readonly timeoutMs = parseInt(
|
|
process.env.LLM_TIMEOUT_MS ?? '180000',
|
|
10,
|
|
);
|
|
|
|
/** GET {base}/models → the `data[].id` strings. */
|
|
async listModels(): Promise<string[]> {
|
|
const res = await fetch(`${this.apiBase}/models`, {
|
|
signal: AbortSignal.timeout(this.timeoutMs),
|
|
});
|
|
if (!res.ok) {
|
|
throw new Error(
|
|
`listModels failed: HTTP ${res.status} ${(await res.text()).slice(0, 200)}`,
|
|
);
|
|
}
|
|
const body = (await res.json()) as { data?: { id: string }[] };
|
|
return (body.data ?? []).map((m) => m.id);
|
|
}
|
|
|
|
/** POST {base}/chat/completions → `choices[0].message.content`. */
|
|
async chat(
|
|
messages: ChatMessage[],
|
|
opts?: { maxTokens?: number; temperature?: number },
|
|
): Promise<string> {
|
|
const res = await fetch(`${this.apiBase}/chat/completions`, {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: JSON.stringify({ model: this.model, messages, ...opts }),
|
|
signal: AbortSignal.timeout(this.timeoutMs),
|
|
});
|
|
if (!res.ok) {
|
|
throw new Error(
|
|
`chat failed: HTTP ${res.status} ${(await res.text()).slice(0, 200)}`,
|
|
);
|
|
}
|
|
const body = (await res.json()) as {
|
|
choices?: { message?: { content?: string } }[];
|
|
};
|
|
const content = body.choices?.[0]?.message?.content;
|
|
if (typeof content !== 'string') {
|
|
throw new Error('chat failed: no choices[0].message.content in response');
|
|
}
|
|
return content;
|
|
}
|
|
|
|
/** Thin wrapper over chat(): the stated primary use of this service. */
|
|
async summarize(text: string): Promise<string> {
|
|
return this.chat([
|
|
{ role: 'system', content: 'Summarize the following text concisely.' },
|
|
{ role: 'user', content: text },
|
|
]);
|
|
}
|
|
}
|