Add OpenAI-compatible LLM client service (v0.19.0)
CI / build + unit (pull_request) Successful in 1m14s
CI / build + unit (pull_request) Successful in 1m14s
LlmService wraps the local OpenAI-compatible server (default http://192.168.1.70:1111/v1, no auth) with native fetch only — no new dependencies. listModels() -> data[].id; chat(messages, opts?) -> choices[0].message.content (non-2xx throws with status + body snippet); summarize(text) = chat() with a concise-summary system prompt. Env: LLM_API_BASE / LLM_MODEL (qwen3.8-27b) / LLM_TIMEOUT_MS (60000, AbortSignal.timeout). Plain feature module wired into app.module.ts; docs synced (AGENTS.md, README env list).
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { LlmService } from './llm.service.js';
|
||||
|
||||
@Module({
|
||||
providers: [LlmService],
|
||||
exports: [LlmService],
|
||||
})
|
||||
export class LlmModule {}
|
||||
@@ -0,0 +1,104 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { ChatMessage } from './llm.service.js';
|
||||
import { LlmService } from './llm.service.js';
|
||||
|
||||
const BASE = 'http://192.168.1.70:1111/v1';
|
||||
|
||||
function json(body: unknown): Response {
|
||||
return { ok: true, status: 200, json: async () => body } as Response;
|
||||
}
|
||||
|
||||
/** Fresh service with a clean env (explicit base/model/timeout). */
|
||||
function makeService(): LlmService {
|
||||
process.env.LLM_API_BASE = BASE;
|
||||
process.env.LLM_MODEL = 'qwen3.8-27b';
|
||||
process.env.LLM_TIMEOUT_MS = '60000';
|
||||
return new LlmService();
|
||||
}
|
||||
|
||||
describe('LlmService', () => {
|
||||
let fetchMock: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
for (const k of ['LLM_API_BASE', 'LLM_MODEL', 'LLM_TIMEOUT_MS'])
|
||||
delete process.env[k];
|
||||
});
|
||||
|
||||
it('listModels returns the data[].id strings from GET /models', async () => {
|
||||
const svc = makeService();
|
||||
fetchMock.mockResolvedValueOnce(
|
||||
json({
|
||||
data: [
|
||||
{ id: 'qwen3.8-27b' },
|
||||
{ id: 'text-embedding-nomic-embed-text-v1.5' },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(svc.listModels()).resolves.toEqual([
|
||||
'qwen3.8-27b',
|
||||
'text-embedding-nomic-embed-text-v1.5',
|
||||
]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(fetchMock.mock.calls[0][0]).toBe(`${BASE}/models`);
|
||||
});
|
||||
|
||||
it('chat posts model+messages and returns choices[0].message.content', async () => {
|
||||
const svc = makeService();
|
||||
fetchMock.mockResolvedValueOnce(
|
||||
json({ choices: [{ message: { content: 'hello back' } }] }),
|
||||
);
|
||||
|
||||
const messages: ChatMessage[] = [
|
||||
{ role: 'system', content: 'be brief' },
|
||||
{ role: 'user', content: 'hi' },
|
||||
];
|
||||
await expect(
|
||||
svc.chat(messages, { maxTokens: 50, temperature: 0.2 }),
|
||||
).resolves.toBe('hello back');
|
||||
|
||||
const [url, init] = fetchMock.mock.calls[0];
|
||||
expect(url).toBe(`${BASE}/chat/completions`);
|
||||
expect(init.method).toBe('POST');
|
||||
expect(JSON.parse(init.body)).toEqual({
|
||||
model: 'qwen3.8-27b',
|
||||
messages,
|
||||
maxTokens: 50,
|
||||
temperature: 0.2,
|
||||
});
|
||||
});
|
||||
|
||||
it('chat throws on non-2xx with the HTTP status in the message', async () => {
|
||||
const svc = makeService();
|
||||
fetchMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 503,
|
||||
text: async () => 'upstream down',
|
||||
} as Response);
|
||||
|
||||
await expect(svc.chat([{ role: 'user', content: 'hi' }])).rejects.toThrow(
|
||||
/HTTP 503.*upstream down/,
|
||||
);
|
||||
});
|
||||
|
||||
it('summarize calls chat with the system prompt plus the user text', async () => {
|
||||
const svc = makeService();
|
||||
fetchMock.mockResolvedValueOnce(
|
||||
json({ choices: [{ message: { content: 'the summary' } }] }),
|
||||
);
|
||||
|
||||
await expect(svc.summarize('a long text')).resolves.toBe('the summary');
|
||||
|
||||
const [, init] = fetchMock.mock.calls[0];
|
||||
expect(JSON.parse(init.body).messages).toEqual([
|
||||
{ role: 'system', content: 'Summarize the following text concisely.' },
|
||||
{ role: 'user', content: 'a long text' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/** A single chat message in OpenAI chat-completions shape. */
|
||||
export interface ChatMessage {
|
||||
role: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class LlmService {
|
||||
// read at construction, like other services' env config
|
||||
private readonly apiBase =
|
||||
process.env.LLM_API_BASE ?? 'http://192.168.1.70:1111/v1';
|
||||
private readonly model = process.env.LLM_MODEL ?? 'qwen3.8-27b';
|
||||
// local 27B model can be slow — generous default
|
||||
private readonly timeoutMs = parseInt(
|
||||
process.env.LLM_TIMEOUT_MS ?? '60000',
|
||||
10,
|
||||
);
|
||||
|
||||
/** GET {base}/models → the `data[].id` strings. */
|
||||
async listModels(): Promise<string[]> {
|
||||
const res = await fetch(`${this.apiBase}/models`, {
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`listModels failed: HTTP ${res.status} ${(await res.text()).slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
const body = (await res.json()) as { data?: { id: string }[] };
|
||||
return (body.data ?? []).map((m) => m.id);
|
||||
}
|
||||
|
||||
/** POST {base}/chat/completions → `choices[0].message.content`. */
|
||||
async chat(
|
||||
messages: ChatMessage[],
|
||||
opts?: { maxTokens?: number; temperature?: number },
|
||||
): Promise<string> {
|
||||
const res = await fetch(`${this.apiBase}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ model: this.model, messages, ...opts }),
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`chat failed: HTTP ${res.status} ${(await res.text()).slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
const body = (await res.json()) as {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
};
|
||||
const content = body.choices?.[0]?.message?.content;
|
||||
if (typeof content !== 'string') {
|
||||
throw new Error('chat failed: no choices[0].message.content in response');
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
/** Thin wrapper over chat(): the stated primary use of this service. */
|
||||
async summarize(text: string): Promise<string> {
|
||||
return this.chat([
|
||||
{ role: 'system', content: 'Summarize the following text concisely.' },
|
||||
{ role: 'user', content: text },
|
||||
]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user