diff --git a/AGENTS.md b/AGENTS.md index fb2ae49..ce31846 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ ## API 1. **Live base URL** — `http://bot.dev.kkhost.pl` (Traefik → docker-compose stack on this host; container port 3000 via external network dev_net, no published ports). -2. **Endpoints** — all under the `/api` global prefix: `GET /api/status` (`{ connected, myNodeNum, host, port }`), `POST /api/send` (body `{ "text": "...", "to"?: }` — numbers pass through as-is (integer 1..0xffffffff), names resolve like `GET /api/nodes/:id`, omitted/""/"!b"/"broadcast" → broadcast; unknown or ambiguous targets are a 400. Returns `{ id }`), `GET /api/channels` (the device's Meshtastic channels as `{ index, name, uplinkEnabled, downlinkEnabled }[]`; psk/keys redacted), `GET /api/device` (device info captured during the initial sync: `{ myNodeNum, firmwareVersion?, metadata?, localConfig: {
: ...}, moduleConfig: {: ...} }` — DeviceMetadata as-is plus per-section Config/ModuleConfig messages; security publicKey/privateKey/adminKey and network wifiPsk redacted to `"REDACTED"`; in-memory, reset on reconnect), `GET /api/messages` (message history newest first: rx text with `from`, tx sends with `to` + `status` pending→delivered/failed via routing ACKs — broadcasts stay pending because firmware strips want_ack; last 100 in memory (seeded from SQLite on startup) — backed by SQLite: re-upserted every MESH_POLL_MS and pruned to the newest 1000 rows, so it survives process restarts and bridge reconnects), `POST /api/messages` (body `{ "text": "...", "from"?: }`, returns `{ id }`; synthetic inbox injection for external triggers — records an rx entry in the history as if received on the mesh (channel 0) so external systems can push messages to trigger communication later; NOT transmitted over RF, nothing goes to the bridge or air; appears in `GET /api/messages` like any other rx message), `GET /api/telemetry` (telemetry history newest first as `{ from, at /* epoch ms */, kind, data }[]`: live TELEMETRY_APP packets — device metrics and environment sensor readings where nodes have them; kind is the meshtastic.Telemetry oneof case name (deviceMetrics/environmentMetrics/airQualityMetrics/powerMetrics/localStats/healthMetrics/hostMetrics/trafficManagementStats), data that variant's payload with proto uint64 fields as decimal strings; last 100, in-memory, reset on reconnect), `GET /api/nodes` (known nodes sorted by num as `{ num, longName?, shortName?, hwModel?, batteryLevel?, lastHeard: number|null, position?: { lat, lon, alt? }, snr?, hopsAway?, viaMqtt? }[]`, from the initial sync plus live POSITION_APP updates; reset on reconnect) and `GET /api/nodes/:id` (lookup by exact node number/numeric string, then case-insensitive exact long/short name, else unique substring; 404 when ambiguous or missing). Scalar API docs are served at `/` (OpenAPI JSON at `/docs-json`). Keep this list in sync with the controller. All `POST /api/send` traffic targets channel index 3 (`test`) on purpose — never the primary channel; override via `MESH_SEND_CHANNEL` (default `3`). Message history is persisted to SQLite: file path from `MESH_DB_PATH` (default `./data/mesh-bot.db`, parent dir auto-created), poller re-upserts every `MESH_POLL_MS` ms (default 1000) and prunes the table to the newest 1000 rows. Outbound notification webhook: when an RX text arrives on channel `MESH_NOTIFY_CHANNEL` (default `3`), a fire-and-forget HTTP POST of `{text, from}` (`from` omitted when absent) goes to `MESH_NOTIFY_URL` — unset/empty disables it entirely; no retries, no response handling. Injected messages (`POST /api/messages`, channel 0) trigger it too when the notify channel is set to match. Messages from our own node are never notified (self-echo filter). +2. **Endpoints** — all under the `/api` global prefix: `GET /api/status` (`{ connected, myNodeNum, host, port }`), `POST /api/send` (body `{ "text": "...", "to"?: }` — numbers pass through as-is (integer 1..0xffffffff), names resolve like `GET /api/nodes/:id`, omitted/""/"!b"/"broadcast" → broadcast; unknown or ambiguous targets are a 400. Returns `{ id }`), `GET /api/channels` (the device's Meshtastic channels as `{ index, name, uplinkEnabled, downlinkEnabled }[]`; psk/keys redacted), `GET /api/device` (device info captured during the initial sync: `{ myNodeNum, firmwareVersion?, metadata?, localConfig: {
: ...}, moduleConfig: {: ...} }` — DeviceMetadata as-is plus per-section Config/ModuleConfig messages; security publicKey/privateKey/adminKey and network wifiPsk redacted to `"REDACTED"`; in-memory, reset on reconnect), `GET /api/messages` (message history newest first: rx text with `from`, tx sends with `to` + `status` pending→delivered/failed via routing ACKs — broadcasts stay pending because firmware strips want_ack; last 100 in memory (seeded from SQLite on startup) — backed by SQLite: re-upserted every MESH_POLL_MS and pruned to the newest 1000 rows, so it survives process restarts and bridge reconnects), `POST /api/messages` (body `{ "text": "...", "from"?: }`, returns `{ id }`; synthetic inbox injection for external triggers — records an rx entry in the history as if received on the mesh (channel 0) so external systems can push messages to trigger communication later; NOT transmitted over RF, nothing goes to the bridge or air; appears in `GET /api/messages` like any other rx message), `GET /api/telemetry` (telemetry history newest first as `{ from, at /* epoch ms */, kind, data }[]`: live TELEMETRY_APP packets — device metrics and environment sensor readings where nodes have them; kind is the meshtastic.Telemetry oneof case name (deviceMetrics/environmentMetrics/airQualityMetrics/powerMetrics/localStats/healthMetrics/hostMetrics/trafficManagementStats), data that variant's payload with proto uint64 fields as decimal strings; last 100, in-memory, reset on reconnect), `GET /api/nodes` (known nodes sorted by num as `{ num, longName?, shortName?, hwModel?, batteryLevel?, lastHeard: number|null, position?: { lat, lon, alt? }, snr?, hopsAway?, viaMqtt? }[]`, from the initial sync plus live POSITION_APP updates; reset on reconnect) and `GET /api/nodes/:id` (lookup by exact node number/numeric string, then case-insensitive exact long/short name, else unique substring; 404 when ambiguous or missing). Scalar API docs are served at `/` (OpenAPI JSON at `/docs-json`). Keep this list in sync with the controller. All `POST /api/send` traffic targets channel index 3 (`test`) on purpose — never the primary channel; override via `MESH_SEND_CHANNEL` (default `3`). Message history is persisted to SQLite: file path from `MESH_DB_PATH` (default `./data/mesh-bot.db`, parent dir auto-created), poller re-upserts every `MESH_POLL_MS` ms (default 1000) and prunes the table to the newest 1000 rows. Outbound notification webhook: when an RX text arrives on channel `MESH_NOTIFY_CHANNEL` (default `3`), a fire-and-forget HTTP POST of `{text, from}` (`from` omitted when absent) goes to `MESH_NOTIFY_URL` — unset/empty disables it entirely; failed POSTs are retried up to 3 times with backoff (2 s / 10 s / 30 s) before being dropped; no response handling. Injected messages (`POST /api/messages`, channel 0) trigger it too when the notify channel is set to match. Messages from our own node are never notified (self-echo filter). 3. **MCP endpoint** — `POST /mcp` (excluded from the `/api` prefix in main.ts, like Scalar at `/`): an in-process MCP server over the official SDK's Streamable HTTP transport in stateless mode (`sessionIdGenerator: undefined`, fresh `McpServer` + transport per request). Tools map 1:1 to MeshService methods: `mesh_status`, `send_message(text, to?)` (same resolution as `POST /api/send`; errors surface as tool errors with the same messages), `list_channels`, `list_nodes`, `get_node(id)` (tool error when missing/ambiguous), `get_messages`, `inject_message(text, from?)` (synthetic inbox injection for external triggers — records an rx entry in the history as if received on the mesh; NOT sent over RF; same validation as `POST /api/messages`), `get_telemetry` (same bounded recent window as `GET /api/telemetry`), `device_info` (same redaction as `GET /api/device`). Results are JSON in a text content block; sends ride channel index 3 like everything else. Note: the SDK's root entry is broken upstream (#2273) — import via subpaths (`@modelcontextprotocol/sdk/server/mcp.js` etc.). ## Other diff --git a/CHANGELOG.md b/CHANGELOG.md index f231a84..7c97025 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v0.12.3 — 2026-09-25 09:40 + +- Notify webhook POSTs are now retried instead of dropped on the first failure: `notifyRx`'s fire-and-forget POST runs through a retry loop with up to 4 attempts total (initial + 3 retries) and backoff delays `[2 s, 10 s, 30 s]` (~45 s window). Each failed attempt logs WARN with the attempt number and error; after the final failure it logs WARN `notify dropped after N attempts: ...` and stops. Success on any attempt ends the loop (unchanged success behavior/logging). Fire-and-forget semantics preserved — the RX handler never awaits the loop, which runs as an un-awaited async IIFE exactly like today's single POST. Self-echo filter and channel-match logic unchanged; backoff is hardcoded (no new env vars). +- New private `MeshService.postNotify()` retry helper + injectable `sleep()` so specs can skip the delays without fake timers. Two new specs: fetch fails twice then succeeds → exactly 3 attempts, no dropped log, success path taken; fetch always fails → exactly 4 attempts, final WARN `notify dropped after 4 attempts`, nothing thrown. All 59 pre-existing tests stay green (61 total). +- Docs: README `MESH_NOTIFY_URL` bullet and AGENTS.md webhook note now state failed POSTs are retried up to 3 times with backoff (2 s / 10 s / 30 s) before being dropped. + ## v0.12.2 — 2026-09-25 08:44 - New compose service `forwarder` (`ops/forwarder`, `python:3.12-slim`, stdlib only, no pip installs): runs the notify forwarder that was previously a host-side `nohup` python script — receives the bot's outbound notify webhook POSTs (`{text, from?}` on `MESH_NOTIFY_URL`) and forwards each one as a message to an opencode API session (Basic auth). diff --git a/README.md b/README.md index 760c706..4362b2a 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Config lives in `.env` (gitignored) and is passed via compose environment — cr - `MESH_PORT` (default `4403`) — bridge port - `MESH_RECONNECT_MS` (default `5000`) — reconnect interval in ms - `MESH_SEND_CHANNEL` (default `3`) — channel index all sends target (`test`; never the primary channel) -- `MESH_NOTIFY_URL` — outbound notification webhook URL; when set (non-empty), an inbound rx text message arriving on channel `MESH_NOTIFY_CHANNEL` fires a fire-and-forget HTTP POST with JSON body `{text, from}` (`from` omitted when absent). Synthetic injected messages (`POST /api/messages`, channel 0) trigger it too when the notify channel is set to match. Messages from our own node are never notified (self-echo filter). Unset or empty = disabled (no fetch at all) +- `MESH_NOTIFY_URL` — outbound notification webhook URL; when set (non-empty), an inbound rx text message arriving on channel `MESH_NOTIFY_CHANNEL` fires a fire-and-forget HTTP POST with JSON body `{text, from}` (`from` omitted when absent). Failed POSTs are retried up to 3 times with backoff (2 s / 10 s / 30 s) before being dropped. Synthetic injected messages (`POST /api/messages`, channel 0) trigger it too when the notify channel is set to match. Messages from our own node are never notified (self-echo filter). Unset or empty = disabled (no fetch at all) - `MESH_NOTIFY_CHANNEL` (default `3`) — channel index that triggers the notify webhook - `MESH_DB_PATH` (default `./data/mesh-bot.db`) — SQLite file backing the message history (parent dir created if missing; in Docker, mounted at `/app/data` via a named volume so it survives container recreation) - `MESH_POLL_MS` (default `1000`) — how often the poller re-upserts the message history into SQLite and prunes rows beyond the newest 1000 diff --git a/package.json b/package.json index a6629b9..15fd07c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mesh-bot", - "version": "0.12.2", + "version": "0.12.3", "description": "", "author": "", "private": true, diff --git a/src/mesh/mesh.service.spec.ts b/src/mesh/mesh.service.spec.ts index 6e158c5..be7ed20 100644 --- a/src/mesh/mesh.service.spec.ts +++ b/src/mesh/mesh.service.spec.ts @@ -1,4 +1,5 @@ import { create, fromBinary, toBinary } from '@bufbuild/protobuf'; +import { Logger } from '@nestjs/common'; import * as P from '@meshtastic/protobufs'; import { DatabaseSync } from 'node:sqlite'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -789,6 +790,69 @@ describe('MeshService notify webhook', () => { bridge.close(); } }); + + it('retries a failed webhook POST with backoff and stops on success', async () => { + process.env.MESH_NOTIFY_URL = 'http://notify.example/hook'; + process.env.MESH_NOTIFY_CHANNEL = '0'; + const fetchMock = vi + .fn() + .mockRejectedValueOnce(new Error('boom 1')) + .mockRejectedValueOnce(new Error('boom 2')) + .mockResolvedValueOnce(undefined); + vi.stubGlobal('fetch', fetchMock); + const warnSpy = vi.spyOn(Logger.prototype, 'warn'); + const sleep = vi.fn(async (_ms: number) => {}); + const svc = new MeshService(); + (svc as unknown as { sleep(ms: number): Promise }).sleep = sleep; + try { + expect(() => svc.injectMessage('hi', 42)).not.toThrow(); + + await waitFor(() => fetchMock.mock.calls.length === 3); + // success on the third attempt: exactly 3 POSTs, no dropped log + expect(fetchMock).toHaveBeenCalledTimes(3); + expect(sleep).toHaveBeenNthCalledWith(1, 2_000); + expect(sleep).toHaveBeenNthCalledWith(2, 10_000); + const warns = warnSpy.mock.calls.map((c) => String(c[0])); + expect(warns.some((w) => w.includes('attempt 1/4'))).toBe(true); + expect(warns.some((w) => w.includes('attempt 2/4'))).toBe(true); + expect(warns.some((w) => w.includes('notify dropped'))).toBe(false); + } finally { + warnSpy.mockRestore(); + vi.unstubAllGlobals(); + delete process.env.MESH_NOTIFY_URL; + delete process.env.MESH_NOTIFY_CHANNEL; + } + }); + + it('gives up after 4 attempts and logs notify dropped', async () => { + process.env.MESH_NOTIFY_URL = 'http://notify.example/hook'; + process.env.MESH_NOTIFY_CHANNEL = '0'; + const fetchMock = vi.fn().mockRejectedValue(new Error('down')); + vi.stubGlobal('fetch', fetchMock); + const warnSpy = vi.spyOn(Logger.prototype, 'warn'); + const sleep = vi.fn(async (_ms: number) => {}); + const svc = new MeshService(); + (svc as unknown as { sleep(ms: number): Promise }).sleep = sleep; + try { + expect(() => svc.injectMessage('hi', 42)).not.toThrow(); + + await waitFor(() => + warnSpy.mock.calls.some((c) => + String(c[0]).includes('notify dropped after 4 attempts'), + ), + ); + // exactly 4 attempts, final WARN with the exact message, nothing thrown + expect(fetchMock).toHaveBeenCalledTimes(4); + const warns = warnSpy.mock.calls.map((c) => String(c[0])); + expect(warns).toContain('notify dropped after 4 attempts: down'); + expect(sleep).toHaveBeenNthCalledWith(3, 30_000); + } finally { + warnSpy.mockRestore(); + vi.unstubAllGlobals(); + delete process.env.MESH_NOTIFY_URL; + delete process.env.MESH_NOTIFY_CHANNEL; + } + }); }); describe('MeshService injectMessage', () => { diff --git a/src/mesh/mesh.service.ts b/src/mesh/mesh.service.ts index 7ae526c..a8c1de1 100644 --- a/src/mesh/mesh.service.ts +++ b/src/mesh/mesh.service.ts @@ -130,6 +130,8 @@ function jsonSafe(value: T): T { // key material must never leave the process — replaced with a marker so callers can see the field exists const REDACTED = 'REDACTED'; +// ponytail: hardcoded notify-webhook backoff [2s, 10s, 30s] (~45 s window) — no env knob; tune here if outages run longer +const NOTIFY_RETRY_DELAYS_MS = [2_000, 10_000, 30_000]; /** Config sections that carry secrets: security (PKI keys) and network (wifi psk). */ function redactSection( name: string, @@ -441,16 +443,40 @@ export class MeshService implements OnModuleInit, OnModuleDestroy { } const body: Record = { text }; if (from !== undefined) body.from = from; - void fetch(this.notifyUrl, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify(body), - }).catch((err: unknown) => - this.logger.warn('notify webhook failed: ' + (err as Error).message), - ); + void this.postNotify(body); } } + /** POST the webhook with retries: up to NOTIFY_RETRY_DELAYS_MS.length + 1 attempts total. */ + private async postNotify(body: Record): Promise { + const maxAttempts = NOTIFY_RETRY_DELAYS_MS.length + 1; + for (let attempt = 1; ; attempt++) { + try { + await fetch(this.notifyUrl, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + return; // success on any attempt — same as today's single POST + } catch (err: unknown) { + const msg = (err as Error).message; + this.logger.warn( + `notify webhook failed (attempt ${attempt}/${maxAttempts}): ${msg}`, + ); + if (attempt === maxAttempts) { + this.logger.warn(`notify dropped after ${attempt} attempts: ${msg}`); + return; + } + await this.sleep(NOTIFY_RETRY_DELAYS_MS[attempt - 1]); + } + } + } + + /** Injectable so specs can skip the backoff delays. */ + private sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + /** * Telemetry records, newest first: TELEMETRY_APP packets from any node — device metrics * (battery/voltage/uptime/channel+air utilization) and environment sensor readings where nodes have them.