AGENTS.md

@@ -17,7 +17,7 @@
 ## API
 
 1. **Live base URL** — `http://bot.dev.kkhost.pl` (Traefik → docker-compose stack on this host; container port 3000 via external network dev_net, no published ports).
-2. **Endpoints** — all under the `/api` global prefix: `GET /api/status` (`{ connected, myNodeNum, host, port }`), `POST /api/send` (body `{ "text": "...", "to"?: <node number | node name | "!b" | "broadcast"> }` — numbers pass through as-is (integer 1..0xffffffff), names resolve like `GET /api/nodes/:id`, omitted/""/"!b"/"broadcast" → broadcast; unknown or ambiguous targets are a 400. Returns `{ id }`), `GET /api/channels` (the device's Meshtastic channels as `{ index, name, uplinkEnabled, downlinkEnabled }[]`; psk/keys redacted), `GET /api/device` (device info captured during the initial sync: `{ myNodeNum, firmwareVersion?, metadata?, localConfig: {<section>: ...}, moduleConfig: {<module>: ...} }` — DeviceMetadata as-is plus per-section Config/ModuleConfig messages; security publicKey/privateKey/adminKey and network wifiPsk redacted to `"REDACTED"`; in-memory, reset on reconnect), `GET /api/messages` (message history newest first: rx text with `from`, tx sends with `to` + `status` pending→delivered/failed via routing ACKs — broadcasts stay pending because firmware strips want_ack; last 100 in memory (seeded from SQLite on startup) — backed by SQLite: re-upserted every MESH_POLL_MS and pruned to the newest 1000 rows, so it survives process restarts and bridge reconnects), `POST /api/messages` (body `{ "text": "...", "from"?: <node number> }`, returns `{ id }`; synthetic inbox injection for external triggers — records an rx entry in the history as if received on the mesh (channel 0) so external systems can push messages to trigger communication later; NOT transmitted over RF, nothing goes to the bridge or air; appears in `GET /api/messages` like any other rx message), `GET /api/telemetry` (telemetry history newest first as `{ from, at /* epoch ms */, kind, data }[]`: live TELEMETRY_APP packets — device metrics and environment sensor readings where nodes have them; kind is the meshtastic.Telemetry oneof case name (deviceMetrics/environmentMetrics/airQualityMetrics/powerMetrics/localStats/healthMetrics/hostMetrics/trafficManagementStats), data that variant's payload with proto uint64 fields as decimal strings; last 100, in-memory, reset on reconnect), `GET /api/nodes` (known nodes sorted by num as `{ num, longName?, shortName?, hwModel?, batteryLevel?, lastHeard: number|null, position?: { lat, lon, alt? }, snr?, hopsAway?, viaMqtt? }[]`, from the initial sync plus live POSITION_APP updates; reset on reconnect) and `GET /api/nodes/:id` (lookup by exact node number/numeric string, then case-insensitive exact long/short name, else unique substring; 404 when ambiguous or missing). Scalar API docs are served at `/` (OpenAPI JSON at `/docs-json`). Keep this list in sync with the controller. All `POST /api/send` traffic targets channel index 3 (`test`) on purpose — never the primary channel; override via `MESH_SEND_CHANNEL` (default `3`). Message history is persisted to SQLite: file path from `MESH_DB_PATH` (default `./data/mesh-bot.db`, parent dir auto-created), poller re-upserts every `MESH_POLL_MS` ms (default 1000) and prunes the table to the newest 1000 rows. Outbound notification webhook: when an RX text arrives on channel `MESH_NOTIFY_CHANNEL` (default `3`), a fire-and-forget HTTP POST of `{text, from}` (`from` omitted when absent) goes to `MESH_NOTIFY_URL` — unset/empty disables it entirely; no retries, no response handling. Injected messages (`POST /api/messages`, channel 0) trigger it too when the notify channel is set to match. Messages from our own node are never notified (self-echo filter).
+2. **Endpoints** — all under the `/api` global prefix: `GET /api/status` (`{ connected, myNodeNum, host, port }`), `POST /api/send` (body `{ "text": "...", "to"?: <node number | node name | "!b" | "broadcast"> }` — numbers pass through as-is (integer 1..0xffffffff), names resolve like `GET /api/nodes/:id`, omitted/""/"!b"/"broadcast" → broadcast; unknown or ambiguous targets are a 400. Returns `{ id }`), `GET /api/channels` (the device's Meshtastic channels as `{ index, name, uplinkEnabled, downlinkEnabled }[]`; psk/keys redacted), `GET /api/device` (device info captured during the initial sync: `{ myNodeNum, firmwareVersion?, metadata?, localConfig: {<section>: ...}, moduleConfig: {<module>: ...} }` — DeviceMetadata as-is plus per-section Config/ModuleConfig messages; security publicKey/privateKey/adminKey and network wifiPsk redacted to `"REDACTED"`; in-memory, reset on reconnect), `GET /api/messages` (message history newest first: rx text with `from`, tx sends with `to` + `status` pending→delivered/failed via routing ACKs — broadcasts stay pending because firmware strips want_ack; last 100 in memory (seeded from SQLite on startup) — backed by SQLite: re-upserted every MESH_POLL_MS and pruned to the newest 1000 rows, so it survives process restarts and bridge reconnects), `POST /api/messages` (body `{ "text": "...", "from"?: <node number> }`, returns `{ id }`; synthetic inbox injection for external triggers — records an rx entry in the history as if received on the mesh (channel 0) so external systems can push messages to trigger communication later; NOT transmitted over RF, nothing goes to the bridge or air; appears in `GET /api/messages` like any other rx message), `GET /api/telemetry` (telemetry history newest first as `{ from, at /* epoch ms */, kind, data }[]`: live TELEMETRY_APP packets — device metrics and environment sensor readings where nodes have them; kind is the meshtastic.Telemetry oneof case name (deviceMetrics/environmentMetrics/airQualityMetrics/powerMetrics/localStats/healthMetrics/hostMetrics/trafficManagementStats), data that variant's payload with proto uint64 fields as decimal strings; last 100, in-memory, reset on reconnect), `GET /api/nodes` (known nodes sorted by num as `{ num, longName?, shortName?, hwModel?, batteryLevel?, lastHeard: number|null, position?: { lat, lon, alt? }, snr?, hopsAway?, viaMqtt? }[]`, from the initial sync plus live POSITION_APP updates; reset on reconnect) and `GET /api/nodes/:id` (lookup by exact node number/numeric string, then case-insensitive exact long/short name, else unique substring; 404 when ambiguous or missing). Scalar API docs are served at `/` (OpenAPI JSON at `/docs-json`). Keep this list in sync with the controller. All `POST /api/send` traffic targets channel index 3 (`test`) on purpose — never the primary channel; override via `MESH_SEND_CHANNEL` (default `3`). Message history is persisted to SQLite: file path from `MESH_DB_PATH` (default `./data/mesh-bot.db`, parent dir auto-created), poller re-upserts every `MESH_POLL_MS` ms (default 1000) and prunes the table to the newest 1000 rows. Outbound notification webhook: when an RX text arrives on channel `MESH_NOTIFY_CHANNEL` (default `3`), a fire-and-forget HTTP POST of `{text, from}` (`from` omitted when absent) goes to `MESH_NOTIFY_URL` — unset/empty disables it entirely; failed POSTs are retried up to 3 times with backoff (2 s / 10 s / 30 s) before being dropped; no response handling. Injected messages (`POST /api/messages`, channel 0) trigger it too when the notify channel is set to match. Messages from our own node are never notified (self-echo filter).
 3. **MCP endpoint** — `POST /mcp` (excluded from the `/api` prefix in main.ts, like Scalar at `/`): an in-process MCP server over the official SDK's Streamable HTTP transport in stateless mode (`sessionIdGenerator: undefined`, fresh `McpServer` + transport per request). Tools map 1:1 to MeshService methods: `mesh_status`, `send_message(text, to?)` (same resolution as `POST /api/send`; errors surface as tool errors with the same messages), `list_channels`, `list_nodes`, `get_node(id)` (tool error when missing/ambiguous), `get_messages`, `inject_message(text, from?)` (synthetic inbox injection for external triggers — records an rx entry in the history as if received on the mesh; NOT sent over RF; same validation as `POST /api/messages`), `get_telemetry` (same bounded recent window as `GET /api/telemetry`), `device_info` (same redaction as `GET /api/device`). Results are JSON in a text content block; sends ride channel index 3 like everything else. Note: the SDK's root entry is broken upstream (#2273) — import via subpaths (`@modelcontextprotocol/sdk/server/mcp.js` etc.).
 
 ## Other

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v0.12.3 — 2026-09-25 09:40
+
+- Notify webhook POSTs are now retried instead of dropped on the first failure: `notifyRx`'s fire-and-forget POST runs through a retry loop with up to 4 attempts total (initial + 3 retries) and backoff delays `[2 s, 10 s, 30 s]` (~45 s window). Each failed attempt logs WARN with the attempt number and error; after the final failure it logs WARN `notify dropped after N attempts: ...` and stops. Success on any attempt ends the loop (unchanged success behavior/logging). Fire-and-forget semantics preserved — the RX handler never awaits the loop, which runs as an un-awaited async IIFE exactly like today's single POST. Self-echo filter and channel-match logic unchanged; backoff is hardcoded (no new env vars).
+- New private `MeshService.postNotify()` retry helper + injectable `sleep()` so specs can skip the delays without fake timers. Two new specs: fetch fails twice then succeeds → exactly 3 attempts, no dropped log, success path taken; fetch always fails → exactly 4 attempts, final WARN `notify dropped after 4 attempts`, nothing thrown. All 59 pre-existing tests stay green (61 total).
+- Docs: README `MESH_NOTIFY_URL` bullet and AGENTS.md webhook note now state failed POSTs are retried up to 3 times with backoff (2 s / 10 s / 30 s) before being dropped.
+
 ## v0.12.2 — 2026-09-25 08:44
 
 - New compose service `forwarder` (`ops/forwarder`, `python:3.12-slim`, stdlib only, no pip installs): runs the notify forwarder that was previously a host-side `nohup` python script — receives the bot's outbound notify webhook POSTs (`{text, from?}` on `MESH_NOTIFY_URL`) and forwards each one as a message to an opencode API session (Basic auth).

README.md

@@ -113,7 +113,7 @@ Config lives in `.env` (gitignored) and is passed via compose environment — cr
 - `MESH_PORT` (default `4403`) — bridge port
 - `MESH_RECONNECT_MS` (default `5000`) — reconnect interval in ms
 - `MESH_SEND_CHANNEL` (default `3`) — channel index all sends target (`test`; never the primary channel)
-- `MESH_NOTIFY_URL` — outbound notification webhook URL; when set (non-empty), an inbound rx text message arriving on channel `MESH_NOTIFY_CHANNEL` fires a fire-and-forget HTTP POST with JSON body `{text, from}` (`from` omitted when absent). Synthetic injected messages (`POST /api/messages`, channel 0) trigger it too when the notify channel is set to match. Messages from our own node are never notified (self-echo filter). Unset or empty = disabled (no fetch at all)
+- `MESH_NOTIFY_URL` — outbound notification webhook URL; when set (non-empty), an inbound rx text message arriving on channel `MESH_NOTIFY_CHANNEL` fires a fire-and-forget HTTP POST with JSON body `{text, from}` (`from` omitted when absent). Failed POSTs are retried up to 3 times with backoff (2 s / 10 s / 30 s) before being dropped. Synthetic injected messages (`POST /api/messages`, channel 0) trigger it too when the notify channel is set to match. Messages from our own node are never notified (self-echo filter). Unset or empty = disabled (no fetch at all)
 - `MESH_NOTIFY_CHANNEL` (default `3`) — channel index that triggers the notify webhook
 - `MESH_DB_PATH` (default `./data/mesh-bot.db`) — SQLite file backing the message history (parent dir created if missing; in Docker, mounted at `/app/data` via a named volume so it survives container recreation)
 - `MESH_POLL_MS` (default `1000`) — how often the poller re-upserts the message history into SQLite and prunes rows beyond the newest 1000

package.json

@@ -1,6 +1,6 @@
 {
   "name": "mesh-bot",
-  "version": "0.12.2",
+  "version": "0.12.3",
   "description": "",
   "author": "",
   "private": true,

src/mesh/mesh.service.spec.ts

@@ -1,4 +1,5 @@
 import { create, fromBinary, toBinary } from '@bufbuild/protobuf';
+import { Logger } from '@nestjs/common';
 import * as P from '@meshtastic/protobufs';
 import { DatabaseSync } from 'node:sqlite';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
@@ -789,6 +790,69 @@ describe('MeshService notify webhook', () => {
       bridge.close();
     }
   });
+
+  it('retries a failed webhook POST with backoff and stops on success', async () => {
+    process.env.MESH_NOTIFY_URL = 'http://notify.example/hook';
+    process.env.MESH_NOTIFY_CHANNEL = '0';
+    const fetchMock = vi
+      .fn()
+      .mockRejectedValueOnce(new Error('boom 1'))
+      .mockRejectedValueOnce(new Error('boom 2'))
+      .mockResolvedValueOnce(undefined);
+    vi.stubGlobal('fetch', fetchMock);
+    const warnSpy = vi.spyOn(Logger.prototype, 'warn');
+    const sleep = vi.fn(async (_ms: number) => {});
+    const svc = new MeshService();
+    (svc as unknown as { sleep(ms: number): Promise<void> }).sleep = sleep;
+    try {
+      expect(() => svc.injectMessage('hi', 42)).not.toThrow();
+
+      await waitFor(() => fetchMock.mock.calls.length === 3);
+      // success on the third attempt: exactly 3 POSTs, no dropped log
+      expect(fetchMock).toHaveBeenCalledTimes(3);
+      expect(sleep).toHaveBeenNthCalledWith(1, 2_000);
+      expect(sleep).toHaveBeenNthCalledWith(2, 10_000);
+      const warns = warnSpy.mock.calls.map((c) => String(c[0]));
+      expect(warns.some((w) => w.includes('attempt 1/4'))).toBe(true);
+      expect(warns.some((w) => w.includes('attempt 2/4'))).toBe(true);
+      expect(warns.some((w) => w.includes('notify dropped'))).toBe(false);
+    } finally {
+      warnSpy.mockRestore();
+      vi.unstubAllGlobals();
+      delete process.env.MESH_NOTIFY_URL;
+      delete process.env.MESH_NOTIFY_CHANNEL;
+    }
+  });
+
+  it('gives up after 4 attempts and logs notify dropped', async () => {
+    process.env.MESH_NOTIFY_URL = 'http://notify.example/hook';
+    process.env.MESH_NOTIFY_CHANNEL = '0';
+    const fetchMock = vi.fn().mockRejectedValue(new Error('down'));
+    vi.stubGlobal('fetch', fetchMock);
+    const warnSpy = vi.spyOn(Logger.prototype, 'warn');
+    const sleep = vi.fn(async (_ms: number) => {});
+    const svc = new MeshService();
+    (svc as unknown as { sleep(ms: number): Promise<void> }).sleep = sleep;
+    try {
+      expect(() => svc.injectMessage('hi', 42)).not.toThrow();
+
+      await waitFor(() =>
+        warnSpy.mock.calls.some((c) =>
+          String(c[0]).includes('notify dropped after 4 attempts'),
+        ),
+      );
+      // exactly 4 attempts, final WARN with the exact message, nothing thrown
+      expect(fetchMock).toHaveBeenCalledTimes(4);
+      const warns = warnSpy.mock.calls.map((c) => String(c[0]));
+      expect(warns).toContain('notify dropped after 4 attempts: down');
+      expect(sleep).toHaveBeenNthCalledWith(3, 30_000);
+    } finally {
+      warnSpy.mockRestore();
+      vi.unstubAllGlobals();
+      delete process.env.MESH_NOTIFY_URL;
+      delete process.env.MESH_NOTIFY_CHANNEL;
+    }
+  });
 });
 
 describe('MeshService injectMessage', () => {

src/mesh/mesh.service.ts

@@ -130,6 +130,8 @@ function jsonSafe<T>(value: T): T {
 
 // key material must never leave the process — replaced with a marker so callers can see the field exists
 const REDACTED = 'REDACTED';
+// ponytail: hardcoded notify-webhook backoff [2s, 10s, 30s] (~45 s window) — no env knob; tune here if outages run longer
+const NOTIFY_RETRY_DELAYS_MS = [2_000, 10_000, 30_000];
 /** Config sections that carry secrets: security (PKI keys) and network (wifi psk). */
 function redactSection(
   name: string,
@@ -441,16 +443,40 @@ export class MeshService implements OnModuleInit, OnModuleDestroy {
       }
       const body: Record<string, unknown> = { text };
       if (from !== undefined) body.from = from;
-      void fetch(this.notifyUrl, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify(body),
-      }).catch((err: unknown) =>
-        this.logger.warn('notify webhook failed: ' + (err as Error).message),
-      );
+      void this.postNotify(body);
     }
   }
 
+  /** POST the webhook with retries: up to NOTIFY_RETRY_DELAYS_MS.length + 1 attempts total. */
+  private async postNotify(body: Record<string, unknown>): Promise<void> {
+    const maxAttempts = NOTIFY_RETRY_DELAYS_MS.length + 1;
+    for (let attempt = 1; ; attempt++) {
+      try {
+        await fetch(this.notifyUrl, {
+          method: 'POST',
+          headers: { 'content-type': 'application/json' },
+          body: JSON.stringify(body),
+        });
+        return; // success on any attempt — same as today's single POST
+      } catch (err: unknown) {
+        const msg = (err as Error).message;
+        this.logger.warn(
+          `notify webhook failed (attempt ${attempt}/${maxAttempts}): ${msg}`,
+        );
+        if (attempt === maxAttempts) {
+          this.logger.warn(`notify dropped after ${attempt} attempts: ${msg}`);
+          return;
+        }
+        await this.sleep(NOTIFY_RETRY_DELAYS_MS[attempt - 1]);
+      }
+    }
+  }
+
+  /** Injectable so specs can skip the backoff delays. */
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
   /**
    * Telemetry records, newest first: TELEMETRY_APP packets from any node — device metrics
    * (battery/voltage/uptime/channel+air utilization) and environment sensor readings where nodes have them.