Retry notify webhook POST with backoff (2 s / 10 s / 30 s) before dropping (#27)

src/mesh/mesh.service.spec.ts

@@ -1,4 +1,5 @@
 import { create, fromBinary, toBinary } from '@bufbuild/protobuf';
+import { Logger } from '@nestjs/common';
 import * as P from '@meshtastic/protobufs';
 import { DatabaseSync } from 'node:sqlite';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
@@ -789,6 +790,69 @@ describe('MeshService notify webhook', () => {
       bridge.close();
     }
   });
+
+  it('retries a failed webhook POST with backoff and stops on success', async () => {
+    process.env.MESH_NOTIFY_URL = 'http://notify.example/hook';
+    process.env.MESH_NOTIFY_CHANNEL = '0';
+    const fetchMock = vi
+      .fn()
+      .mockRejectedValueOnce(new Error('boom 1'))
+      .mockRejectedValueOnce(new Error('boom 2'))
+      .mockResolvedValueOnce(undefined);
+    vi.stubGlobal('fetch', fetchMock);
+    const warnSpy = vi.spyOn(Logger.prototype, 'warn');
+    const sleep = vi.fn(async (_ms: number) => {});
+    const svc = new MeshService();
+    (svc as unknown as { sleep(ms: number): Promise<void> }).sleep = sleep;
+    try {
+      expect(() => svc.injectMessage('hi', 42)).not.toThrow();
+
+      await waitFor(() => fetchMock.mock.calls.length === 3);
+      // success on the third attempt: exactly 3 POSTs, no dropped log
+      expect(fetchMock).toHaveBeenCalledTimes(3);
+      expect(sleep).toHaveBeenNthCalledWith(1, 2_000);
+      expect(sleep).toHaveBeenNthCalledWith(2, 10_000);
+      const warns = warnSpy.mock.calls.map((c) => String(c[0]));
+      expect(warns.some((w) => w.includes('attempt 1/4'))).toBe(true);
+      expect(warns.some((w) => w.includes('attempt 2/4'))).toBe(true);
+      expect(warns.some((w) => w.includes('notify dropped'))).toBe(false);
+    } finally {
+      warnSpy.mockRestore();
+      vi.unstubAllGlobals();
+      delete process.env.MESH_NOTIFY_URL;
+      delete process.env.MESH_NOTIFY_CHANNEL;
+    }
+  });
+
+  it('gives up after 4 attempts and logs notify dropped', async () => {
+    process.env.MESH_NOTIFY_URL = 'http://notify.example/hook';
+    process.env.MESH_NOTIFY_CHANNEL = '0';
+    const fetchMock = vi.fn().mockRejectedValue(new Error('down'));
+    vi.stubGlobal('fetch', fetchMock);
+    const warnSpy = vi.spyOn(Logger.prototype, 'warn');
+    const sleep = vi.fn(async (_ms: number) => {});
+    const svc = new MeshService();
+    (svc as unknown as { sleep(ms: number): Promise<void> }).sleep = sleep;
+    try {
+      expect(() => svc.injectMessage('hi', 42)).not.toThrow();
+
+      await waitFor(() =>
+        warnSpy.mock.calls.some((c) =>
+          String(c[0]).includes('notify dropped after 4 attempts'),
+        ),
+      );
+      // exactly 4 attempts, final WARN with the exact message, nothing thrown
+      expect(fetchMock).toHaveBeenCalledTimes(4);
+      const warns = warnSpy.mock.calls.map((c) => String(c[0]));
+      expect(warns).toContain('notify dropped after 4 attempts: down');
+      expect(sleep).toHaveBeenNthCalledWith(3, 30_000);
+    } finally {
+      warnSpy.mockRestore();
+      vi.unstubAllGlobals();
+      delete process.env.MESH_NOTIFY_URL;
+      delete process.env.MESH_NOTIFY_CHANNEL;
+    }
+  });
 });
 
 describe('MeshService injectMessage', () => {

src/mesh/mesh.service.ts

@@ -130,6 +130,8 @@ function jsonSafe<T>(value: T): T {
 
 // key material must never leave the process — replaced with a marker so callers can see the field exists
 const REDACTED = 'REDACTED';
+// ponytail: hardcoded notify-webhook backoff [2s, 10s, 30s] (~45 s window) — no env knob; tune here if outages run longer
+const NOTIFY_RETRY_DELAYS_MS = [2_000, 10_000, 30_000];
 /** Config sections that carry secrets: security (PKI keys) and network (wifi psk). */
 function redactSection(
   name: string,
@@ -441,16 +443,40 @@ export class MeshService implements OnModuleInit, OnModuleDestroy {
       }
       const body: Record<string, unknown> = { text };
       if (from !== undefined) body.from = from;
-      void fetch(this.notifyUrl, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify(body),
-      }).catch((err: unknown) =>
-        this.logger.warn('notify webhook failed: ' + (err as Error).message),
-      );
+      void this.postNotify(body);
     }
   }
 
+  /** POST the webhook with retries: up to NOTIFY_RETRY_DELAYS_MS.length + 1 attempts total. */
+  private async postNotify(body: Record<string, unknown>): Promise<void> {
+    const maxAttempts = NOTIFY_RETRY_DELAYS_MS.length + 1;
+    for (let attempt = 1; ; attempt++) {
+      try {
+        await fetch(this.notifyUrl, {
+          method: 'POST',
+          headers: { 'content-type': 'application/json' },
+          body: JSON.stringify(body),
+        });
+        return; // success on any attempt — same as today's single POST
+      } catch (err: unknown) {
+        const msg = (err as Error).message;
+        this.logger.warn(
+          `notify webhook failed (attempt ${attempt}/${maxAttempts}): ${msg}`,
+        );
+        if (attempt === maxAttempts) {
+          this.logger.warn(`notify dropped after ${attempt} attempts: ${msg}`);
+          return;
+        }
+        await this.sleep(NOTIFY_RETRY_DELAYS_MS[attempt - 1]);
+      }
+    }
+  }
+
+  /** Injectable so specs can skip the backoff delays. */
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
   /**
    * Telemetry records, newest first: TELEMETRY_APP packets from any node — device metrics
    * (battery/voltage/uptime/channel+air utilization) and environment sensor readings where nodes have them.