Force reconnect when the bridge connection goes silent (v0.21.1)
CI / build + unit (pull_request) Successful in 1m8s

Production incident: the bridge TCP connection went zombie — outbound sends
kept working for ~10 h while inbound packets stopped entirely, and
/api/status stayed connected:true because nothing detects inbound silence on
an established connection.

- Track lastPacketAt: updated on every chunk received from the bridge in the
  lowest-level socket 'data' path (any frame kind, decodable or not) and reset
  when a connection successfully starts, so a fresh connection can't
  self-trigger.
- Idle watchdog: a 30 s interval checks connected AND now - lastPacketAt >
  MESH_IDLE_TIMEOUT_MS (new env, default 600000 = 10 min) — logs WARN with the
  idle duration and destroys the socket so the EXISTING connect/retry loop
  re-establishes it with a fresh sync. No changes to connect-error retry or
  backoff; this only adds detection for established-but-silent connections.
- TCP keepalive: socket.setKeepAlive(true, 30_000) on the bridge socket —
  cheap insurance against dead peers at the TCP level.

Tests (fake-bridge based): silence past an overridden timeout forces a
disconnect + reconnect; inbound traffic keeps resetting the idle clock so no
false trigger; the default timeout does not fire in a test window; and
setKeepAlive(true, 30_000) is asserted on the socket. checkIdle() is public so
tests drive a tick without waiting for the real interval (like flushHistory()).
This commit is contained in:
2026-09-26 06:50:01 +00:00
parent 73ce49c741
commit af1c77fd54
5 changed files with 145 additions and 2 deletions
+102
View File
@@ -1,6 +1,7 @@
import { create, fromBinary, toBinary } from '@bufbuild/protobuf';
import * as P from '@meshtastic/protobufs';
import { BadRequestException, NotFoundException } from '@nestjs/common';
import * as net from 'node:net';
import { DatabaseSync } from 'node:sqlite';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { BROADCAST_ADDR, extractFrames, frame } from './codec.js';
@@ -249,6 +250,107 @@ describe('MeshService connect', () => {
});
});
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
describe('MeshService idle watchdog', () => {
it('forces a reconnect when an established connection goes silent past MESH_IDLE_TIMEOUT_MS', async () => {
const bridge = await startBridge();
process.env.MESH_HOST = '127.0.0.1';
process.env.MESH_PORT = String(bridge.port);
process.env.MESH_RECONNECT_MS = '50'; // fast retry so the re-establishment is observable
process.env.MESH_IDLE_TIMEOUT_MS = '100'; // override — the default (600000) would never fire in a test
const svc = new MeshService();
try {
svc.onModuleInit();
await waitFor(
() => bridge.sockets.length === 1 && svc.status().connected,
);
// zombie case: connected but no inbound traffic at all — wait past the timeout, then drive a watchdog tick
await sleep(250);
svc.checkIdle();
// socket torn down → existing connect/retry loop re-establishes with a fresh session
await waitFor(() => bridge.sockets.length >= 2 && svc.status().connected);
} finally {
delete process.env.MESH_RECONNECT_MS;
delete process.env.MESH_IDLE_TIMEOUT_MS;
svc.onModuleDestroy();
bridge.close();
}
});
it('inbound traffic keeps resetting the idle clock — no false trigger while packets arrive', async () => {
const bridge = await startBridge();
process.env.MESH_HOST = '127.0.0.1';
process.env.MESH_PORT = String(bridge.port);
process.env.MESH_IDLE_TIMEOUT_MS = '100';
const svc = new MeshService();
try {
svc.onModuleInit();
await waitFor(
() => bridge.sockets.length === 1 && svc.status().connected,
);
// steady drip of inbound frames for well over the timeout in wall time — each chunk resets lastPacketAt
for (let i = 0; i < 5; i++) {
await sleep(60);
bridge.sockets[0].write(nodeInfoFrame(42, kamilNodeInfo()));
}
svc.checkIdle(); // wall time is past the timeout, but traffic kept resetting it
await sleep(100); // enough for a destroy to surface as 'close' (connected would flip false)
expect(svc.status().connected).toBe(true);
expect(bridge.sockets.length).toBe(1);
} finally {
delete process.env.MESH_IDLE_TIMEOUT_MS;
svc.onModuleDestroy();
bridge.close();
}
});
it('does not trigger under the default timeout (MESH_IDLE_TIMEOUT_MS unset)', async () => {
const bridge = await startBridge();
process.env.MESH_HOST = '127.0.0.1';
process.env.MESH_PORT = String(bridge.port);
delete process.env.MESH_IDLE_TIMEOUT_MS; // default 600000 — a test can't wait that long, so silence must NOT fire
const svc = new MeshService();
try {
svc.onModuleInit();
await waitFor(
() => bridge.sockets.length === 1 && svc.status().connected,
);
await sleep(250); // silent, but far under the default budget
svc.checkIdle();
expect(svc.status().connected).toBe(true);
expect(bridge.sockets.length).toBe(1);
} finally {
delete process.env.MESH_IDLE_TIMEOUT_MS;
svc.onModuleDestroy();
bridge.close();
}
});
it('enables TCP keepalive on the bridge socket', async () => {
const setKeepAlive = vi.spyOn(net.Socket.prototype, 'setKeepAlive');
const bridge = await startBridge();
process.env.MESH_HOST = '127.0.0.1';
process.env.MESH_PORT = String(bridge.port);
const svc = new MeshService();
try {
svc.onModuleInit();
await waitFor(() => svc.status().connected);
expect(setKeepAlive).toHaveBeenCalledWith(true, 30_000);
} finally {
setKeepAlive.mockRestore();
svc.onModuleDestroy();
bridge.close();
}
});
});
describe('MeshService send', () => {
it('sends on the test channel (index 3) by default, never primary', async () => {
const bridge = await startBridge();