Force reconnect when the bridge connection goes silent (v0.21.1)
CI / build + unit (pull_request) Successful in 1m8s
CI / build + unit (pull_request) Successful in 1m8s
Production incident: the bridge TCP connection went zombie — outbound sends kept working for ~10 h while inbound packets stopped entirely, and /api/status stayed connected:true because nothing detects inbound silence on an established connection. - Track lastPacketAt: updated on every chunk received from the bridge in the lowest-level socket 'data' path (any frame kind, decodable or not) and reset when a connection successfully starts, so a fresh connection can't self-trigger. - Idle watchdog: a 30 s interval checks connected AND now - lastPacketAt > MESH_IDLE_TIMEOUT_MS (new env, default 600000 = 10 min) — logs WARN with the idle duration and destroys the socket so the EXISTING connect/retry loop re-establishes it with a fresh sync. No changes to connect-error retry or backoff; this only adds detection for established-but-silent connections. - TCP keepalive: socket.setKeepAlive(true, 30_000) on the bridge socket — cheap insurance against dead peers at the TCP level. Tests (fake-bridge based): silence past an overridden timeout forces a disconnect + reconnect; inbound traffic keeps resetting the idle clock so no false trigger; the default timeout does not fire in a test window; and setKeepAlive(true, 30_000) is asserted on the socket. checkIdle() is public so tests drive a tick without waiting for the real interval (like flushHistory()).
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import { create, fromBinary, toBinary } from '@bufbuild/protobuf';
|
||||
import * as P from '@meshtastic/protobufs';
|
||||
import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import * as net from 'node:net';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { BROADCAST_ADDR, extractFrames, frame } from './codec.js';
|
||||
@@ -249,6 +250,107 @@ describe('MeshService connect', () => {
|
||||
});
|
||||
});
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
describe('MeshService idle watchdog', () => {
|
||||
it('forces a reconnect when an established connection goes silent past MESH_IDLE_TIMEOUT_MS', async () => {
|
||||
const bridge = await startBridge();
|
||||
process.env.MESH_HOST = '127.0.0.1';
|
||||
process.env.MESH_PORT = String(bridge.port);
|
||||
process.env.MESH_RECONNECT_MS = '50'; // fast retry so the re-establishment is observable
|
||||
process.env.MESH_IDLE_TIMEOUT_MS = '100'; // override — the default (600000) would never fire in a test
|
||||
const svc = new MeshService();
|
||||
try {
|
||||
svc.onModuleInit();
|
||||
await waitFor(
|
||||
() => bridge.sockets.length === 1 && svc.status().connected,
|
||||
);
|
||||
|
||||
// zombie case: connected but no inbound traffic at all — wait past the timeout, then drive a watchdog tick
|
||||
await sleep(250);
|
||||
svc.checkIdle();
|
||||
|
||||
// socket torn down → existing connect/retry loop re-establishes with a fresh session
|
||||
await waitFor(() => bridge.sockets.length >= 2 && svc.status().connected);
|
||||
} finally {
|
||||
delete process.env.MESH_RECONNECT_MS;
|
||||
delete process.env.MESH_IDLE_TIMEOUT_MS;
|
||||
svc.onModuleDestroy();
|
||||
bridge.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('inbound traffic keeps resetting the idle clock — no false trigger while packets arrive', async () => {
|
||||
const bridge = await startBridge();
|
||||
process.env.MESH_HOST = '127.0.0.1';
|
||||
process.env.MESH_PORT = String(bridge.port);
|
||||
process.env.MESH_IDLE_TIMEOUT_MS = '100';
|
||||
const svc = new MeshService();
|
||||
try {
|
||||
svc.onModuleInit();
|
||||
await waitFor(
|
||||
() => bridge.sockets.length === 1 && svc.status().connected,
|
||||
);
|
||||
|
||||
// steady drip of inbound frames for well over the timeout in wall time — each chunk resets lastPacketAt
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await sleep(60);
|
||||
bridge.sockets[0].write(nodeInfoFrame(42, kamilNodeInfo()));
|
||||
}
|
||||
|
||||
svc.checkIdle(); // wall time is past the timeout, but traffic kept resetting it
|
||||
await sleep(100); // enough for a destroy to surface as 'close' (connected would flip false)
|
||||
expect(svc.status().connected).toBe(true);
|
||||
expect(bridge.sockets.length).toBe(1);
|
||||
} finally {
|
||||
delete process.env.MESH_IDLE_TIMEOUT_MS;
|
||||
svc.onModuleDestroy();
|
||||
bridge.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not trigger under the default timeout (MESH_IDLE_TIMEOUT_MS unset)', async () => {
|
||||
const bridge = await startBridge();
|
||||
process.env.MESH_HOST = '127.0.0.1';
|
||||
process.env.MESH_PORT = String(bridge.port);
|
||||
delete process.env.MESH_IDLE_TIMEOUT_MS; // default 600000 — a test can't wait that long, so silence must NOT fire
|
||||
const svc = new MeshService();
|
||||
try {
|
||||
svc.onModuleInit();
|
||||
await waitFor(
|
||||
() => bridge.sockets.length === 1 && svc.status().connected,
|
||||
);
|
||||
|
||||
await sleep(250); // silent, but far under the default budget
|
||||
svc.checkIdle();
|
||||
|
||||
expect(svc.status().connected).toBe(true);
|
||||
expect(bridge.sockets.length).toBe(1);
|
||||
} finally {
|
||||
delete process.env.MESH_IDLE_TIMEOUT_MS;
|
||||
svc.onModuleDestroy();
|
||||
bridge.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('enables TCP keepalive on the bridge socket', async () => {
|
||||
const setKeepAlive = vi.spyOn(net.Socket.prototype, 'setKeepAlive');
|
||||
const bridge = await startBridge();
|
||||
process.env.MESH_HOST = '127.0.0.1';
|
||||
process.env.MESH_PORT = String(bridge.port);
|
||||
const svc = new MeshService();
|
||||
try {
|
||||
svc.onModuleInit();
|
||||
await waitFor(() => svc.status().connected);
|
||||
expect(setKeepAlive).toHaveBeenCalledWith(true, 30_000);
|
||||
} finally {
|
||||
setKeepAlive.mockRestore();
|
||||
svc.onModuleDestroy();
|
||||
bridge.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('MeshService send', () => {
|
||||
it('sends on the test channel (index 3) by default, never primary', async () => {
|
||||
const bridge = await startBridge();
|
||||
|
||||
Reference in New Issue
Block a user