fix: prevent Bonjour probe retry loops (#107710)

Prevents duplicate Bonjour advertisement attempts by delegating probing, conflict resolution, retries, and interface republishing to ciao.

Related: #98448

Prepared head SHA: 4bd1638acf

Co-authored-by: Shakker <165377636+shakkernerd@users.noreply.github.com>
Reviewed-by: @shakkernerd
This commit is contained in:
Shakker
2026-07-14 19:28:47 +01:00
committed by GitHub
parent 0571e0ecff
commit e63e49b1b0
3 changed files with 80 additions and 458 deletions
+6 -6
View File
@@ -114,12 +114,12 @@ If browsing works but resolving fails, you're usually hitting a LAN policy or mD
The gateway writes a rolling log file (printed on startup as `gateway log file: ...`). Look for `bonjour:` lines, especially:
- `bonjour: advertise failed ...`
- `bonjour: suppressing ciao cancellation ...`
- `bonjour: suppressing ciao netmask assertion ...`
- `bonjour: ... name conflict resolved` / `hostname conflict resolved`
- `bonjour: watchdog detected non-announced service ...`
- `bonjour: disabling advertiser after ... failed restarts ...`
The watchdog treats active `probing`, `announcing`, and fresh conflict-renames as in-progress states. If the service never reaches `announced`, OpenClaw recreates the advertiser and, after repeated failures, disables Bonjour for that gateway process instead of re-advertising forever.
OpenClaw starts each Bonjour service once and leaves probing, retry, name-conflict resolution, and interface-change republishing to the mDNS responder. This avoids overlapping publish attempts during normal network churn. Repeated internal self-probe messages are suppressed so they cannot flood the gateway log.
When multiple OpenClaw gateways advertise from the same host, Bonjour may append suffixes such as `(2)` or `(3)` to keep service instance names unique. Those suffixes are normal conflict resolution and do not indicate duplicate OCM supervision.
Bonjour uses the system hostname for the advertised `.local` host when it's a valid DNS label. If the system hostname contains spaces, underscores, or another invalid DNS-label character, OpenClaw falls back to `openclaw.local`. Set `OPENCLAW_MDNS_HOSTNAME=<name>` before starting the gateway when you need an explicit host label.
@@ -202,13 +202,13 @@ If a node no longer auto-discovers the gateway after Docker setup:
dns-sd -B _openclaw-gw._tcp local.
```
If browsing is empty, or Gateway logs show repeated ciao watchdog cancellations, restore `OPENCLAW_DISABLE_BONJOUR=1` and use a direct or Tailnet route.
If browsing is empty, or Gateway logs show repeated ciao probe failures, restore `OPENCLAW_DISABLE_BONJOUR=1` and use a direct or Tailnet route.
## Common failure modes
- **Bonjour doesn't cross networks**: use Tailnet or SSH.
- **Multicast blocked**: some Wi-Fi networks disable mDNS.
- **Advertiser stuck in probing/announcing**: hosts with blocked multicast, container bridges, WSL, or interface churn can leave the ciao advertiser in a non-announced state. OpenClaw retries a few times, then disables Bonjour for the current gateway process instead of restarting the advertiser forever.
- **Advertiser stuck in probing/announcing**: hosts with blocked multicast, container bridges, WSL, or interface churn can leave the responder in a non-announced state. The gateway remains available through direct, SSH, Tailnet, or wide-area DNS-SD routes; disable LAN Bonjour with `discovery.mdns.mode: "off"` or `OPENCLAW_DISABLE_BONJOUR=1` when multicast is unavailable.
- **Docker bridge networking**: Bonjour auto-disables in detected containers. Set `OPENCLAW_DISABLE_BONJOUR=0` only for host, macvlan, or another mDNS-capable network.
- **Sleep/interface churn**: macOS may temporarily drop mDNS results; retry.
- **Browse works but resolve fails**: keep machine names simple (avoid emojis or punctuation), then restart the gateway. The service instance name derives from the host name, so overly complex names can confuse some resolvers.
+53 -230
View File
@@ -54,10 +54,10 @@ function mockCall(mock: ReturnType<typeof vi.fn>, index = 0): unknown[] {
function enableAdvertiserUnitMode(hostname = "test-host") {
// Allow advertiser to run in unit tests.
delete process.env.VITEST;
process.env.NODE_ENV = "development";
vi.stubEnv("VITEST", undefined);
vi.stubEnv("NODE_ENV", "development");
vi.spyOn(os, "hostname").mockReturnValue(hostname);
process.env.OPENCLAW_MDNS_HOSTNAME = hostname;
vi.stubEnv("OPENCLAW_MDNS_HOSTNAME", hostname);
}
function mockCiaoService(params?: {
@@ -134,18 +134,7 @@ describe("gateway bonjour advertiser", () => {
txt?: unknown;
};
const prevEnv = { ...process.env };
afterEach(() => {
for (const key of Object.keys(process.env)) {
if (!(key in prevEnv)) {
delete process.env[key];
}
}
for (const [key, value] of Object.entries(prevEnv)) {
process.env[key] = value;
}
createService.mockClear();
getResponder.mockReset();
shutdown.mockClear();
@@ -155,6 +144,7 @@ describe("gateway bonjour advertiser", () => {
logger.warn.mockClear();
logger.debug.mockClear();
vi.useRealTimers();
vi.unstubAllEnvs();
vi.restoreAllMocks();
});
@@ -235,7 +225,7 @@ describe("gateway bonjour advertiser", () => {
it("honors truthy OPENCLAW_DISABLE_BONJOUR values", async () => {
enableAdvertiserUnitMode();
process.env.OPENCLAW_DISABLE_BONJOUR = "true";
vi.stubEnv("OPENCLAW_DISABLE_BONJOUR", "true");
const started = await startAdvertiser({
gatewayPort: 18789,
@@ -261,8 +251,8 @@ describe("gateway bonjour advertiser", () => {
it("auto-disables Bonjour on Fly Machines without Docker sentinel files", async () => {
enableAdvertiserUnitMode();
process.env.FLY_MACHINE_ID = "3d8d5459a03038";
process.env.FLY_APP_NAME = "openclaw-clawcks-test";
vi.stubEnv("FLY_MACHINE_ID", "3d8d5459a03038");
vi.stubEnv("FLY_APP_NAME", "openclaw-clawcks-test");
vi.spyOn(fs, "existsSync").mockReturnValue(false);
vi.spyOn(fs, "readFileSync").mockReturnValue("10:cpuset:/\n9:perf_event:/\n8:memory:/\n0::/\n");
@@ -277,7 +267,7 @@ describe("gateway bonjour advertiser", () => {
it("honors explicit Bonjour opt-in inside detected containers", async () => {
enableAdvertiserUnitMode();
process.env.OPENCLAW_DISABLE_BONJOUR = "0";
vi.stubEnv("OPENCLAW_DISABLE_BONJOUR", "0");
vi.spyOn(fs, "existsSync").mockImplementation((filePath) => String(filePath) === "/.dockerenv");
const destroy = vi.fn().mockResolvedValue(undefined);
@@ -383,15 +373,12 @@ describe("gateway bonjour advertiser", () => {
await started.stop();
});
it("logs advertise failures and retries via watchdog", async () => {
it("logs advertise failures without starting a competing retry loop", async () => {
enableAdvertiserUnitMode();
vi.useFakeTimers();
const destroy = vi.fn().mockResolvedValue(undefined);
const advertise = vi
.fn()
.mockRejectedValueOnce(new Error("boom")) // initial advertise fails
.mockResolvedValue(undefined); // watchdog retry succeeds
const advertise = vi.fn().mockRejectedValue(new Error("boom"));
mockCiaoService({ advertise, destroy, serviceState: "unannounced" });
const started = await startAdvertiser({
@@ -406,16 +393,14 @@ describe("gateway bonjour advertiser", () => {
await Promise.resolve();
expectWarnContaining("advertise failed");
// watchdog first retries, then recreates the advertiser after the service
// stays unhealthy across multiple 5s ticks.
await vi.advanceTimersByTimeAsync(25_000);
expect(advertise).toHaveBeenCalledTimes(3);
expect(createService).toHaveBeenCalledTimes(2);
await vi.advanceTimersByTimeAsync(60_000);
expect(advertise).toHaveBeenCalledTimes(1);
expect(createService).toHaveBeenCalledTimes(1);
await started.stop();
await vi.advanceTimersByTimeAsync(60_000);
expect(advertise).toHaveBeenCalledTimes(3);
expect(advertise).toHaveBeenCalledTimes(1);
});
it("handles advertise throwing synchronously", async () => {
@@ -529,132 +514,13 @@ describe("gateway bonjour advertiser", () => {
}
});
it("recreates the advertiser when ciao gets stuck announcing", async () => {
it("never overlaps ciao lifecycle states or conflict handling with another advertise call", async () => {
enableAdvertiserUnitMode();
vi.useFakeTimers();
const stateRef = { value: "announcing" };
const events: string[] = [];
const cleanupException = vi.fn();
const cleanupRejection = vi.fn();
let advertiseCount = 0;
const destroy = vi.fn().mockImplementation(async () => {
events.push("destroy");
});
const advertise = vi.fn().mockImplementation(() => {
advertiseCount += 1;
events.push(`advertise:${advertiseCount}`);
if (advertiseCount === 1) {
stateRef.value = "announcing";
return new Promise<void>(() => {});
}
stateRef.value = "announced";
return Promise.resolve();
});
mockCiaoService({ advertise, destroy, stateRef });
registerUncaughtExceptionHandler.mockImplementation(() => cleanupException);
registerUnhandledRejectionHandler.mockImplementation(() => cleanupRejection);
const started = await startAdvertiser({
gatewayPort: 18789,
sshPort: 2222,
});
expect(createService).toHaveBeenCalledTimes(1);
expect(advertise).toHaveBeenCalledTimes(1);
expect(registerUncaughtExceptionHandler).toHaveBeenCalledTimes(1);
expect(registerUnhandledRejectionHandler).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(25_000);
expectWarnContaining("restarting advertiser");
expect(createService).toHaveBeenCalledTimes(2);
expect(advertise).toHaveBeenCalledTimes(2);
expect(destroy).toHaveBeenCalledTimes(1);
expect(shutdown).not.toHaveBeenCalled();
expect(cleanupException).not.toHaveBeenCalled();
expect(cleanupRejection).not.toHaveBeenCalled();
expect(events).toEqual(["advertise:1", "destroy", "advertise:2"]);
await started.stop();
expect(destroy).toHaveBeenCalledTimes(2);
expect(shutdown).toHaveBeenCalledTimes(1);
expect(cleanupException).toHaveBeenCalledTimes(1);
expect(cleanupRejection).toHaveBeenCalledTimes(1);
});
it("treats probing-to-announcing churn as one unhealthy window", async () => {
enableAdvertiserUnitMode();
vi.useFakeTimers();
const stateRef = { value: "probing" };
const stateRef = { value: "unannounced" };
const destroy = vi.fn().mockResolvedValue(undefined);
const advertise = vi.fn().mockResolvedValue(undefined);
mockCiaoService({ advertise, destroy, stateRef });
const started = await startAdvertiser({
gatewayPort: 18789,
sshPort: 2222,
});
expect(createService).toHaveBeenCalledTimes(1);
expect(advertise).toHaveBeenCalledTimes(1);
setTimeout(() => {
stateRef.value = "announcing";
}, 10_000);
await vi.advanceTimersByTimeAsync(25_000);
expectWarnContaining("service stuck in announcing");
expect(createService).toHaveBeenCalledTimes(2);
expect(advertise).toHaveBeenCalledTimes(2);
expect(destroy).toHaveBeenCalledTimes(1);
expect(shutdown).not.toHaveBeenCalled();
await started.stop();
expect(shutdown).toHaveBeenCalledTimes(1);
});
it("does not re-advertise while ciao is still probing", async () => {
enableAdvertiserUnitMode();
vi.useFakeTimers();
const stateRef = { value: "probing" };
const destroy = vi.fn().mockResolvedValue(undefined);
const advertise = vi.fn().mockResolvedValue(undefined);
mockCiaoService({ advertise, destroy, stateRef });
const started = await startAdvertiser({
gatewayPort: 18789,
sshPort: 2222,
});
expect(advertise).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(15_000);
expect(advertise).toHaveBeenCalledTimes(1);
expect(createService).toHaveBeenCalledTimes(1);
expect(warnMessages().join("\n")).not.toContain(
"watchdog detected non-announced service; attempting re-advertise",
);
await vi.advanceTimersByTimeAsync(10_000);
expectWarnContaining("service stuck in probing");
expect(createService).toHaveBeenCalledTimes(2);
await started.stop();
});
it("defers probing recovery while a name conflict is still settling", async () => {
enableAdvertiserUnitMode();
vi.useFakeTimers();
const stateRef = { value: "probing" };
const destroy = vi.fn().mockResolvedValue(undefined);
const advertise = vi.fn().mockResolvedValue(undefined);
const advertise = vi.fn(() => new Promise<void>(() => {}));
const listenerMap = new Map<string, (value: unknown) => void>();
mockCiaoService({ advertise, destroy, stateRef, listenerMap });
@@ -663,101 +529,58 @@ describe("gateway bonjour advertiser", () => {
sshPort: 2222,
});
await vi.advanceTimersByTimeAsync(10_000);
listenerMap.get("name-change")?.("test-host (OpenClaw) (2)");
await vi.advanceTimersByTimeAsync(15_000);
expect(createService).toHaveBeenCalledTimes(1);
expect(advertise).toHaveBeenCalledTimes(1);
for (const state of ["probing", "announcing", "unannounced", "probed", "announced"]) {
stateRef.value = state;
await vi.advanceTimersByTimeAsync(60_000);
}
listenerMap.get("name-change")?.("test-host (OpenClaw) (2)");
listenerMap.get("hostname-change")?.("test-host-(2)");
expectWarnContaining('name conflict resolved; newName="test-host (OpenClaw) (2)"');
await vi.advanceTimersByTimeAsync(20_000);
expectWarnContaining("service stuck in probing");
expect(createService).toHaveBeenCalledTimes(2);
await started.stop();
});
it("disables bonjour for the process after repeated stuck advertiser restarts", async () => {
enableAdvertiserUnitMode();
vi.useFakeTimers();
const stateRef = { value: "announcing" };
const destroy = vi.fn().mockResolvedValue(undefined);
const advertise = vi.fn(() => new Promise<void>(() => {}));
mockCiaoService({ advertise, destroy, stateRef });
const started = await startAdvertiser({
gatewayPort: 18789,
sshPort: 2222,
});
await vi.advanceTimersByTimeAsync(55_000);
expectWarnContaining("disabling advertiser after 1 stuck-state restart");
expect(createService).toHaveBeenCalledTimes(2);
expect(advertise).toHaveBeenCalledTimes(2);
expect(destroy).toHaveBeenCalledTimes(2);
expect(shutdown).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(60_000);
expect(createService).toHaveBeenCalledTimes(2);
expect(advertise).toHaveBeenCalledTimes(2);
expectWarnContaining('hostname conflict resolved; newHostname="test-host-(2)"');
expect(createService).toHaveBeenCalledTimes(1);
expect(advertise).toHaveBeenCalledTimes(1);
expect(destroy).not.toHaveBeenCalled();
expect(shutdown).not.toHaveBeenCalled();
expect(warnMessages().join("\n")).not.toMatch(
/watchdog|restarting advertiser|disabling advertiser/,
);
await started.stop();
expect(destroy).toHaveBeenCalledTimes(1);
expect(shutdown).toHaveBeenCalledTimes(1);
});
it("disables bonjour when the advertiser flaps within a sliding window", async () => {
it("makes advertiser shutdown idempotent", async () => {
enableAdvertiserUnitMode();
vi.useFakeTimers();
const stateRef = { value: "announced" };
const destroy = vi.fn().mockResolvedValue(undefined);
const advertise = vi.fn().mockResolvedValue(undefined);
mockCiaoService({ advertise, destroy, stateRef });
const cleanupException = vi.fn();
const cleanupRejection = vi.fn();
mockCiaoService({ advertise, destroy });
registerUncaughtExceptionHandler.mockImplementation(() => cleanupException);
registerUnhandledRejectionHandler.mockImplementation(() => cleanupRejection);
const started = await startAdvertiser({
gatewayPort: 18789,
sshPort: 2222,
});
for (let cycle = 0; cycle < 12; cycle += 1) {
stateRef.value = "announced";
await vi.advanceTimersByTimeAsync(5_000);
stateRef.value = "probing";
await vi.advanceTimersByTimeAsync(25_000);
if (
logger.warn.mock.calls.some(
(call) => typeof call[0] === "string" && call[0].includes("disabling advertiser after"),
)
) {
break;
}
}
await Promise.all([started.stop(), started.stop()]);
const disableLog = logger.warn.mock.calls.find(
(call) => typeof call[0] === "string" && call[0].includes("disabling advertiser after"),
);
if (!disableLog) {
throw new Error("expected advertiser disable warning after repeated restarts");
}
expect(String(disableLog[0])).toMatch(/restarts within \d+ minutes/);
const advertiseCallsAtDisable = advertise.mock.calls.length;
const createServiceCallsAtDisable = createService.mock.calls.length;
await vi.advanceTimersByTimeAsync(5 * 60_000);
expect(advertise).toHaveBeenCalledTimes(advertiseCallsAtDisable);
expect(createService).toHaveBeenCalledTimes(createServiceCallsAtDisable);
await started.stop();
expect(destroy).toHaveBeenCalledTimes(1);
expect(shutdown).toHaveBeenCalledTimes(1);
expect(cleanupException).toHaveBeenCalledTimes(1);
expect(cleanupRejection).toHaveBeenCalledTimes(1);
});
it("normalizes hostnames with domains for service names", async () => {
// Allow advertiser to run in unit tests.
delete process.env.VITEST;
process.env.NODE_ENV = "development";
vi.stubEnv("VITEST", undefined);
vi.stubEnv("NODE_ENV", "development");
vi.spyOn(os, "hostname").mockReturnValue("Mac.localdomain");
@@ -781,9 +604,9 @@ describe("gateway bonjour advertiser", () => {
it("falls back to openclaw when system hostname is invalid for DNS", async () => {
// Allow advertiser to run in unit tests.
delete process.env.VITEST;
process.env.NODE_ENV = "development";
delete process.env.OPENCLAW_MDNS_HOSTNAME;
vi.stubEnv("VITEST", undefined);
vi.stubEnv("NODE_ENV", "development");
vi.stubEnv("OPENCLAW_MDNS_HOSTNAME", undefined);
vi.spyOn(os, "hostname").mockReturnValue("My_Lobster Host");
const destroy = vi.fn().mockResolvedValue(undefined);
@@ -877,9 +700,9 @@ describe("gateway bonjour advertiser", () => {
it("uses system hostname when OPENCLAW_MDNS_HOSTNAME is unset", async () => {
// Allow advertiser to run in unit tests.
delete process.env.VITEST;
process.env.NODE_ENV = "development";
delete process.env.OPENCLAW_MDNS_HOSTNAME;
vi.stubEnv("VITEST", undefined);
vi.stubEnv("NODE_ENV", "development");
vi.stubEnv("OPENCLAW_MDNS_HOSTNAME", undefined);
vi.spyOn(os, "hostname").mockReturnValue("Lobster");
const destroy = vi.fn().mockResolvedValue(undefined);
+21 -222
View File
@@ -1,10 +1,10 @@
/** Publishes gateway/canvas/SSH records and repairs stuck or conflicting ciao advertisements. */
/** Publishes gateway/canvas/SSH records through one ciao-owned advertisement lifecycle. */
import fs from "node:fs";
import os from "node:os";
import type { CiaoService } from "@homebridge/ciao";
import type { PluginLogger } from "openclaw/plugin-sdk/plugin-entry";
import { isTruthyEnvValue } from "openclaw/plugin-sdk/runtime-env";
import { classifyCiaoProcessError, type CiaoProcessErrorClassification } from "./ciao.js";
import { classifyCiaoProcessError } from "./ciao.js";
import { formatBonjourError } from "./errors.js";
type GatewayBonjourAdvertiser = {
@@ -25,12 +25,7 @@ type GatewayBonjourAdvertiseOpts = {
minimal?: boolean;
};
type BonjourCycle = Array<{ label: string; svc: CiaoService }>;
type ServiceStateTracker = {
state: CiaoService["serviceState"];
sinceMs: number;
};
type BonjourServices = Array<{ label: string; svc: CiaoService }>;
type ConsoleLogFn = (...args: unknown[]) => void;
type UncaughtExceptionHandler = (error: unknown) => boolean;
@@ -42,18 +37,6 @@ type BonjourAdvertiserDeps = {
registerUnhandledRejectionHandler: (handler: UnhandledRejectionHandler) => () => void;
};
const WATCHDOG_INTERVAL_MS = 5_000;
const REPAIR_DEBOUNCE_MS = 30_000;
const CONFLICT_SETTLE_MS = 30_000;
// LAN announce typically takes 12-13s on Mac/iOS. A 20s threshold avoids false-positive
// restart teardowns while still catching advertisers that never complete.
// See https://github.com/openclaw/openclaw/issues/72481
const STUCK_ANNOUNCING_MS = 20_000;
const MAX_CONSECUTIVE_RESTARTS = 3;
const MAX_CONSECUTIVE_STUCK_STATE_RESTARTS = 1;
// Bound total restarts because flapping can briefly reset the consecutive counter.
const RESTART_WINDOW_MS = 30 * 60_000;
const MAX_RESTARTS_IN_WINDOW = 5;
const CIAO_SELF_PROBE_RETRY_FRAGMENT =
"failed probing with reason: Error: Can't probe for a service which is announced already.";
@@ -210,12 +193,6 @@ export async function startGatewayBonjourAdvertiser(
if (isDisabledByEnv()) {
return { stop: async () => {} };
}
const announcedState = "announced" as CiaoService["serviceState"];
const activeStates = new Set<CiaoService["serviceState"]>([
announcedState,
"announcing" as CiaoService["serviceState"],
"probing" as CiaoService["serviceState"],
]);
const logger = {
info: deps.logger?.info ?? defaultLogger.info,
@@ -223,7 +200,6 @@ export async function startGatewayBonjourAdvertiser(
debug: deps.logger?.debug ?? defaultLogger.debug,
};
let restoreConsoleLog: () => void = () => {};
let requestCiaoRecovery: ((classification: CiaoProcessErrorClassification) => void) | undefined;
let cleanupUnhandledRejection: (() => void) | undefined;
let cleanupUncaughtException: (() => void) | undefined;
let processHandlersCleaned = false;
@@ -255,7 +231,6 @@ export async function startGatewayBonjourAdvertiser(
);
} else {
logger.warn(`bonjour: suppressing ciao netmask assertion: ${classification.formatted}`);
requestCiaoRecovery?.(classification);
}
return true;
};
@@ -310,8 +285,8 @@ export async function startGatewayBonjourAdvertiser(
const responder = getResponder();
function createCycle(): BonjourCycle {
const services: BonjourCycle = [];
function createServices(): BonjourServices {
const services: BonjourServices = [];
const gateway = responder.createService({
name: safeServiceName(instanceName),
@@ -329,14 +304,8 @@ export async function startGatewayBonjourAdvertiser(
return services;
}
async function stopCycle(
cycle: BonjourCycle | null,
optsValue?: { shutdownResponder?: boolean },
) {
if (!cycle) {
return;
}
for (const { svc } of cycle) {
async function stopServices(services: BonjourServices) {
for (const { svc } of services) {
try {
await svc.destroy();
} catch {
@@ -344,25 +313,21 @@ export async function startGatewayBonjourAdvertiser(
}
}
try {
if (optsValue?.shutdownResponder) {
await responder.shutdown();
}
await responder.shutdown();
} catch {
/* ignore */
}
}
function attachConflictListeners(services: BonjourCycle) {
function attachConflictListeners(services: BonjourServices) {
for (const { label, svc } of services) {
try {
svc.on("name-change", (name) => {
markConflictObserved(label, svc);
logger.warn(
`bonjour: ${label} name conflict resolved; newName=${JSON.stringify(name)}`,
);
});
svc.on("hostname-change", (nextHostname) => {
markConflictObserved(label, svc);
logger.warn(
`bonjour: ${label} hostname conflict resolved; newHostname=${JSON.stringify(nextHostname)}`,
);
@@ -387,7 +352,6 @@ export async function startGatewayBonjourAdvertiser(
svc,
)}): ${classification.formatted}`,
);
requestCiaoRecovery?.(classification);
return;
}
logger.warn(
@@ -395,7 +359,7 @@ export async function startGatewayBonjourAdvertiser(
);
}
function startAdvertising(services: BonjourCycle) {
function startAdvertising(services: BonjourServices) {
for (const { label, svc } of services) {
try {
void svc
@@ -418,184 +382,19 @@ export async function startGatewayBonjourAdvertiser(
)}, gatewayPort=${opts.gatewayPort}${opts.minimal ? ", minimal=true" : `, sshPort=${opts.sshPort ?? 22}`})`,
);
let stopped = false;
let recreatePromise: Promise<void> | null = null;
let disabled = false;
let consecutiveRestarts = 0;
let consecutiveStuckStateRestarts = 0;
const restartTimestamps: number[] = [];
let cycle: BonjourCycle | null = createCycle();
const stateTracker = new Map<string, ServiceStateTracker>();
const conflictTracker = new Map<string, number>();
const markConflictObserved = (label: string, svc: CiaoService) => {
const now = Date.now();
conflictTracker.set(label, now);
stateTracker.set(label, { state: svc.serviceState, sinceMs: now });
};
const updateStateTrackers = (services: BonjourCycle) => {
const now = Date.now();
for (const { label, svc } of services) {
const nextState = svc.serviceState;
const current = stateTracker.get(label);
const nextEnteredAt =
current && current.state !== announcedState && nextState !== announcedState
? current.sinceMs
: now;
if (!current || current.state !== nextState || current.sinceMs !== nextEnteredAt) {
stateTracker.set(label, { state: nextState, sinceMs: nextEnteredAt });
}
}
};
const recreateAdvertiser = async (reason: string, optsLocal?: { stuckState?: boolean }) => {
if (stopped || disabled) {
return;
}
if (recreatePromise) {
return recreatePromise;
}
recreatePromise = (async () => {
consecutiveRestarts += 1;
consecutiveStuckStateRestarts = optsLocal?.stuckState
? consecutiveStuckStateRestarts + 1
: 0;
const now = Date.now();
while (
restartTimestamps.length > 0 &&
now - (restartTimestamps[0] ?? 0) > RESTART_WINDOW_MS
) {
restartTimestamps.shift();
}
restartTimestamps.push(now);
const tooManyConsecutive = consecutiveRestarts > MAX_CONSECUTIVE_RESTARTS;
const tooManyStuckStates =
consecutiveStuckStateRestarts > MAX_CONSECUTIVE_STUCK_STATE_RESTARTS;
const tooManyInWindow = restartTimestamps.length >= MAX_RESTARTS_IN_WINDOW;
if (tooManyConsecutive || tooManyStuckStates || tooManyInWindow) {
disabled = true;
const detail = tooManyConsecutive
? `${MAX_CONSECUTIVE_RESTARTS} failed restarts`
: tooManyStuckStates
? `${MAX_CONSECUTIVE_STUCK_STATE_RESTARTS} stuck-state restart`
: `${MAX_RESTARTS_IN_WINDOW} restarts within ${Math.round(
RESTART_WINDOW_MS / 60_000,
)} minutes`;
logger.warn(
`bonjour: disabling advertiser after ${detail} (${reason}); set discovery.mdns.mode="off" or OPENCLAW_DISABLE_BONJOUR=1 to disable mDNS discovery`,
);
const previous = cycle;
cycle = null;
stateTracker.clear();
conflictTracker.clear();
await stopCycle(previous, { shutdownResponder: true });
restoreConsoleLog();
return;
}
logger.warn(`bonjour: restarting advertiser (${reason})`);
const previous = cycle;
await stopCycle(previous);
cycle = createCycle();
stateTracker.clear();
conflictTracker.clear();
attachConflictListeners(cycle);
startAdvertising(cycle);
})().finally(() => {
recreatePromise = null;
});
return recreatePromise;
};
requestCiaoRecovery = (classification) => {
void recreateAdvertiser(`ciao ${classification.kind}: ${classification.formatted}`);
};
attachConflictListeners(cycle);
startAdvertising(cycle);
const lastRepairAttempt = new Map<string, number>();
const watchdog = setInterval(() => {
if (stopped || recreatePromise) {
return;
}
if (disabled || !cycle) {
return;
}
updateStateTrackers(cycle);
for (const { label, svc } of cycle) {
const now = Date.now();
const state = svc.serviceState;
if (state === announcedState) {
consecutiveRestarts = 0;
consecutiveStuckStateRestarts = 0;
conflictTracker.delete(label);
}
const lastConflictAt = conflictTracker.get(label);
if (lastConflictAt !== undefined && now - lastConflictAt >= CONFLICT_SETTLE_MS) {
conflictTracker.delete(label);
}
if (lastConflictAt !== undefined && now - lastConflictAt < CONFLICT_SETTLE_MS) {
continue;
}
const tracked = stateTracker.get(label);
if (state !== announcedState && tracked && now - tracked.sinceMs >= STUCK_ANNOUNCING_MS) {
void recreateAdvertiser(
`service stuck in ${state} for ${now - tracked.sinceMs}ms (${serviceSummary(
label,
svc,
)})`,
{ stuckState: true },
);
return;
}
if (activeStates.has(state)) {
continue;
}
let key = label;
try {
key = `${label}:${svc.getFQDN()}`;
} catch {
// ignore
}
const last = lastRepairAttempt.get(key) ?? 0;
if (now - last < REPAIR_DEBOUNCE_MS) {
continue;
}
lastRepairAttempt.set(key, now);
logger.warn(
`bonjour: watchdog detected non-announced service; attempting re-advertise (${serviceSummary(
label,
svc,
)})`,
);
try {
void svc.advertise().catch((err: unknown) => {
logger.warn(
`bonjour: watchdog re-advertise failed (${serviceSummary(label, svc)}): ${formatBonjourError(err)}`,
);
});
} catch (err) {
logger.warn(
`bonjour: watchdog re-advertise threw (${serviceSummary(label, svc)}): ${formatBonjourError(err)}`,
);
}
}
}, WATCHDOG_INTERVAL_MS);
watchdog.unref?.();
const services = createServices();
attachConflictListeners(services);
startAdvertising(services);
let stopPromise: Promise<void> | null = null;
return {
stop: async () => {
stopped = true;
clearInterval(watchdog);
try {
await recreatePromise;
} catch {
// ignore
}
await stopCycle(cycle, { shutdownResponder: true });
restoreConsoleLog();
cleanupProcessHandlers();
stop: () => {
stopPromise ??= (async () => {
await stopServices(services);
restoreConsoleLog();
cleanupProcessHandlers();
})();
return stopPromise;
},
};
} catch (err) {