mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix: prevent Bonjour probe retry loops (#107710)
Prevents duplicate Bonjour advertisement attempts by delegating probing, conflict resolution, retries, and interface republishing to ciao.
Related: #98448
Prepared head SHA: 4bd1638acf
Co-authored-by: Shakker <165377636+shakkernerd@users.noreply.github.com>
Reviewed-by: @shakkernerd
This commit is contained in:
@@ -114,12 +114,12 @@ If browsing works but resolving fails, you're usually hitting a LAN policy or mD
|
||||
The gateway writes a rolling log file (printed on startup as `gateway log file: ...`). Look for `bonjour:` lines, especially:
|
||||
|
||||
- `bonjour: advertise failed ...`
|
||||
- `bonjour: suppressing ciao cancellation ...`
|
||||
- `bonjour: suppressing ciao netmask assertion ...`
|
||||
- `bonjour: ... name conflict resolved` / `hostname conflict resolved`
|
||||
- `bonjour: watchdog detected non-announced service ...`
|
||||
- `bonjour: disabling advertiser after ... failed restarts ...`
|
||||
|
||||
The watchdog treats active `probing`, `announcing`, and fresh conflict-renames as in-progress states. If the service never reaches `announced`, OpenClaw recreates the advertiser and, after repeated failures, disables Bonjour for that gateway process instead of re-advertising forever.
|
||||
OpenClaw starts each Bonjour service once and leaves probing, retry, name-conflict resolution, and interface-change republishing to the mDNS responder. This avoids overlapping publish attempts during normal network churn. Repeated internal self-probe messages are suppressed so they cannot flood the gateway log.
|
||||
|
||||
When multiple OpenClaw gateways advertise from the same host, Bonjour may append suffixes such as `(2)` or `(3)` to keep service instance names unique. Those suffixes are normal conflict resolution and do not indicate duplicate OCM supervision.
|
||||
|
||||
Bonjour uses the system hostname for the advertised `.local` host when it's a valid DNS label. If the system hostname contains spaces, underscores, or another invalid DNS-label character, OpenClaw falls back to `openclaw.local`. Set `OPENCLAW_MDNS_HOSTNAME=<name>` before starting the gateway when you need an explicit host label.
|
||||
|
||||
@@ -202,13 +202,13 @@ If a node no longer auto-discovers the gateway after Docker setup:
|
||||
dns-sd -B _openclaw-gw._tcp local.
|
||||
```
|
||||
|
||||
If browsing is empty, or Gateway logs show repeated ciao watchdog cancellations, restore `OPENCLAW_DISABLE_BONJOUR=1` and use a direct or Tailnet route.
|
||||
If browsing is empty, or Gateway logs show repeated ciao probe failures, restore `OPENCLAW_DISABLE_BONJOUR=1` and use a direct or Tailnet route.
|
||||
|
||||
## Common failure modes
|
||||
|
||||
- **Bonjour doesn't cross networks**: use Tailnet or SSH.
|
||||
- **Multicast blocked**: some Wi-Fi networks disable mDNS.
|
||||
- **Advertiser stuck in probing/announcing**: hosts with blocked multicast, container bridges, WSL, or interface churn can leave the ciao advertiser in a non-announced state. OpenClaw retries a few times, then disables Bonjour for the current gateway process instead of restarting the advertiser forever.
|
||||
- **Advertiser stuck in probing/announcing**: hosts with blocked multicast, container bridges, WSL, or interface churn can leave the responder in a non-announced state. The gateway remains available through direct, SSH, Tailnet, or wide-area DNS-SD routes; disable LAN Bonjour with `discovery.mdns.mode: "off"` or `OPENCLAW_DISABLE_BONJOUR=1` when multicast is unavailable.
|
||||
- **Docker bridge networking**: Bonjour auto-disables in detected containers. Set `OPENCLAW_DISABLE_BONJOUR=0` only for host, macvlan, or another mDNS-capable network.
|
||||
- **Sleep/interface churn**: macOS may temporarily drop mDNS results; retry.
|
||||
- **Browse works but resolve fails**: keep machine names simple (avoid emojis or punctuation), then restart the gateway. The service instance name derives from the host name, so overly complex names can confuse some resolvers.
|
||||
|
||||
@@ -54,10 +54,10 @@ function mockCall(mock: ReturnType<typeof vi.fn>, index = 0): unknown[] {
|
||||
|
||||
function enableAdvertiserUnitMode(hostname = "test-host") {
|
||||
// Allow advertiser to run in unit tests.
|
||||
delete process.env.VITEST;
|
||||
process.env.NODE_ENV = "development";
|
||||
vi.stubEnv("VITEST", undefined);
|
||||
vi.stubEnv("NODE_ENV", "development");
|
||||
vi.spyOn(os, "hostname").mockReturnValue(hostname);
|
||||
process.env.OPENCLAW_MDNS_HOSTNAME = hostname;
|
||||
vi.stubEnv("OPENCLAW_MDNS_HOSTNAME", hostname);
|
||||
}
|
||||
|
||||
function mockCiaoService(params?: {
|
||||
@@ -134,18 +134,7 @@ describe("gateway bonjour advertiser", () => {
|
||||
txt?: unknown;
|
||||
};
|
||||
|
||||
const prevEnv = { ...process.env };
|
||||
|
||||
afterEach(() => {
|
||||
for (const key of Object.keys(process.env)) {
|
||||
if (!(key in prevEnv)) {
|
||||
delete process.env[key];
|
||||
}
|
||||
}
|
||||
for (const [key, value] of Object.entries(prevEnv)) {
|
||||
process.env[key] = value;
|
||||
}
|
||||
|
||||
createService.mockClear();
|
||||
getResponder.mockReset();
|
||||
shutdown.mockClear();
|
||||
@@ -155,6 +144,7 @@ describe("gateway bonjour advertiser", () => {
|
||||
logger.warn.mockClear();
|
||||
logger.debug.mockClear();
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllEnvs();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
@@ -235,7 +225,7 @@ describe("gateway bonjour advertiser", () => {
|
||||
|
||||
it("honors truthy OPENCLAW_DISABLE_BONJOUR values", async () => {
|
||||
enableAdvertiserUnitMode();
|
||||
process.env.OPENCLAW_DISABLE_BONJOUR = "true";
|
||||
vi.stubEnv("OPENCLAW_DISABLE_BONJOUR", "true");
|
||||
|
||||
const started = await startAdvertiser({
|
||||
gatewayPort: 18789,
|
||||
@@ -261,8 +251,8 @@ describe("gateway bonjour advertiser", () => {
|
||||
|
||||
it("auto-disables Bonjour on Fly Machines without Docker sentinel files", async () => {
|
||||
enableAdvertiserUnitMode();
|
||||
process.env.FLY_MACHINE_ID = "3d8d5459a03038";
|
||||
process.env.FLY_APP_NAME = "openclaw-clawcks-test";
|
||||
vi.stubEnv("FLY_MACHINE_ID", "3d8d5459a03038");
|
||||
vi.stubEnv("FLY_APP_NAME", "openclaw-clawcks-test");
|
||||
vi.spyOn(fs, "existsSync").mockReturnValue(false);
|
||||
vi.spyOn(fs, "readFileSync").mockReturnValue("10:cpuset:/\n9:perf_event:/\n8:memory:/\n0::/\n");
|
||||
|
||||
@@ -277,7 +267,7 @@ describe("gateway bonjour advertiser", () => {
|
||||
|
||||
it("honors explicit Bonjour opt-in inside detected containers", async () => {
|
||||
enableAdvertiserUnitMode();
|
||||
process.env.OPENCLAW_DISABLE_BONJOUR = "0";
|
||||
vi.stubEnv("OPENCLAW_DISABLE_BONJOUR", "0");
|
||||
vi.spyOn(fs, "existsSync").mockImplementation((filePath) => String(filePath) === "/.dockerenv");
|
||||
|
||||
const destroy = vi.fn().mockResolvedValue(undefined);
|
||||
@@ -383,15 +373,12 @@ describe("gateway bonjour advertiser", () => {
|
||||
await started.stop();
|
||||
});
|
||||
|
||||
it("logs advertise failures and retries via watchdog", async () => {
|
||||
it("logs advertise failures without starting a competing retry loop", async () => {
|
||||
enableAdvertiserUnitMode();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const destroy = vi.fn().mockResolvedValue(undefined);
|
||||
const advertise = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("boom")) // initial advertise fails
|
||||
.mockResolvedValue(undefined); // watchdog retry succeeds
|
||||
const advertise = vi.fn().mockRejectedValue(new Error("boom"));
|
||||
mockCiaoService({ advertise, destroy, serviceState: "unannounced" });
|
||||
|
||||
const started = await startAdvertiser({
|
||||
@@ -406,16 +393,14 @@ describe("gateway bonjour advertiser", () => {
|
||||
await Promise.resolve();
|
||||
expectWarnContaining("advertise failed");
|
||||
|
||||
// watchdog first retries, then recreates the advertiser after the service
|
||||
// stays unhealthy across multiple 5s ticks.
|
||||
await vi.advanceTimersByTimeAsync(25_000);
|
||||
expect(advertise).toHaveBeenCalledTimes(3);
|
||||
expect(createService).toHaveBeenCalledTimes(2);
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(advertise).toHaveBeenCalledTimes(1);
|
||||
expect(createService).toHaveBeenCalledTimes(1);
|
||||
|
||||
await started.stop();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(advertise).toHaveBeenCalledTimes(3);
|
||||
expect(advertise).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("handles advertise throwing synchronously", async () => {
|
||||
@@ -529,132 +514,13 @@ describe("gateway bonjour advertiser", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("recreates the advertiser when ciao gets stuck announcing", async () => {
|
||||
it("never overlaps ciao lifecycle states or conflict handling with another advertise call", async () => {
|
||||
enableAdvertiserUnitMode();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const stateRef = { value: "announcing" };
|
||||
const events: string[] = [];
|
||||
const cleanupException = vi.fn();
|
||||
const cleanupRejection = vi.fn();
|
||||
let advertiseCount = 0;
|
||||
const destroy = vi.fn().mockImplementation(async () => {
|
||||
events.push("destroy");
|
||||
});
|
||||
const advertise = vi.fn().mockImplementation(() => {
|
||||
advertiseCount += 1;
|
||||
events.push(`advertise:${advertiseCount}`);
|
||||
if (advertiseCount === 1) {
|
||||
stateRef.value = "announcing";
|
||||
return new Promise<void>(() => {});
|
||||
}
|
||||
stateRef.value = "announced";
|
||||
return Promise.resolve();
|
||||
});
|
||||
mockCiaoService({ advertise, destroy, stateRef });
|
||||
registerUncaughtExceptionHandler.mockImplementation(() => cleanupException);
|
||||
registerUnhandledRejectionHandler.mockImplementation(() => cleanupRejection);
|
||||
|
||||
const started = await startAdvertiser({
|
||||
gatewayPort: 18789,
|
||||
sshPort: 2222,
|
||||
});
|
||||
|
||||
expect(createService).toHaveBeenCalledTimes(1);
|
||||
expect(advertise).toHaveBeenCalledTimes(1);
|
||||
expect(registerUncaughtExceptionHandler).toHaveBeenCalledTimes(1);
|
||||
expect(registerUnhandledRejectionHandler).toHaveBeenCalledTimes(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(25_000);
|
||||
|
||||
expectWarnContaining("restarting advertiser");
|
||||
expect(createService).toHaveBeenCalledTimes(2);
|
||||
expect(advertise).toHaveBeenCalledTimes(2);
|
||||
expect(destroy).toHaveBeenCalledTimes(1);
|
||||
expect(shutdown).not.toHaveBeenCalled();
|
||||
expect(cleanupException).not.toHaveBeenCalled();
|
||||
expect(cleanupRejection).not.toHaveBeenCalled();
|
||||
expect(events).toEqual(["advertise:1", "destroy", "advertise:2"]);
|
||||
|
||||
await started.stop();
|
||||
expect(destroy).toHaveBeenCalledTimes(2);
|
||||
expect(shutdown).toHaveBeenCalledTimes(1);
|
||||
expect(cleanupException).toHaveBeenCalledTimes(1);
|
||||
expect(cleanupRejection).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("treats probing-to-announcing churn as one unhealthy window", async () => {
|
||||
enableAdvertiserUnitMode();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const stateRef = { value: "probing" };
|
||||
const stateRef = { value: "unannounced" };
|
||||
const destroy = vi.fn().mockResolvedValue(undefined);
|
||||
const advertise = vi.fn().mockResolvedValue(undefined);
|
||||
mockCiaoService({ advertise, destroy, stateRef });
|
||||
|
||||
const started = await startAdvertiser({
|
||||
gatewayPort: 18789,
|
||||
sshPort: 2222,
|
||||
});
|
||||
|
||||
expect(createService).toHaveBeenCalledTimes(1);
|
||||
expect(advertise).toHaveBeenCalledTimes(1);
|
||||
|
||||
setTimeout(() => {
|
||||
stateRef.value = "announcing";
|
||||
}, 10_000);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(25_000);
|
||||
|
||||
expectWarnContaining("service stuck in announcing");
|
||||
expect(createService).toHaveBeenCalledTimes(2);
|
||||
expect(advertise).toHaveBeenCalledTimes(2);
|
||||
expect(destroy).toHaveBeenCalledTimes(1);
|
||||
expect(shutdown).not.toHaveBeenCalled();
|
||||
|
||||
await started.stop();
|
||||
expect(shutdown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not re-advertise while ciao is still probing", async () => {
|
||||
enableAdvertiserUnitMode();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const stateRef = { value: "probing" };
|
||||
const destroy = vi.fn().mockResolvedValue(undefined);
|
||||
const advertise = vi.fn().mockResolvedValue(undefined);
|
||||
mockCiaoService({ advertise, destroy, stateRef });
|
||||
|
||||
const started = await startAdvertiser({
|
||||
gatewayPort: 18789,
|
||||
sshPort: 2222,
|
||||
});
|
||||
|
||||
expect(advertise).toHaveBeenCalledTimes(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(15_000);
|
||||
|
||||
expect(advertise).toHaveBeenCalledTimes(1);
|
||||
expect(createService).toHaveBeenCalledTimes(1);
|
||||
expect(warnMessages().join("\n")).not.toContain(
|
||||
"watchdog detected non-announced service; attempting re-advertise",
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
|
||||
expectWarnContaining("service stuck in probing");
|
||||
expect(createService).toHaveBeenCalledTimes(2);
|
||||
|
||||
await started.stop();
|
||||
});
|
||||
|
||||
it("defers probing recovery while a name conflict is still settling", async () => {
|
||||
enableAdvertiserUnitMode();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const stateRef = { value: "probing" };
|
||||
const destroy = vi.fn().mockResolvedValue(undefined);
|
||||
const advertise = vi.fn().mockResolvedValue(undefined);
|
||||
const advertise = vi.fn(() => new Promise<void>(() => {}));
|
||||
const listenerMap = new Map<string, (value: unknown) => void>();
|
||||
mockCiaoService({ advertise, destroy, stateRef, listenerMap });
|
||||
|
||||
@@ -663,101 +529,58 @@ describe("gateway bonjour advertiser", () => {
|
||||
sshPort: 2222,
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
listenerMap.get("name-change")?.("test-host (OpenClaw) (2)");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(15_000);
|
||||
|
||||
expect(createService).toHaveBeenCalledTimes(1);
|
||||
expect(advertise).toHaveBeenCalledTimes(1);
|
||||
|
||||
for (const state of ["probing", "announcing", "unannounced", "probed", "announced"]) {
|
||||
stateRef.value = state;
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
}
|
||||
listenerMap.get("name-change")?.("test-host (OpenClaw) (2)");
|
||||
listenerMap.get("hostname-change")?.("test-host-(2)");
|
||||
expectWarnContaining('name conflict resolved; newName="test-host (OpenClaw) (2)"');
|
||||
|
||||
await vi.advanceTimersByTimeAsync(20_000);
|
||||
|
||||
expectWarnContaining("service stuck in probing");
|
||||
expect(createService).toHaveBeenCalledTimes(2);
|
||||
|
||||
await started.stop();
|
||||
});
|
||||
|
||||
it("disables bonjour for the process after repeated stuck advertiser restarts", async () => {
|
||||
enableAdvertiserUnitMode();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const stateRef = { value: "announcing" };
|
||||
const destroy = vi.fn().mockResolvedValue(undefined);
|
||||
const advertise = vi.fn(() => new Promise<void>(() => {}));
|
||||
mockCiaoService({ advertise, destroy, stateRef });
|
||||
|
||||
const started = await startAdvertiser({
|
||||
gatewayPort: 18789,
|
||||
sshPort: 2222,
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(55_000);
|
||||
|
||||
expectWarnContaining("disabling advertiser after 1 stuck-state restart");
|
||||
expect(createService).toHaveBeenCalledTimes(2);
|
||||
expect(advertise).toHaveBeenCalledTimes(2);
|
||||
expect(destroy).toHaveBeenCalledTimes(2);
|
||||
expect(shutdown).toHaveBeenCalledTimes(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(createService).toHaveBeenCalledTimes(2);
|
||||
expect(advertise).toHaveBeenCalledTimes(2);
|
||||
expectWarnContaining('hostname conflict resolved; newHostname="test-host-(2)"');
|
||||
expect(createService).toHaveBeenCalledTimes(1);
|
||||
expect(advertise).toHaveBeenCalledTimes(1);
|
||||
expect(destroy).not.toHaveBeenCalled();
|
||||
expect(shutdown).not.toHaveBeenCalled();
|
||||
expect(warnMessages().join("\n")).not.toMatch(
|
||||
/watchdog|restarting advertiser|disabling advertiser/,
|
||||
);
|
||||
|
||||
await started.stop();
|
||||
expect(destroy).toHaveBeenCalledTimes(1);
|
||||
expect(shutdown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("disables bonjour when the advertiser flaps within a sliding window", async () => {
|
||||
it("makes advertiser shutdown idempotent", async () => {
|
||||
enableAdvertiserUnitMode();
|
||||
vi.useFakeTimers();
|
||||
|
||||
const stateRef = { value: "announced" };
|
||||
const destroy = vi.fn().mockResolvedValue(undefined);
|
||||
const advertise = vi.fn().mockResolvedValue(undefined);
|
||||
mockCiaoService({ advertise, destroy, stateRef });
|
||||
const cleanupException = vi.fn();
|
||||
const cleanupRejection = vi.fn();
|
||||
mockCiaoService({ advertise, destroy });
|
||||
registerUncaughtExceptionHandler.mockImplementation(() => cleanupException);
|
||||
registerUnhandledRejectionHandler.mockImplementation(() => cleanupRejection);
|
||||
|
||||
const started = await startAdvertiser({
|
||||
gatewayPort: 18789,
|
||||
sshPort: 2222,
|
||||
});
|
||||
|
||||
for (let cycle = 0; cycle < 12; cycle += 1) {
|
||||
stateRef.value = "announced";
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
stateRef.value = "probing";
|
||||
await vi.advanceTimersByTimeAsync(25_000);
|
||||
if (
|
||||
logger.warn.mock.calls.some(
|
||||
(call) => typeof call[0] === "string" && call[0].includes("disabling advertiser after"),
|
||||
)
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
await Promise.all([started.stop(), started.stop()]);
|
||||
|
||||
const disableLog = logger.warn.mock.calls.find(
|
||||
(call) => typeof call[0] === "string" && call[0].includes("disabling advertiser after"),
|
||||
);
|
||||
if (!disableLog) {
|
||||
throw new Error("expected advertiser disable warning after repeated restarts");
|
||||
}
|
||||
expect(String(disableLog[0])).toMatch(/restarts within \d+ minutes/);
|
||||
|
||||
const advertiseCallsAtDisable = advertise.mock.calls.length;
|
||||
const createServiceCallsAtDisable = createService.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(5 * 60_000);
|
||||
expect(advertise).toHaveBeenCalledTimes(advertiseCallsAtDisable);
|
||||
expect(createService).toHaveBeenCalledTimes(createServiceCallsAtDisable);
|
||||
|
||||
await started.stop();
|
||||
expect(destroy).toHaveBeenCalledTimes(1);
|
||||
expect(shutdown).toHaveBeenCalledTimes(1);
|
||||
expect(cleanupException).toHaveBeenCalledTimes(1);
|
||||
expect(cleanupRejection).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("normalizes hostnames with domains for service names", async () => {
|
||||
// Allow advertiser to run in unit tests.
|
||||
delete process.env.VITEST;
|
||||
process.env.NODE_ENV = "development";
|
||||
vi.stubEnv("VITEST", undefined);
|
||||
vi.stubEnv("NODE_ENV", "development");
|
||||
|
||||
vi.spyOn(os, "hostname").mockReturnValue("Mac.localdomain");
|
||||
|
||||
@@ -781,9 +604,9 @@ describe("gateway bonjour advertiser", () => {
|
||||
|
||||
it("falls back to openclaw when system hostname is invalid for DNS", async () => {
|
||||
// Allow advertiser to run in unit tests.
|
||||
delete process.env.VITEST;
|
||||
process.env.NODE_ENV = "development";
|
||||
delete process.env.OPENCLAW_MDNS_HOSTNAME;
|
||||
vi.stubEnv("VITEST", undefined);
|
||||
vi.stubEnv("NODE_ENV", "development");
|
||||
vi.stubEnv("OPENCLAW_MDNS_HOSTNAME", undefined);
|
||||
vi.spyOn(os, "hostname").mockReturnValue("My_Lobster Host");
|
||||
|
||||
const destroy = vi.fn().mockResolvedValue(undefined);
|
||||
@@ -877,9 +700,9 @@ describe("gateway bonjour advertiser", () => {
|
||||
|
||||
it("uses system hostname when OPENCLAW_MDNS_HOSTNAME is unset", async () => {
|
||||
// Allow advertiser to run in unit tests.
|
||||
delete process.env.VITEST;
|
||||
process.env.NODE_ENV = "development";
|
||||
delete process.env.OPENCLAW_MDNS_HOSTNAME;
|
||||
vi.stubEnv("VITEST", undefined);
|
||||
vi.stubEnv("NODE_ENV", "development");
|
||||
vi.stubEnv("OPENCLAW_MDNS_HOSTNAME", undefined);
|
||||
vi.spyOn(os, "hostname").mockReturnValue("Lobster");
|
||||
|
||||
const destroy = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
/** Publishes gateway/canvas/SSH records and repairs stuck or conflicting ciao advertisements. */
|
||||
/** Publishes gateway/canvas/SSH records through one ciao-owned advertisement lifecycle. */
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import type { CiaoService } from "@homebridge/ciao";
|
||||
import type { PluginLogger } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { isTruthyEnvValue } from "openclaw/plugin-sdk/runtime-env";
|
||||
import { classifyCiaoProcessError, type CiaoProcessErrorClassification } from "./ciao.js";
|
||||
import { classifyCiaoProcessError } from "./ciao.js";
|
||||
import { formatBonjourError } from "./errors.js";
|
||||
|
||||
type GatewayBonjourAdvertiser = {
|
||||
@@ -25,12 +25,7 @@ type GatewayBonjourAdvertiseOpts = {
|
||||
minimal?: boolean;
|
||||
};
|
||||
|
||||
type BonjourCycle = Array<{ label: string; svc: CiaoService }>;
|
||||
|
||||
type ServiceStateTracker = {
|
||||
state: CiaoService["serviceState"];
|
||||
sinceMs: number;
|
||||
};
|
||||
type BonjourServices = Array<{ label: string; svc: CiaoService }>;
|
||||
|
||||
type ConsoleLogFn = (...args: unknown[]) => void;
|
||||
type UncaughtExceptionHandler = (error: unknown) => boolean;
|
||||
@@ -42,18 +37,6 @@ type BonjourAdvertiserDeps = {
|
||||
registerUnhandledRejectionHandler: (handler: UnhandledRejectionHandler) => () => void;
|
||||
};
|
||||
|
||||
const WATCHDOG_INTERVAL_MS = 5_000;
|
||||
const REPAIR_DEBOUNCE_MS = 30_000;
|
||||
const CONFLICT_SETTLE_MS = 30_000;
|
||||
// LAN announce typically takes 12-13s on Mac/iOS. A 20s threshold avoids false-positive
|
||||
// restart teardowns while still catching advertisers that never complete.
|
||||
// See https://github.com/openclaw/openclaw/issues/72481
|
||||
const STUCK_ANNOUNCING_MS = 20_000;
|
||||
const MAX_CONSECUTIVE_RESTARTS = 3;
|
||||
const MAX_CONSECUTIVE_STUCK_STATE_RESTARTS = 1;
|
||||
// Bound total restarts because flapping can briefly reset the consecutive counter.
|
||||
const RESTART_WINDOW_MS = 30 * 60_000;
|
||||
const MAX_RESTARTS_IN_WINDOW = 5;
|
||||
const CIAO_SELF_PROBE_RETRY_FRAGMENT =
|
||||
"failed probing with reason: Error: Can't probe for a service which is announced already.";
|
||||
|
||||
@@ -210,12 +193,6 @@ export async function startGatewayBonjourAdvertiser(
|
||||
if (isDisabledByEnv()) {
|
||||
return { stop: async () => {} };
|
||||
}
|
||||
const announcedState = "announced" as CiaoService["serviceState"];
|
||||
const activeStates = new Set<CiaoService["serviceState"]>([
|
||||
announcedState,
|
||||
"announcing" as CiaoService["serviceState"],
|
||||
"probing" as CiaoService["serviceState"],
|
||||
]);
|
||||
|
||||
const logger = {
|
||||
info: deps.logger?.info ?? defaultLogger.info,
|
||||
@@ -223,7 +200,6 @@ export async function startGatewayBonjourAdvertiser(
|
||||
debug: deps.logger?.debug ?? defaultLogger.debug,
|
||||
};
|
||||
let restoreConsoleLog: () => void = () => {};
|
||||
let requestCiaoRecovery: ((classification: CiaoProcessErrorClassification) => void) | undefined;
|
||||
let cleanupUnhandledRejection: (() => void) | undefined;
|
||||
let cleanupUncaughtException: (() => void) | undefined;
|
||||
let processHandlersCleaned = false;
|
||||
@@ -255,7 +231,6 @@ export async function startGatewayBonjourAdvertiser(
|
||||
);
|
||||
} else {
|
||||
logger.warn(`bonjour: suppressing ciao netmask assertion: ${classification.formatted}`);
|
||||
requestCiaoRecovery?.(classification);
|
||||
}
|
||||
return true;
|
||||
};
|
||||
@@ -310,8 +285,8 @@ export async function startGatewayBonjourAdvertiser(
|
||||
|
||||
const responder = getResponder();
|
||||
|
||||
function createCycle(): BonjourCycle {
|
||||
const services: BonjourCycle = [];
|
||||
function createServices(): BonjourServices {
|
||||
const services: BonjourServices = [];
|
||||
|
||||
const gateway = responder.createService({
|
||||
name: safeServiceName(instanceName),
|
||||
@@ -329,14 +304,8 @@ export async function startGatewayBonjourAdvertiser(
|
||||
return services;
|
||||
}
|
||||
|
||||
async function stopCycle(
|
||||
cycle: BonjourCycle | null,
|
||||
optsValue?: { shutdownResponder?: boolean },
|
||||
) {
|
||||
if (!cycle) {
|
||||
return;
|
||||
}
|
||||
for (const { svc } of cycle) {
|
||||
async function stopServices(services: BonjourServices) {
|
||||
for (const { svc } of services) {
|
||||
try {
|
||||
await svc.destroy();
|
||||
} catch {
|
||||
@@ -344,25 +313,21 @@ export async function startGatewayBonjourAdvertiser(
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (optsValue?.shutdownResponder) {
|
||||
await responder.shutdown();
|
||||
}
|
||||
await responder.shutdown();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
|
||||
function attachConflictListeners(services: BonjourCycle) {
|
||||
function attachConflictListeners(services: BonjourServices) {
|
||||
for (const { label, svc } of services) {
|
||||
try {
|
||||
svc.on("name-change", (name) => {
|
||||
markConflictObserved(label, svc);
|
||||
logger.warn(
|
||||
`bonjour: ${label} name conflict resolved; newName=${JSON.stringify(name)}`,
|
||||
);
|
||||
});
|
||||
svc.on("hostname-change", (nextHostname) => {
|
||||
markConflictObserved(label, svc);
|
||||
logger.warn(
|
||||
`bonjour: ${label} hostname conflict resolved; newHostname=${JSON.stringify(nextHostname)}`,
|
||||
);
|
||||
@@ -387,7 +352,6 @@ export async function startGatewayBonjourAdvertiser(
|
||||
svc,
|
||||
)}): ${classification.formatted}`,
|
||||
);
|
||||
requestCiaoRecovery?.(classification);
|
||||
return;
|
||||
}
|
||||
logger.warn(
|
||||
@@ -395,7 +359,7 @@ export async function startGatewayBonjourAdvertiser(
|
||||
);
|
||||
}
|
||||
|
||||
function startAdvertising(services: BonjourCycle) {
|
||||
function startAdvertising(services: BonjourServices) {
|
||||
for (const { label, svc } of services) {
|
||||
try {
|
||||
void svc
|
||||
@@ -418,184 +382,19 @@ export async function startGatewayBonjourAdvertiser(
|
||||
)}, gatewayPort=${opts.gatewayPort}${opts.minimal ? ", minimal=true" : `, sshPort=${opts.sshPort ?? 22}`})`,
|
||||
);
|
||||
|
||||
let stopped = false;
|
||||
let recreatePromise: Promise<void> | null = null;
|
||||
let disabled = false;
|
||||
let consecutiveRestarts = 0;
|
||||
let consecutiveStuckStateRestarts = 0;
|
||||
const restartTimestamps: number[] = [];
|
||||
let cycle: BonjourCycle | null = createCycle();
|
||||
const stateTracker = new Map<string, ServiceStateTracker>();
|
||||
const conflictTracker = new Map<string, number>();
|
||||
|
||||
const markConflictObserved = (label: string, svc: CiaoService) => {
|
||||
const now = Date.now();
|
||||
conflictTracker.set(label, now);
|
||||
stateTracker.set(label, { state: svc.serviceState, sinceMs: now });
|
||||
};
|
||||
|
||||
const updateStateTrackers = (services: BonjourCycle) => {
|
||||
const now = Date.now();
|
||||
for (const { label, svc } of services) {
|
||||
const nextState = svc.serviceState;
|
||||
const current = stateTracker.get(label);
|
||||
const nextEnteredAt =
|
||||
current && current.state !== announcedState && nextState !== announcedState
|
||||
? current.sinceMs
|
||||
: now;
|
||||
if (!current || current.state !== nextState || current.sinceMs !== nextEnteredAt) {
|
||||
stateTracker.set(label, { state: nextState, sinceMs: nextEnteredAt });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const recreateAdvertiser = async (reason: string, optsLocal?: { stuckState?: boolean }) => {
|
||||
if (stopped || disabled) {
|
||||
return;
|
||||
}
|
||||
if (recreatePromise) {
|
||||
return recreatePromise;
|
||||
}
|
||||
recreatePromise = (async () => {
|
||||
consecutiveRestarts += 1;
|
||||
consecutiveStuckStateRestarts = optsLocal?.stuckState
|
||||
? consecutiveStuckStateRestarts + 1
|
||||
: 0;
|
||||
const now = Date.now();
|
||||
while (
|
||||
restartTimestamps.length > 0 &&
|
||||
now - (restartTimestamps[0] ?? 0) > RESTART_WINDOW_MS
|
||||
) {
|
||||
restartTimestamps.shift();
|
||||
}
|
||||
restartTimestamps.push(now);
|
||||
const tooManyConsecutive = consecutiveRestarts > MAX_CONSECUTIVE_RESTARTS;
|
||||
const tooManyStuckStates =
|
||||
consecutiveStuckStateRestarts > MAX_CONSECUTIVE_STUCK_STATE_RESTARTS;
|
||||
const tooManyInWindow = restartTimestamps.length >= MAX_RESTARTS_IN_WINDOW;
|
||||
if (tooManyConsecutive || tooManyStuckStates || tooManyInWindow) {
|
||||
disabled = true;
|
||||
const detail = tooManyConsecutive
|
||||
? `${MAX_CONSECUTIVE_RESTARTS} failed restarts`
|
||||
: tooManyStuckStates
|
||||
? `${MAX_CONSECUTIVE_STUCK_STATE_RESTARTS} stuck-state restart`
|
||||
: `${MAX_RESTARTS_IN_WINDOW} restarts within ${Math.round(
|
||||
RESTART_WINDOW_MS / 60_000,
|
||||
)} minutes`;
|
||||
logger.warn(
|
||||
`bonjour: disabling advertiser after ${detail} (${reason}); set discovery.mdns.mode="off" or OPENCLAW_DISABLE_BONJOUR=1 to disable mDNS discovery`,
|
||||
);
|
||||
const previous = cycle;
|
||||
cycle = null;
|
||||
stateTracker.clear();
|
||||
conflictTracker.clear();
|
||||
await stopCycle(previous, { shutdownResponder: true });
|
||||
restoreConsoleLog();
|
||||
return;
|
||||
}
|
||||
logger.warn(`bonjour: restarting advertiser (${reason})`);
|
||||
const previous = cycle;
|
||||
await stopCycle(previous);
|
||||
cycle = createCycle();
|
||||
stateTracker.clear();
|
||||
conflictTracker.clear();
|
||||
attachConflictListeners(cycle);
|
||||
startAdvertising(cycle);
|
||||
})().finally(() => {
|
||||
recreatePromise = null;
|
||||
});
|
||||
return recreatePromise;
|
||||
};
|
||||
requestCiaoRecovery = (classification) => {
|
||||
void recreateAdvertiser(`ciao ${classification.kind}: ${classification.formatted}`);
|
||||
};
|
||||
attachConflictListeners(cycle);
|
||||
startAdvertising(cycle);
|
||||
|
||||
const lastRepairAttempt = new Map<string, number>();
|
||||
const watchdog = setInterval(() => {
|
||||
if (stopped || recreatePromise) {
|
||||
return;
|
||||
}
|
||||
if (disabled || !cycle) {
|
||||
return;
|
||||
}
|
||||
updateStateTrackers(cycle);
|
||||
for (const { label, svc } of cycle) {
|
||||
const now = Date.now();
|
||||
const state = svc.serviceState;
|
||||
if (state === announcedState) {
|
||||
consecutiveRestarts = 0;
|
||||
consecutiveStuckStateRestarts = 0;
|
||||
conflictTracker.delete(label);
|
||||
}
|
||||
const lastConflictAt = conflictTracker.get(label);
|
||||
if (lastConflictAt !== undefined && now - lastConflictAt >= CONFLICT_SETTLE_MS) {
|
||||
conflictTracker.delete(label);
|
||||
}
|
||||
if (lastConflictAt !== undefined && now - lastConflictAt < CONFLICT_SETTLE_MS) {
|
||||
continue;
|
||||
}
|
||||
const tracked = stateTracker.get(label);
|
||||
if (state !== announcedState && tracked && now - tracked.sinceMs >= STUCK_ANNOUNCING_MS) {
|
||||
void recreateAdvertiser(
|
||||
`service stuck in ${state} for ${now - tracked.sinceMs}ms (${serviceSummary(
|
||||
label,
|
||||
svc,
|
||||
)})`,
|
||||
{ stuckState: true },
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (activeStates.has(state)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let key = label;
|
||||
try {
|
||||
key = `${label}:${svc.getFQDN()}`;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
const last = lastRepairAttempt.get(key) ?? 0;
|
||||
if (now - last < REPAIR_DEBOUNCE_MS) {
|
||||
continue;
|
||||
}
|
||||
lastRepairAttempt.set(key, now);
|
||||
|
||||
logger.warn(
|
||||
`bonjour: watchdog detected non-announced service; attempting re-advertise (${serviceSummary(
|
||||
label,
|
||||
svc,
|
||||
)})`,
|
||||
);
|
||||
try {
|
||||
void svc.advertise().catch((err: unknown) => {
|
||||
logger.warn(
|
||||
`bonjour: watchdog re-advertise failed (${serviceSummary(label, svc)}): ${formatBonjourError(err)}`,
|
||||
);
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
`bonjour: watchdog re-advertise threw (${serviceSummary(label, svc)}): ${formatBonjourError(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}, WATCHDOG_INTERVAL_MS);
|
||||
watchdog.unref?.();
|
||||
const services = createServices();
|
||||
attachConflictListeners(services);
|
||||
startAdvertising(services);
|
||||
let stopPromise: Promise<void> | null = null;
|
||||
|
||||
return {
|
||||
stop: async () => {
|
||||
stopped = true;
|
||||
clearInterval(watchdog);
|
||||
try {
|
||||
await recreatePromise;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
await stopCycle(cycle, { shutdownResponder: true });
|
||||
restoreConsoleLog();
|
||||
cleanupProcessHandlers();
|
||||
stop: () => {
|
||||
stopPromise ??= (async () => {
|
||||
await stopServices(services);
|
||||
restoreConsoleLog();
|
||||
cleanupProcessHandlers();
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
};
|
||||
} catch (err) {
|
||||
|
||||
Reference in New Issue
Block a user