fix(scripts): retry auth alerts after ntfy rejects delivery (#110760)

* fix(scripts): retry auth alerts after delivery failure * test(scripts): cover mixed auth notification outcomes Co-authored-by: 杨浩宇0668001029 <yang.haoyu@xydigit.com> --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
2026-07-21 02:06:43 +00:00 · 2026-07-19 00:10:32 +01:00
parent 14a714901b
commit 2dce9d3012
2 changed files with 180 additions and 7 deletions
@@ -32,6 +32,7 @@ MIN_INTERVAL=3600
 send_notification() {
    local message="$1"
    local priority="${2:-default}"
+    local notification_sent=0

    echo "$(date '+%Y-%m-%d %H:%M:%S') - $message"

@@ -46,23 +47,30 @@ send_notification() {
        # Check if we can still use openclaw
        if "$SCRIPT_DIR/claude-auth-status.sh" simple 2>/dev/null | grep -q "OK\|EXPIRING"; then
            echo "Sending via OpenClaw to $NOTIFY_PHONE..."
-            openclaw send --to "$NOTIFY_PHONE" --message "$message" 2>/dev/null || true
+            if openclaw send --to "$NOTIFY_PHONE" --message "$message" 2>/dev/null; then
+                notification_sent=1
+            fi
        fi
    fi

    # Send via ntfy.sh if configured
    if [ -n "$NOTIFY_NTFY" ]; then
        echo "Sending via ntfy.sh to $NOTIFY_NTFY..."
-        curl -s --connect-timeout 5 --max-time 15 -o /dev/null \
+        if curl -fsS --connect-timeout 5 --max-time 15 -o /dev/null \
            -H "Title: OpenClaw Auth Alert" \
            -H "Priority: $priority" \
            -H "Tags: warning,key" \
            -d "$message" \
-            "https://ntfy.sh/$NOTIFY_NTFY" || true
+            "https://ntfy.sh/$NOTIFY_NTFY"; then
+            notification_sent=1
+        fi
    fi

-    # Update state
-    echo "$NOW" > "$STATE_FILE"
+    if [ "$notification_sent" -eq 1 ]; then
+        echo "$NOW" > "$STATE_FILE"
+    else
+        echo "No notification delivered; cooldown not updated" >&2
+    fi
 }

 # Check auth status
@@ -1,5 +1,8 @@
 // Auth monitor tests cover optional systemd and Termux helper script contracts.
-import { readFileSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { describe, expect, it } from "vitest";

 const AUTH_MONITOR_PATH = "scripts/auth-monitor.sh";
@@ -17,6 +20,83 @@ function readScript(path: string): string {
  return readFileSync(path, "utf8");
 }

+function createAuthMonitorHarness() {
+  const home = mkdtempSync(join(tmpdir(), "openclaw-auth-monitor-"));
+  const binDir = join(home, "bin");
+  const curlLog = join(home, "curl.log");
+  const openclawLog = join(home, "openclaw.log");
+  const stateFile = join(home, ".openclaw", "auth-monitor-state");
+  mkdirSync(binDir);
+  writeFileSync(
+    join(binDir, "curl"),
+    '#!/bin/sh\nprintf "called\\n" >> "$FAKE_CURL_LOG"\nexit "$FAKE_CURL_EXIT_CODE"\n',
+    { mode: 0o755 },
+  );
+  writeFileSync(
+    join(binDir, "openclaw"),
+    [
+      "#!/bin/sh",
+      'if [ "$1" = "models" ]; then',
+      "  exit 1",
+      "fi",
+      'printf "called\\n" >> "$FAKE_OPENCLAW_LOG"',
+      'exit "$FAKE_OPENCLAW_EXIT_CODE"',
+      "",
+    ].join("\n"),
+    { mode: 0o755 },
+  );
+
+  return {
+    curlLog,
+    home,
+    openclawLog,
+    stateFile,
+    cleanup: () => rmSync(home, { recursive: true, force: true }),
+    enablePhoneAuth: () => {
+      const expiresAt = Date.now() + 90 * 60 * 1000;
+      mkdirSync(join(home, ".claude"), { recursive: true });
+      mkdirSync(join(home, ".openclaw", "agents", "main", "agent"), { recursive: true });
+      writeFileSync(
+        join(home, ".claude", ".credentials.json"),
+        JSON.stringify({ claudeAiOauth: { expiresAt } }),
+      );
+      writeFileSync(
+        join(home, ".openclaw", "agents", "main", "agent", "auth-profiles.json"),
+        JSON.stringify({
+          profiles: { "anthropic:default": { expires: expiresAt, provider: "anthropic" } },
+        }),
+      );
+    },
+    run: ({
+      curlExitCode = 0,
+      notifyNtfy = "test-topic",
+      notifyPhone = "",
+      openclawExitCode = 0,
+    }: {
+      curlExitCode?: number;
+      notifyNtfy?: string;
+      notifyPhone?: string;
+      openclawExitCode?: number;
+    } = {}) =>
+      spawnSync("bash", [AUTH_MONITOR_PATH], {
+        cwd: process.cwd(),
+        encoding: "utf8",
+        env: {
+          ...process.env,
+          FAKE_CURL_EXIT_CODE: String(curlExitCode),
+          FAKE_CURL_LOG: curlLog,
+          FAKE_OPENCLAW_EXIT_CODE: String(openclawExitCode),
+          FAKE_OPENCLAW_LOG: openclawLog,
+          HOME: home,
+          NOTIFY_NTFY: notifyNtfy,
+          NOTIFY_PHONE: notifyPhone,
+          PATH: `${binDir}:${process.env.PATH ?? ""}`,
+          WARN_HOURS: "2",
+        },
+      }),
+  };
+}
+
 describe("auth monitoring scripts", () => {
  it("keeps systemd install rendering free of checked-in host paths", () => {
    const setup = readScript(SETUP_AUTH_SYSTEM_PATH);
@@ -53,7 +133,92 @@ describe("auth monitoring scripts", () => {
  it("bounds ntfy notification requests", () => {
    const script = readScript(AUTH_MONITOR_PATH);

-    expect(script).toContain("curl -s --connect-timeout 5 --max-time 15 -o /dev/null");
+    expect(script).toContain("curl -fsS --connect-timeout 5 --max-time 15 -o /dev/null");
+  });
+
+  it("retries after ntfy rejects a notification", () => {
+    const harness = createAuthMonitorHarness();
+
+    try {
+      const rejected = harness.run({ curlExitCode: 22 });
+      expect(rejected.status).toBe(1);
+      expect(existsSync(harness.stateFile)).toBe(false);
+      expect(rejected.stderr).toContain("No notification delivered; cooldown not updated");
+
+      const retry = harness.run();
+      expect(retry.stdout).toContain("Sending via ntfy.sh to test-topic...");
+      expect(retry.stdout).not.toContain("Skipping notification (sent recently)");
+      expect(readFileSync(harness.curlLog, "utf8").trim().split("\n")).toHaveLength(2);
+    } finally {
+      harness.cleanup();
+    }
+  });
+
+  it("rate-limits after ntfy accepts a notification", () => {
+    const harness = createAuthMonitorHarness();
+
+    try {
+      const accepted = harness.run();
+      expect(accepted.status).toBe(1);
+      expect(existsSync(harness.stateFile)).toBe(true);
+
+      const throttled = harness.run();
+      expect(throttled.stdout).toContain("Skipping notification (sent recently)");
+      expect(readFileSync(harness.curlLog, "utf8").trim().split("\n")).toHaveLength(1);
+    } finally {
+      harness.cleanup();
+    }
+  });
+
+  it("rate-limits after any configured notification channel succeeds", () => {
+    const harness = createAuthMonitorHarness();
+    harness.enablePhoneAuth();
+
+    try {
+      const delivered = harness.run({
+        curlExitCode: 22,
+        notifyPhone: "+15550000000",
+      });
+      expect(delivered.status).toBe(0);
+      expect(existsSync(harness.stateFile)).toBe(true);
+
+      const throttled = harness.run({
+        curlExitCode: 22,
+        notifyPhone: "+15550000000",
+      });
+      expect(throttled.stdout).toContain("Skipping notification (sent recently)");
+      expect(readFileSync(harness.openclawLog, "utf8").trim().split("\n")).toHaveLength(1);
+      expect(readFileSync(harness.curlLog, "utf8").trim().split("\n")).toHaveLength(1);
+    } finally {
+      harness.cleanup();
+    }
+  });
+
+  it("retries when all configured notification channels fail", () => {
+    const harness = createAuthMonitorHarness();
+    harness.enablePhoneAuth();
+
+    try {
+      const failed = harness.run({
+        curlExitCode: 22,
+        notifyPhone: "+15550000000",
+        openclawExitCode: 1,
+      });
+      expect(failed.status).toBe(0);
+      expect(existsSync(harness.stateFile)).toBe(false);
+      expect(failed.stderr).toContain("No notification delivered; cooldown not updated");
+
+      const retry = harness.run({
+        curlExitCode: 22,
+        notifyPhone: "+15550000000",
+        openclawExitCode: 1,
+      });
+      expect(retry.stdout).not.toContain("Skipping notification (sent recently)");
+      expect(readFileSync(harness.openclawLog, "utf8").trim().split("\n")).toHaveLength(2);
+      expect(readFileSync(harness.curlLog, "utf8").trim().split("\n")).toHaveLength(2);
+    } finally {
+      harness.cleanup();
+    }
  });

  it("keeps mobile reauth wired to local auth status and Claude token setup", () => {