mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(openai): harden codex catalog, routing opt-out, and WHAM half-open reprobe (#108683)
- Align OAuth model-discovery client_version with the pinned @openai/codex package (contract test ties it to extensions/codex/package.json). - Static OAuth fallback advertises only the proven GPT-5.6 subscription route (Sol); Terra/Luna come from live discovery when entitled. (#105445) - Honor the deprecated whole-agent agentRuntime openclaw opt-out for implicit Codex plugin installation; explicit model policy still wins. (#95131) - Half-open reprobe for long WHAM subscription blocks: bounded 45-minute background probes with durable claims, generation checks, and process dedupe so restored capacity unblocks profiles early. (#107262)
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
// Openai tests cover openai provider plugin behavior.
|
||||
import fs from "node:fs";
|
||||
import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
import type { Context, Model, SimpleStreamOptions } from "openclaw/plugin-sdk/llm";
|
||||
import {
|
||||
@@ -165,6 +166,19 @@ vi.mock("openclaw/plugin-sdk/provider-stream-family", async (importOriginal) =>
|
||||
};
|
||||
});
|
||||
|
||||
const OPENAI_CODEX_MODELS_URL = `${OPENAI_CODEX_RESPONSES_BASE_URL}/models?client_version=${readPinnedCodexClientVersion()}`;
|
||||
|
||||
function readPinnedCodexClientVersion(): string {
|
||||
const packageJson = JSON.parse(
|
||||
fs.readFileSync(new URL("../codex/package.json", import.meta.url), "utf8"),
|
||||
) as { dependencies?: Record<string, unknown> };
|
||||
const version = packageJson.dependencies?.["@openai/codex"];
|
||||
if (typeof version !== "string") {
|
||||
throw new Error("expected an exact @openai/codex dependency");
|
||||
}
|
||||
return version;
|
||||
}
|
||||
|
||||
function runWrappedPayloadCase(params: {
|
||||
wrap: NonNullable<ReturnType<typeof buildOpenAIProvider>["wrapStreamFn"]>;
|
||||
provider: string;
|
||||
@@ -676,9 +690,7 @@ describe("buildOpenAIProvider", () => {
|
||||
maxTokens: 64_000,
|
||||
});
|
||||
expect(fetchSpy).toHaveBeenCalledOnce();
|
||||
expect(fetchSpy.mock.calls[0]?.[0]).toBe(
|
||||
"https://chatgpt.com/backend-api/codex/models?client_version=1.0.0",
|
||||
);
|
||||
expect(fetchSpy.mock.calls[0]?.[0]).toBe(OPENAI_CODEX_MODELS_URL);
|
||||
const headers = fetchSpy.mock.calls[0]?.[1]?.headers;
|
||||
expect(headers).toBeInstanceOf(Headers);
|
||||
if (!(headers instanceof Headers)) {
|
||||
@@ -691,6 +703,25 @@ describe("buildOpenAIProvider", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("uses the managed Codex package version for OAuth model discovery", async () => {
|
||||
const pinnedVersion = readPinnedCodexClientVersion();
|
||||
const fetchGuard: LiveModelCatalogFetchGuard = vi.fn(async (params) => ({
|
||||
response: Response.json({ models: [{ slug: "gpt-5.5", visibility: "list" }] }),
|
||||
finalUrl: params.url,
|
||||
release: async () => undefined,
|
||||
}));
|
||||
|
||||
await buildOpenAICodexLiveProviderConfig({
|
||||
discoveryApiKey: "placeholder",
|
||||
fetchGuard,
|
||||
});
|
||||
|
||||
const requestUrl = vi.mocked(fetchGuard).mock.calls[0]?.[0].url;
|
||||
expect(new URL(requestUrl ?? "https://placeholder").searchParams.get("client_version")).toBe(
|
||||
pinnedVersion,
|
||||
);
|
||||
});
|
||||
|
||||
it("uses runtime OAuth profiles when catalog auth resolution is empty", async () => {
|
||||
mocks.resolveApiKeyForProvider.mockResolvedValue({
|
||||
mode: "oauth",
|
||||
@@ -797,9 +828,7 @@ describe("buildOpenAIProvider", () => {
|
||||
maxTokens: 128_000,
|
||||
});
|
||||
const fetchParams = vi.mocked(fetchGuard).mock.calls[0]?.[0];
|
||||
expect(fetchParams?.url).toBe(
|
||||
"https://chatgpt.com/backend-api/codex/models?client_version=1.0.0",
|
||||
);
|
||||
expect(fetchParams?.url).toBe(OPENAI_CODEX_MODELS_URL);
|
||||
const init = fetchParams?.init;
|
||||
const headers = init?.headers;
|
||||
expect(headers).toBeInstanceOf(Headers);
|
||||
@@ -876,52 +905,9 @@ describe("buildOpenAIProvider", () => {
|
||||
supportedReasoningEfforts: ["low", "medium", "high", "xhigh", "max", "ultra"],
|
||||
},
|
||||
});
|
||||
expect(provider.models.find((model) => model.id === "gpt-5.6-terra")).toMatchObject({
|
||||
contextWindow: 372_000,
|
||||
contextTokens: 372_000,
|
||||
compat: {
|
||||
supportedReasoningEfforts: ["low", "medium", "high", "xhigh", "max", "ultra"],
|
||||
},
|
||||
});
|
||||
expect(provider.models.find((model) => model.id === "gpt-5.6-luna")).toMatchObject({
|
||||
contextWindow: 372_000,
|
||||
contextTokens: 372_000,
|
||||
compat: { supportedReasoningEfforts: ["low", "medium", "high", "xhigh", "max"] },
|
||||
});
|
||||
expect(provider.models.map((model) => model.id)).not.toContain("gpt-5.6-terra");
|
||||
expect(provider.models.map((model) => model.id)).not.toContain("gpt-5.6-luna");
|
||||
expect(provider.models.map((model) => model.id)).toContain("gpt-5.5");
|
||||
const gpt56Models = Object.fromEntries(
|
||||
provider.models
|
||||
.filter((model) => model.id.startsWith("gpt-5.6-"))
|
||||
.map((model) => [model.id, model]),
|
||||
);
|
||||
for (const modelId of ["gpt-5.6-sol", "gpt-5.6-terra"]) {
|
||||
const model = gpt56Models[modelId];
|
||||
expect(model?.compat?.supportedReasoningEfforts).toContain("ultra");
|
||||
expect(
|
||||
buildOpenAIProvider()
|
||||
.resolveThinkingProfile?.({
|
||||
provider: "openai",
|
||||
modelId,
|
||||
agentRuntime: "codex",
|
||||
api: "openai-chatgpt-responses",
|
||||
compat: model?.compat,
|
||||
} as never)
|
||||
?.levels.map((level) => level.id),
|
||||
).toContain("ultra");
|
||||
}
|
||||
const luna = gpt56Models["gpt-5.6-luna"];
|
||||
expect(luna?.compat?.supportedReasoningEfforts).not.toContain("ultra");
|
||||
const lunaLevels = buildOpenAIProvider()
|
||||
.resolveThinkingProfile?.({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.6-luna",
|
||||
agentRuntime: "codex",
|
||||
api: "openai-chatgpt-responses",
|
||||
compat: luna?.compat,
|
||||
} as never)
|
||||
?.levels.map((level) => level.id);
|
||||
expect(lunaLevels).toContain("max");
|
||||
expect(lunaLevels).not.toContain("ultra");
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
|
||||
@@ -81,7 +81,10 @@ function classifyOpenAiFailoverCode(code: string | undefined) {
|
||||
}
|
||||
}
|
||||
const OPENAI_MODELS_ENDPOINT = "https://api.openai.com/v1/models";
|
||||
const OPENAI_CODEX_MODELS_ENDPOINT = `${OPENAI_CODEX_RESPONSES_BASE_URL}/models?client_version=1.0.0`;
|
||||
// Keep synchronized with extensions/codex's exact @openai/codex dependency;
|
||||
// the provider contract test fails when that managed-runtime pin changes.
|
||||
const OPENAI_CODEX_CLIENT_VERSION = "0.144.4";
|
||||
const OPENAI_CODEX_MODELS_ENDPOINT = `${OPENAI_CODEX_RESPONSES_BASE_URL}/models?client_version=${OPENAI_CODEX_CLIENT_VERSION}`;
|
||||
const OPENAI_MODELS_CACHE_TTL_MS = 60_000;
|
||||
const OPENAI_CODEX_MODELS_CACHE_TTL_MS = 60_000;
|
||||
const OPENAI_GPT_56_DIRECT_CONTEXT_TOKENS = 1_050_000;
|
||||
@@ -440,6 +443,12 @@ function buildOpenAICodexStaticProviderConfig(): ModelProviderConfig {
|
||||
api: "openai-chatgpt-responses",
|
||||
auth: "oauth",
|
||||
models: OPENAI_MANIFEST_PROVIDER.models.flatMap((model) => {
|
||||
const modelId = normalizeLowercaseStringOrEmpty(model.id);
|
||||
// Static OAuth rows are offline hints, not entitlement claims. Keep only
|
||||
// the proven GPT-5.6 subscription route; live discovery may add others.
|
||||
if (modelId.startsWith("gpt-5.6") && modelId !== OPENAI_GPT_56_SOL_MODEL_ID) {
|
||||
return [];
|
||||
}
|
||||
const normalized = normalizeOpenAICodexCatalogModel(model);
|
||||
return normalized ? [normalized] : [];
|
||||
}),
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
/** Agent runtime id normalization and retired runtime-selection compatibility helpers. */
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { normalizeAgentId } from "../routing/session-key.js";
|
||||
|
||||
export type EmbeddedAgentRuntime = "openclaw" | "auto" | (string & {});
|
||||
|
||||
export const OPENCLAW_AGENT_RUNTIME_ID = "openclaw";
|
||||
@@ -31,6 +34,21 @@ export function normalizeOptionalAgentRuntimeId(raw: unknown): EmbeddedAgentRunt
|
||||
return value ? normalizeEmbeddedAgentRuntime(value) : undefined;
|
||||
}
|
||||
|
||||
/** Resolves the deprecated explicit whole-agent runtime override, when present. */
|
||||
export function resolveAgentScopedRuntimeOverride(params: {
|
||||
config?: OpenClawConfig;
|
||||
agentId?: string;
|
||||
}): EmbeddedAgentRuntime | undefined {
|
||||
const agentId = params.agentId ? normalizeAgentId(params.agentId) : undefined;
|
||||
const agentRuntime = agentId
|
||||
? params.config?.agents?.list?.find((entry) => normalizeAgentId(entry.id) === agentId)
|
||||
?.agentRuntime?.id
|
||||
: undefined;
|
||||
return normalizeOptionalAgentRuntimeId(
|
||||
agentRuntime ?? params.config?.agents?.defaults?.agentRuntime?.id,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Whole-agent runtime environment selection is retired. Use
|
||||
* provider/model runtime policy or a registered agent harness instead.
|
||||
|
||||
@@ -127,6 +127,7 @@ function normalizeUsageStatsEntry(raw: unknown): ProfileUsageStats | undefined {
|
||||
errorCount: normalizeFiniteNumber(raw.errorCount),
|
||||
failureCounts: normalizeFailureCounts(raw.failureCounts),
|
||||
lastFailureAt: normalizeFiniteNumber(raw.lastFailureAt),
|
||||
lastProbeAt: normalizeFiniteNumber(raw.lastProbeAt),
|
||||
};
|
||||
for (const key of Object.keys(stats) as Array<keyof ProfileUsageStats>) {
|
||||
if (stats[key] === undefined) {
|
||||
|
||||
@@ -115,6 +115,7 @@ export type ProfileUsageStats = {
|
||||
errorCount?: number;
|
||||
failureCounts?: Partial<Record<AuthProfileFailureReason, number>>;
|
||||
lastFailureAt?: number;
|
||||
lastProbeAt?: number;
|
||||
};
|
||||
|
||||
/** Durable, non-secret auth profile selection state. */
|
||||
|
||||
@@ -6,6 +6,7 @@ type UsageDeps = {
|
||||
|
||||
type AuthProfileUsageTestApi = {
|
||||
setDepsForTest(overrides: Partial<UsageDeps> | null): void;
|
||||
resetWhamReprobeStateForTest(): void;
|
||||
};
|
||||
|
||||
function getTestApi(): AuthProfileUsageTestApi {
|
||||
@@ -16,4 +17,5 @@ function getTestApi(): AuthProfileUsageTestApi {
|
||||
|
||||
export const testing: AuthProfileUsageTestApi = {
|
||||
setDepsForTest: (overrides) => getTestApi().setDepsForTest(overrides),
|
||||
resetWhamReprobeStateForTest: () => getTestApi().resetWhamReprobeStateForTest(),
|
||||
};
|
||||
|
||||
@@ -14,11 +14,15 @@ import {
|
||||
isProfileInCooldown,
|
||||
markAuthProfileBlockedUntil,
|
||||
markAuthProfileFailure,
|
||||
maybeReprobeWhamBlockedProfiles,
|
||||
resolveProfilesUnavailableReason,
|
||||
resolveProfileUnusableUntilForDisplay,
|
||||
} from "./usage.js";
|
||||
import { testing as authProfileUsageTesting } from "./usage.test-support.js";
|
||||
|
||||
// Mirrors the module-local WHAM half-open reprobe interval contract (45 minutes).
|
||||
const WHAM_HALF_OPEN_REPROBE_INTERVAL_MS = 45 * 60 * 1000;
|
||||
|
||||
const storeMocks = vi.hoisted(() => ({
|
||||
saveAuthProfileStore: vi.fn(),
|
||||
updateAuthProfileStoreWithLock: vi.fn().mockResolvedValue(null),
|
||||
@@ -43,6 +47,7 @@ beforeEach(() => {
|
||||
|
||||
afterEach(() => {
|
||||
authProfileUsageTesting.setDepsForTest(null);
|
||||
authProfileUsageTesting.resetWhamReprobeStateForTest();
|
||||
vi.unstubAllGlobals();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
@@ -78,6 +83,16 @@ function mockLockedUpdateForStore(store: AuthProfileStore): void {
|
||||
);
|
||||
}
|
||||
|
||||
function mockLockedUpdatesForStore(store: AuthProfileStore): void {
|
||||
storeMocks.updateAuthProfileStoreWithLock.mockImplementation(
|
||||
async (lockParams: { updater: (store: AuthProfileStore) => boolean }) => {
|
||||
const freshStore = structuredClone(store);
|
||||
lockParams.updater(freshStore);
|
||||
return freshStore;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function expectProfileErrorStateCleared(
|
||||
stats: NonNullable<AuthProfileStore["usageStats"]>[string] | undefined,
|
||||
) {
|
||||
@@ -1181,6 +1196,156 @@ describe("markAuthProfileFailure — WHAM-aware Codex cooldowns", () => {
|
||||
}
|
||||
}
|
||||
|
||||
it("half-opens a stale long WHAM block and clears it when capacity returns", async () => {
|
||||
const now = 1_700_000_000_000;
|
||||
const store = makeStore({
|
||||
"openai:default": {
|
||||
blockedUntil: now + 6 * 24 * 60 * 60 * 1000,
|
||||
blockedReason: "subscription_limit",
|
||||
blockedSource: "wham",
|
||||
},
|
||||
});
|
||||
mockWhamResponse(200, { rate_limit: { limit_reached: false } });
|
||||
mockLockedUpdatesForStore(store);
|
||||
|
||||
maybeReprobeWhamBlockedProfiles({
|
||||
store,
|
||||
profileIds: ["openai:default"],
|
||||
now,
|
||||
});
|
||||
maybeReprobeWhamBlockedProfiles({
|
||||
store,
|
||||
profileIds: ["openai:default"],
|
||||
now,
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(fetchMock).toHaveBeenCalledOnce();
|
||||
expect(store.usageStats?.["openai:default"]?.blockedUntil).toBeUndefined();
|
||||
});
|
||||
expect(store.usageStats?.["openai:default"]?.lastProbeAt).toBe(now);
|
||||
expect(storeMocks.updateAuthProfileStoreWithLock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("leaves non-WHAM blocks outside the half-open probe path", () => {
|
||||
const now = 1_700_000_000_000;
|
||||
const store = makeStore({
|
||||
"openai:default": {
|
||||
blockedUntil: now + 6 * 24 * 60 * 60 * 1000,
|
||||
blockedReason: "subscription_limit",
|
||||
blockedSource: "codex_rate_limits",
|
||||
},
|
||||
});
|
||||
|
||||
maybeReprobeWhamBlockedProfiles({
|
||||
store,
|
||||
profileIds: ["openai:default"],
|
||||
now,
|
||||
});
|
||||
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(storeMocks.updateAuthProfileStoreWithLock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not re-probe a WHAM block inside the half-open interval", () => {
|
||||
const now = 1_700_000_000_000;
|
||||
const store = makeStore({
|
||||
"openai:default": {
|
||||
blockedUntil: now + 6 * 24 * 60 * 60 * 1000,
|
||||
blockedReason: "subscription_limit",
|
||||
blockedSource: "wham",
|
||||
lastProbeAt: now - WHAM_HALF_OPEN_REPROBE_INTERVAL_MS + 1,
|
||||
},
|
||||
});
|
||||
|
||||
maybeReprobeWhamBlockedProfiles({
|
||||
store,
|
||||
profileIds: ["openai:default"],
|
||||
now,
|
||||
});
|
||||
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(storeMocks.updateAuthProfileStoreWithLock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("re-arms a stale WHAM block from the latest blocked snapshot", async () => {
|
||||
const now = 1_700_000_000_000;
|
||||
const store = makeStore({
|
||||
"openai:default": {
|
||||
blockedUntil: now + 6 * 24 * 60 * 60 * 1000,
|
||||
blockedReason: "subscription_limit",
|
||||
blockedSource: "wham",
|
||||
blockedModel: "gpt-5.5",
|
||||
blockedScope: "model",
|
||||
},
|
||||
});
|
||||
mockWhamResponse(200, {
|
||||
rate_limit: {
|
||||
limit_reached: true,
|
||||
primary_window: { used_percent: 100, reset_after_seconds: 3_600 },
|
||||
},
|
||||
});
|
||||
mockLockedUpdatesForStore(store);
|
||||
const dateNowSpy = vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
|
||||
try {
|
||||
maybeReprobeWhamBlockedProfiles({
|
||||
store,
|
||||
profileIds: ["openai:default"],
|
||||
forModel: "gpt-5.5",
|
||||
now,
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(store.usageStats?.["openai:default"]?.blockedUntil).toBe(now + 3_600_000);
|
||||
});
|
||||
} finally {
|
||||
dateNowSpy.mockRestore();
|
||||
}
|
||||
expect(store.usageStats?.["openai:default"]?.lastProbeAt).toBe(now);
|
||||
expect(store.usageStats?.["openai:default"]?.blockedModel).toBe("gpt-5.5");
|
||||
expect(store.usageStats?.["openai:default"]?.blockedScope).toBe("model");
|
||||
});
|
||||
|
||||
it("does not apply an available result over a newer WHAM block", async () => {
|
||||
const now = 1_700_000_000_000;
|
||||
const originalUntil = now + 6 * 24 * 60 * 60 * 1000;
|
||||
const newerUntil = originalUntil + 60_000;
|
||||
const store = makeStore({
|
||||
"openai:default": {
|
||||
blockedUntil: originalUntil,
|
||||
blockedReason: "subscription_limit",
|
||||
blockedSource: "wham",
|
||||
lastFailureAt: now - 1,
|
||||
},
|
||||
});
|
||||
let releaseResponse: ((response: Response) => void) | undefined;
|
||||
fetchMock.mockReturnValueOnce(
|
||||
new Promise<Response>((resolve) => {
|
||||
releaseResponse = resolve;
|
||||
}),
|
||||
);
|
||||
mockLockedUpdatesForStore(store);
|
||||
|
||||
maybeReprobeWhamBlockedProfiles({
|
||||
store,
|
||||
profileIds: ["openai:default"],
|
||||
now,
|
||||
});
|
||||
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledOnce());
|
||||
const stats = store.usageStats?.["openai:default"];
|
||||
if (!stats || !releaseResponse) {
|
||||
throw new Error("expected claimed WHAM probe");
|
||||
}
|
||||
stats.blockedUntil = newerUntil;
|
||||
stats.lastFailureAt = now + 1;
|
||||
releaseResponse(Response.json({ rate_limit: { limit_reached: false } }));
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(storeMocks.updateAuthProfileStoreWithLock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
expect(store.usageStats?.["openai:default"]?.blockedUntil).toBe(newerUntil);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "burst contention",
|
||||
@@ -1244,6 +1409,7 @@ describe("markAuthProfileFailure — WHAM-aware Codex cooldowns", () => {
|
||||
expect(headers.originator).toBe("openclaw");
|
||||
expect(headers["User-Agent"]).toMatch(/^openclaw\//);
|
||||
const stats = store.usageStats?.["openai:default"];
|
||||
expect(stats?.lastProbeAt).toBe(now);
|
||||
if (exactBlocked) {
|
||||
expect(stats?.blockedUntil).toBe(now + expectedMs);
|
||||
expect(stats?.blockedReason).toBe("subscription_limit");
|
||||
|
||||
@@ -55,6 +55,9 @@ const testing = {
|
||||
authProfileUsageDeps.updateAuthProfileStoreWithLock =
|
||||
overrides?.updateAuthProfileStoreWithLock ?? updateAuthProfileStoreWithLock;
|
||||
},
|
||||
resetWhamReprobeStateForTest() {
|
||||
whamReprobesInFlight.clear();
|
||||
},
|
||||
};
|
||||
if (process.env.VITEST || process.env.NODE_ENV === "test") {
|
||||
(globalThis as Record<PropertyKey, unknown>)[Symbol.for("openclaw.authProfileUsageTestApi")] =
|
||||
@@ -96,6 +99,8 @@ const WHAM_PROBE_FAILURE_COOLDOWN_MS = 30_000;
|
||||
const WHAM_HTTP_ERROR_COOLDOWN_MS = 5 * 60 * 1000;
|
||||
const WHAM_TOKEN_EXPIRED_COOLDOWN_MS = 12 * 60 * 60 * 1000;
|
||||
const WHAM_DEAD_ACCOUNT_COOLDOWN_MS = 24 * 60 * 60 * 1000;
|
||||
const WHAM_HALF_OPEN_REPROBE_INTERVAL_MS = 45 * 60 * 1000;
|
||||
const whamReprobesInFlight = new Map<string, Promise<void>>();
|
||||
|
||||
type WhamUsageWindow = {
|
||||
limit_window_seconds?: number;
|
||||
@@ -113,6 +118,7 @@ type WhamUsageResponse = {
|
||||
};
|
||||
|
||||
type WhamCooldownProbeResult = {
|
||||
available?: true;
|
||||
cooldownMs: number;
|
||||
reason: string;
|
||||
blockedUntil?: number;
|
||||
@@ -220,6 +226,7 @@ function applyWhamCooldownResult(params: {
|
||||
if (params.whamResult.blockedUntil) {
|
||||
return {
|
||||
...params.computed,
|
||||
lastProbeAt: params.now,
|
||||
blockedUntil: Math.max(existingActiveBlockedUntil, params.whamResult.blockedUntil),
|
||||
blockedReason: "subscription_limit",
|
||||
blockedSource: params.whamResult.blockedSource ?? "wham",
|
||||
@@ -232,6 +239,7 @@ function applyWhamCooldownResult(params: {
|
||||
}
|
||||
return {
|
||||
...params.computed,
|
||||
lastProbeAt: params.now,
|
||||
cooldownUntil: Math.max(
|
||||
existingActiveCooldownUntil,
|
||||
resolveUsageWindowUntil(params.now, params.whamResult.cooldownMs),
|
||||
@@ -300,7 +308,11 @@ async function probeWhamForCooldown(
|
||||
}
|
||||
|
||||
if (data.rate_limit.limit_reached === false) {
|
||||
return { cooldownMs: WHAM_BURST_COOLDOWN_MS, reason: "wham_burst_contention" };
|
||||
return {
|
||||
available: true,
|
||||
cooldownMs: WHAM_BURST_COOLDOWN_MS,
|
||||
reason: "wham_burst_contention",
|
||||
};
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
@@ -351,6 +363,222 @@ async function probeWhamForCooldown(
|
||||
}
|
||||
}
|
||||
|
||||
function shouldHalfOpenProbeWhamBlock(params: {
|
||||
store: AuthProfileStore;
|
||||
profileId: string;
|
||||
forModel?: string;
|
||||
now: number;
|
||||
}): boolean {
|
||||
const profile = params.store.profiles[params.profileId];
|
||||
const stats = params.store.usageStats?.[params.profileId];
|
||||
if (
|
||||
!stats ||
|
||||
stats.blockedSource !== "wham" ||
|
||||
stats.blockedReason !== "subscription_limit" ||
|
||||
!isActiveUnusableWindow(stats.blockedUntil, params.now) ||
|
||||
isActiveUnusableWindow(stats.cooldownUntil, params.now) ||
|
||||
isActiveUnusableWindow(stats.disabledUntil, params.now) ||
|
||||
!shouldProbeWhamForFailure(profile, "rate_limit")
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
params.forModel &&
|
||||
stats.blockedScope === "model" &&
|
||||
stats.blockedModel &&
|
||||
stats.blockedModel !== params.forModel
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const remainingMs = (stats.blockedUntil ?? 0) - params.now;
|
||||
const sinceLastProbeMs = params.now - (stats.lastProbeAt ?? 0);
|
||||
return (
|
||||
remainingMs > WHAM_HALF_OPEN_REPROBE_INTERVAL_MS &&
|
||||
sinceLastProbeMs >= WHAM_HALF_OPEN_REPROBE_INTERVAL_MS
|
||||
);
|
||||
}
|
||||
|
||||
type WhamBlockGeneration = Pick<
|
||||
ProfileUsageStats,
|
||||
"blockedUntil" | "blockedModel" | "blockedScope" | "lastFailureAt"
|
||||
> & { rateLimitFailureCount?: number };
|
||||
|
||||
function matchesWhamBlockGeneration(
|
||||
stats: ProfileUsageStats,
|
||||
generation: WhamBlockGeneration,
|
||||
): boolean {
|
||||
return (
|
||||
stats.blockedUntil === generation.blockedUntil &&
|
||||
stats.blockedModel === generation.blockedModel &&
|
||||
stats.blockedScope === generation.blockedScope &&
|
||||
stats.lastFailureAt === generation.lastFailureAt &&
|
||||
stats.failureCounts?.rate_limit === generation.rateLimitFailureCount
|
||||
);
|
||||
}
|
||||
|
||||
async function claimWhamHalfOpenReprobe(params: {
|
||||
store: AuthProfileStore;
|
||||
profileId: string;
|
||||
agentDir?: string;
|
||||
forModel?: string;
|
||||
expectedProfile: AuthProfileCredential;
|
||||
startedAt: number;
|
||||
}): Promise<WhamBlockGeneration | null> {
|
||||
let generation: WhamBlockGeneration | undefined;
|
||||
const updated = await authProfileUsageDeps.updateAuthProfileStoreWithLock({
|
||||
agentDir: params.agentDir,
|
||||
updater: (freshStore) => {
|
||||
const currentProfile = freshStore.profiles[params.profileId];
|
||||
if (
|
||||
!isSameWhamCredential(params.expectedProfile, currentProfile) ||
|
||||
!shouldHalfOpenProbeWhamBlock({
|
||||
store: freshStore,
|
||||
profileId: params.profileId,
|
||||
forModel: params.forModel,
|
||||
now: params.startedAt,
|
||||
})
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const currentStats = freshStore.usageStats?.[params.profileId];
|
||||
if (!currentStats) {
|
||||
return false;
|
||||
}
|
||||
generation = {
|
||||
blockedUntil: currentStats.blockedUntil,
|
||||
blockedModel: currentStats.blockedModel,
|
||||
blockedScope: currentStats.blockedScope,
|
||||
lastFailureAt: currentStats.lastFailureAt,
|
||||
rateLimitFailureCount: currentStats.failureCounts?.rate_limit,
|
||||
};
|
||||
updateUsageStatsEntry(freshStore, params.profileId, (existing) => ({
|
||||
...existing,
|
||||
lastProbeAt: params.startedAt,
|
||||
}));
|
||||
return true;
|
||||
},
|
||||
});
|
||||
if (updated && generation) {
|
||||
params.store.usageStats = updated.usageStats;
|
||||
return generation;
|
||||
}
|
||||
if (updated === null) {
|
||||
logDroppedAuthProfileBookkeeping("wham_half_open_claim", params.profileId);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function runWhamHalfOpenReprobe(params: {
|
||||
store: AuthProfileStore;
|
||||
profileId: string;
|
||||
agentDir?: string;
|
||||
forModel?: string;
|
||||
expectedProfile: AuthProfileCredential;
|
||||
startedAt: number;
|
||||
}): Promise<void> {
|
||||
const generation = await claimWhamHalfOpenReprobe(params);
|
||||
if (!generation) {
|
||||
return;
|
||||
}
|
||||
const result = await probeWhamForCooldown(params.store, params.profileId);
|
||||
if (!result || (!result.available && !result.blockedUntil)) {
|
||||
return;
|
||||
}
|
||||
let applied = false;
|
||||
const updated = await authProfileUsageDeps.updateAuthProfileStoreWithLock({
|
||||
agentDir: params.agentDir,
|
||||
updater: (freshStore) => {
|
||||
const currentProfile = freshStore.profiles[params.profileId];
|
||||
const currentStats = freshStore.usageStats?.[params.profileId];
|
||||
if (
|
||||
!currentStats ||
|
||||
currentStats.blockedSource !== "wham" ||
|
||||
currentStats.blockedReason !== "subscription_limit" ||
|
||||
currentStats.lastProbeAt !== params.startedAt ||
|
||||
!matchesWhamBlockGeneration(currentStats, generation) ||
|
||||
!isSameWhamCredential(params.expectedProfile, currentProfile)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
updateUsageStatsEntry(freshStore, params.profileId, (existing) => {
|
||||
if (result.available) {
|
||||
return {
|
||||
...existing,
|
||||
blockedUntil: undefined,
|
||||
blockedReason: undefined,
|
||||
blockedSource: undefined,
|
||||
blockedModel: undefined,
|
||||
blockedScope: undefined,
|
||||
};
|
||||
}
|
||||
if (result.blockedUntil) {
|
||||
return {
|
||||
...existing,
|
||||
blockedUntil: result.blockedUntil,
|
||||
blockedReason: "subscription_limit",
|
||||
blockedSource: "wham",
|
||||
blockedModel: generation.blockedModel,
|
||||
blockedScope: generation.blockedScope,
|
||||
};
|
||||
}
|
||||
return existing ?? {};
|
||||
});
|
||||
applied = true;
|
||||
return true;
|
||||
},
|
||||
});
|
||||
if (updated && applied) {
|
||||
params.store.usageStats = updated.usageStats;
|
||||
} else if (updated === null) {
|
||||
logDroppedAuthProfileBookkeeping("wham_half_open_reprobe", params.profileId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Starts bounded background refreshes for long WHAM-only profile blocks. */
|
||||
export function maybeReprobeWhamBlockedProfiles(params: {
|
||||
store: AuthProfileStore;
|
||||
profileIds: string[];
|
||||
agentDir?: string;
|
||||
forModel?: string;
|
||||
now?: number;
|
||||
}): void {
|
||||
const now = params.now ?? Date.now();
|
||||
for (const profileId of params.profileIds) {
|
||||
if (!shouldHalfOpenProbeWhamBlock({ ...params, profileId, now })) {
|
||||
continue;
|
||||
}
|
||||
const profile = params.store.profiles[profileId];
|
||||
if (!profile) {
|
||||
continue;
|
||||
}
|
||||
const probeKey = `${params.agentDir ?? "default"}\u0000${profileId}`;
|
||||
if (whamReprobesInFlight.has(probeKey)) {
|
||||
continue;
|
||||
}
|
||||
// Keep the current synchronous fallback decision: this attempt still
|
||||
// skips. A deduped refresh updates durable state for the next decision.
|
||||
const task = runWhamHalfOpenReprobe({
|
||||
store: params.store,
|
||||
profileId,
|
||||
agentDir: params.agentDir,
|
||||
forModel: params.forModel,
|
||||
expectedProfile: structuredClone(profile),
|
||||
startedAt: now,
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
authProfileUsageLog.warn("WHAM half-open reprobe failed", {
|
||||
event: "auth_profile_wham_reprobe_error",
|
||||
profileId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
whamReprobesInFlight.delete(probeKey);
|
||||
});
|
||||
whamReprobesInFlight.set(probeKey, task);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer the most likely reason all candidate profiles are currently unavailable.
|
||||
*
|
||||
|
||||
@@ -9,5 +9,6 @@ export { ensureAuthProfileStore, loadAuthProfileStoreForRuntime } from "./auth-p
|
||||
export {
|
||||
getSoonestCooldownExpiry,
|
||||
isProfileInCooldown,
|
||||
maybeReprobeWhamBlockedProfiles,
|
||||
resolveProfilesUnavailableReason,
|
||||
} from "./auth-profiles/usage.js";
|
||||
|
||||
@@ -19,6 +19,7 @@ vi.mock("./auth-profiles/store.js", () => ({
|
||||
vi.mock("./auth-profiles/usage.js", () => ({
|
||||
getSoonestCooldownExpiry: vi.fn(),
|
||||
isProfileInCooldown: vi.fn(),
|
||||
maybeReprobeWhamBlockedProfiles: vi.fn(),
|
||||
resolveProfilesUnavailableReason: vi.fn(),
|
||||
}));
|
||||
|
||||
|
||||
@@ -157,6 +157,7 @@ const authRuntimeMock = vi.hoisted(() => {
|
||||
loadAuthProfileStoreForRuntime: vi.fn((agentDir?: string) => getStore(agentDir)),
|
||||
resolveAuthProfileOrder: (params: { store: AuthProfileStore; provider: string }) =>
|
||||
getProfileIds(params.store, params.provider),
|
||||
maybeReprobeWhamBlockedProfiles: vi.fn(),
|
||||
isProfileInCooldown,
|
||||
resolveProfilesUnavailableReason: (params: {
|
||||
store: AuthProfileStore;
|
||||
|
||||
@@ -1579,6 +1579,12 @@ async function runWithModelFallbackInternal<T>(
|
||||
store: authStore,
|
||||
provider: candidate.provider,
|
||||
});
|
||||
authRuntime.maybeReprobeWhamBlockedProfiles({
|
||||
store: authStore,
|
||||
profileIds,
|
||||
agentDir: params.agentDir,
|
||||
forModel: candidate.model,
|
||||
});
|
||||
const isAnyProfileAvailable = profileIds.some(
|
||||
(id) => !authRuntime.isProfileInCooldown(authStore, id, undefined, candidate.model),
|
||||
);
|
||||
|
||||
@@ -158,6 +158,54 @@ describe("OpenAI runtime routing policy", () => {
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("honors the deprecated whole-agent OpenClaw runtime opt-out", () => {
|
||||
const config = {
|
||||
agents: {
|
||||
defaults: { agentRuntime: { id: "openclaw" } },
|
||||
list: [{ id: "worker", agentRuntime: { id: "openclaw" } }],
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
expect(modelSelectionShouldEnsureCodexPlugin({ model: "openai/gpt-5.5", config })).toBe(false);
|
||||
expect(
|
||||
modelSelectionShouldEnsureCodexPlugin({
|
||||
model: "openai/gpt-5.5",
|
||||
config,
|
||||
agentId: "worker",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps per-model Codex policy above the whole-agent OpenClaw opt-out", () => {
|
||||
const config = {
|
||||
agents: {
|
||||
defaults: {
|
||||
agentRuntime: { id: "openclaw" },
|
||||
models: {
|
||||
"openai/gpt-5.5": { agentRuntime: { id: "codex" } },
|
||||
},
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
expect(modelSelectionShouldEnsureCodexPlugin({ model: "openai/gpt-5.5", config })).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps per-model auto policy above the whole-agent OpenClaw opt-out", () => {
|
||||
const config = {
|
||||
agents: {
|
||||
defaults: {
|
||||
agentRuntime: { id: "openclaw" },
|
||||
models: {
|
||||
"openai/gpt-5.5": { agentRuntime: { id: "auto" } },
|
||||
},
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
expect(modelSelectionShouldEnsureCodexPlugin({ model: "openai/gpt-5.5", config })).toBe(true);
|
||||
});
|
||||
|
||||
it("normalizes OpenAI provider keys before checking custom base URLs", () => {
|
||||
const config = {
|
||||
models: {
|
||||
|
||||
@@ -7,7 +7,11 @@ import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { ProviderRouteOverridePresence } from "../plugin-sdk/provider-model-types.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../routing/session-key.js";
|
||||
import { isDefaultAgentRuntimeId, normalizeOptionalAgentRuntimeId } from "./agent-runtime-id.js";
|
||||
import {
|
||||
isDefaultAgentRuntimeId,
|
||||
normalizeOptionalAgentRuntimeId,
|
||||
resolveAgentScopedRuntimeOverride,
|
||||
} from "./agent-runtime-id.js";
|
||||
import { hasModelExtraParams } from "./model-extra-params.js";
|
||||
import { resolveModelRuntimePolicy } from "./model-runtime-policy.js";
|
||||
import { resolveOpenAIModelRoutes } from "./openai-model-routes.js";
|
||||
@@ -101,17 +105,27 @@ export function modelSelectionShouldEnsureCodexPlugin(params: {
|
||||
const modelRef = params.model?.trim();
|
||||
const slashIndex = modelRef?.indexOf("/") ?? -1;
|
||||
const modelId = slashIndex >= 0 ? modelRef?.slice(slashIndex + 1) : undefined;
|
||||
const configuredRuntime = normalizeOptionalAgentRuntimeId(
|
||||
resolveModelRuntimePolicy({
|
||||
config: params.config,
|
||||
provider,
|
||||
modelId,
|
||||
agentId: params.agentId,
|
||||
}).policy?.id,
|
||||
);
|
||||
const configuredPolicy = resolveModelRuntimePolicy({
|
||||
config: params.config,
|
||||
provider,
|
||||
modelId,
|
||||
agentId: params.agentId,
|
||||
}).policy;
|
||||
const configuredRuntime = normalizeOptionalAgentRuntimeId(configuredPolicy?.id);
|
||||
if (configuredRuntime && !isDefaultAgentRuntimeId(configuredRuntime)) {
|
||||
return configuredRuntime === "codex";
|
||||
}
|
||||
if (!configuredPolicy) {
|
||||
const agentRuntime = resolveAgentScopedRuntimeOverride({
|
||||
config: params.config,
|
||||
agentId: params.agentId,
|
||||
});
|
||||
// Any explicit model policy wins; without one, the shipped whole-agent
|
||||
// opt-out still suppresses implicit Codex installation despite retirement.
|
||||
if (agentRuntime && !isDefaultAgentRuntimeId(agentRuntime)) {
|
||||
return agentRuntime === "codex";
|
||||
}
|
||||
}
|
||||
return (
|
||||
resolveOpenAIImplicitAgentRuntime({
|
||||
provider,
|
||||
|
||||
Reference in New Issue
Block a user