fix: handle explicit silent assistant replies (#92073)

Signed-off-by: sallyom <somalley@redhat.com>
This commit is contained in:
Sally O'Malley
2026-06-11 10:20:08 -04:00
committed by GitHub
parent 9be1699074
commit ea813a2476
3 changed files with 190 additions and 1 deletions
@@ -75,6 +75,23 @@ describe("classifyEmbeddedAgentRunResultForModelFallback", () => {
expect(result).toBeNull();
});
it("does not fallback on deliberate silent terminal replies after payload filtering", () => {
const result = classifyEmbeddedAgentRunResultForModelFallback({
provider: "openai",
model: "gpt-5.5",
result: {
payloads: [],
meta: {
durationMs: 42,
finalAssistantRawText: "NO_REPLY",
finalAssistantVisibleText: "NO_REPLY",
},
},
});
expect(result).toBeNull();
});
it("uses provider-scoped failover matching for business-denial payloads", () => {
const result = classifyEmbeddedAgentRunResultForModelFallback({
provider: "openrouter",
@@ -2706,6 +2706,62 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
).toBe(false);
});
it("treats exact NO_REPLY assistant turns as silent only when the caller allows it", () => {
const attempt = makeAttemptResult({
assistantTexts: ["NO_REPLY"],
lastAssistant: {
role: "assistant",
stopReason: "stop",
provider: "openai",
model: "gpt-5.5",
content: [{ type: "text", text: "NO_REPLY" }],
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
});
expect(
shouldTreatEmptyAssistantReplyAsSilent({
allowEmptyAssistantReplyAsSilent: true,
payloadCount: 0,
aborted: false,
timedOut: false,
attempt,
}),
).toBe(true);
expect(
shouldTreatEmptyAssistantReplyAsSilent({
allowEmptyAssistantReplyAsSilent: false,
payloadCount: 0,
aborted: false,
timedOut: false,
attempt,
}),
).toBe(false);
});
it("treats post-tool exact NO_REPLY assistant turns as intentional silence", () => {
const attempt = makeAttemptResult({
assistantTexts: ["NO_REPLY"],
toolMetas: [{ toolName: "process.poll", meta: "pid=123" }],
lastAssistant: {
role: "assistant",
stopReason: "stop",
provider: "openai",
model: "gpt-5.5",
content: [{ type: "text", text: "NO_REPLY" }],
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
});
expect(
shouldTreatEmptyAssistantReplyAsSilent({
allowEmptyAssistantReplyAsSilent: true,
payloadCount: 0,
aborted: false,
timedOut: false,
attempt,
}),
).toBe(true);
});
it("does not treat error or side-effect empty turns as silent", () => {
const errorAttempt = makeAttemptResult({
assistantTexts: [],
@@ -2717,6 +2773,16 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
content: [],
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
});
const silentErrorAttempt = makeAttemptResult({
assistantTexts: ["NO_REPLY"],
lastAssistant: {
role: "assistant",
stopReason: "error",
provider: "openai",
model: "gpt-5.5",
content: [{ type: "text", text: "NO_REPLY" }],
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
});
const sideEffectAttempt = makeAttemptResult({
assistantTexts: [],
didSendViaMessagingTool: true,
@@ -2751,6 +2817,15 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
attempt: errorAttempt,
}),
).toBe(false);
expect(
shouldTreatEmptyAssistantReplyAsSilent({
allowEmptyAssistantReplyAsSilent: true,
payloadCount: 0,
aborted: false,
timedOut: false,
attempt: silentErrorAttempt,
}),
).toBe(false);
expect(
shouldTreatEmptyAssistantReplyAsSilent({
allowEmptyAssistantReplyAsSilent: true,
@@ -2803,6 +2878,47 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
expect(result.meta.livenessState).toBe("working");
});
it("returns NO_REPLY without retrying exact silent assistant replies when silence is allowed", async () => {
mockedClassifyFailoverReason.mockReturnValue(null);
mockedRunEmbeddedAttempt.mockResolvedValue(
makeAttemptResult({
assistantTexts: ["NO_REPLY"],
lastAssistant: {
role: "assistant",
stopReason: "stop",
provider: "openai",
model: "gpt-5.5",
content: [
{
type: "thinking",
thinking: "internal reasoning",
thinkingSignature: JSON.stringify({ id: "rs_exact_silent", type: "reasoning" }),
},
{ type: "text", text: "NO_REPLY" },
],
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
}),
);
const result = await runEmbeddedAgent({
...overflowBaseRunParams,
allowEmptyAssistantReplyAsSilent: true,
provider: "openai",
model: "gpt-5.5",
runId: "run-exact-silent-assistant-reply",
});
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
const onlyCall = runAttemptCall(0);
expect(onlyCall.prompt).not.toContain(REASONING_ONLY_RETRY_INSTRUCTION);
expect(onlyCall.prompt).not.toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION);
expectNoWarnMessageWith("empty response detected");
expectNoWarnMessageWith("incomplete turn detected");
expect(result.payloads).toEqual([{ text: "NO_REPLY" }]);
expect(result.meta.terminalReplyKind).toBe("silent-empty");
expect(result.meta.livenessState).toBe("working");
});
it("retries post-tool openai-compatible empty stop turns even when empty silence is allowed", async () => {
mockedClassifyFailoverReason.mockReturnValue(null);
mockedResolveModelAsync.mockResolvedValue({
@@ -2862,6 +2978,54 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
expectWarnMessageWith("empty response detected");
});
it("returns NO_REPLY without retrying post-tool exact silent assistant replies", async () => {
mockedClassifyFailoverReason.mockReturnValue(null);
mockedResolveModelAsync.mockResolvedValue({
model: {
id: "step-router-v1",
provider: "stepfun",
contextWindow: 200000,
api: "openai-completions",
},
error: null,
authStorage: {
setRuntimeApiKey: vi.fn(),
},
modelRegistry: {},
});
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: ["NO_REPLY"],
toolMetas: [{ toolName: "process.poll", meta: "pid=123" }],
lastAssistant: {
role: "assistant",
api: "openai-completions",
stopReason: "stop",
provider: "stepfun",
model: "step-router-v1",
content: [{ type: "text", text: "NO_REPLY" }],
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
}),
);
const result = await runEmbeddedAgent({
...overflowBaseRunParams,
allowEmptyAssistantReplyAsSilent: true,
provider: "stepfun",
model: "step-router-v1",
runId: "run-post-tool-exact-silent-retry",
});
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
const onlyCall = runAttemptCall(0);
expect(onlyCall.prompt).not.toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION);
expectNoWarnMessageWith("empty response detected");
expectNoWarnMessageWith("incomplete turn detected");
expect(result.payloads).toEqual([{ text: "NO_REPLY" }]);
expect(result.meta.terminalReplyKind).toBe("silent-empty");
expect(result.meta.livenessState).toBe("working");
});
it("keeps retrying and surfacing clean empty assistant turns without the silence flag", async () => {
mockedClassifyFailoverReason.mockReturnValue(null);
mockedRunEmbeddedAttempt.mockResolvedValue(
@@ -635,7 +635,7 @@ function shouldSkipPlanningOnlyRetry(params: {
);
}
/** Allows configured silent handling for replay-safe empty or reasoning-only assistant turns. */
/** Allows configured silent handling for replay-safe empty, reasoning-only, or explicit silent turns. */
export function shouldTreatEmptyAssistantReplyAsSilent(params: {
allowEmptyAssistantReplyAsSilent?: boolean;
payloadCount: number;
@@ -649,6 +649,14 @@ export function shouldTreatEmptyAssistantReplyAsSilent(params: {
if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) {
return false;
}
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
if (
params.payloadCount === 0 &&
assistant?.stopReason !== "error" &&
hasOnlySilentAssistantReply(params.attempt.assistantTexts)
) {
return true;
}
// Post-tool empty stops are ambiguous provider failures, not intentional silence.
// Let the retry/incomplete-turn paths decide whether replay is safe.
if (