fix(overflow): mark the Summarize band's output as a harness stand-in

The `Summarize` band replaced a tool's real content with an LLM summary
carrying no marker, so the model (and every downstream model that reads the
history) saw the summary as the tool's genuine output -- inconsistent with
this PR's own uniform-explicit-elision-marker goal, which every other path
(spill preview, truncation, read-back cap) already meets.

Add `summary_header` to `_markers.py` (same bracketed, timestamp-free
convention as `spill_header`/`elision_marker`) and prefix it onto the
summary: it states the original size, that the harness summarized it, and
how to recover the original (a retrieval hint when spilled, else re-run the
tool). `Summarize` does not spill, so the recovery clause is the re-run
form.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Douwe Maan
2026-07-10 00:37:29 +00:00
co-authored by Claude Fable 5
parent 279336dd59
commit 80c9d2827a
3 changed files with 37 additions and 5 deletions
@@ -29,6 +29,7 @@ from pydantic_ai_harness.experimental.overflow._markers import (
elision_marker,
missing_handle_message,
spill_header,
summary_header,
)
from pydantic_ai_harness.experimental.overflow._payload import (
is_binary,
@@ -437,7 +438,14 @@ class OverflowingToolOutput(AbstractCapability[AgentDepsT]):
summary = await self._summarize(ctx, call, action, unit.text)
except Exception:
return await self._fallback(ctx, call, action.then, unit)
return summary, None
# Mark the summary the same way every other elision path is marked: an explicit
# bracketed header so the model (and any downstream model) knows this body is a
# harness-generated stand-in, not the tool's real output. `Summarize` does not spill
# the original, so there is no retrieval handle.
size_unit = 'tokens' if self.over_tokens else 'chars'
amount = measure(unit.text, over_tokens=self.over_tokens, tokenizer=self.tokenizer)
header = summary_header(size_desc=f'{amount:,} {size_unit}', handle=None)
return f'{header}\n{summary}', None
async def _summarize(
self,
@@ -48,6 +48,19 @@ def elision_marker(*, omitted: str, handle: str | None) -> str:
return f'[... {omitted} omitted; not stored, re-run the tool for the full output ...]'
def summary_header(*, size_desc: str, handle: str | None) -> str:
"""Lead line of a summarized tool return: what was replaced, by whom, and how to recover it.
Like `spill_header`/`elision_marker`, this makes the elision explicit: the model learns the
body below is a harness-generated summary standing in for the real tool output, not the tool
result itself. When `handle` is set the original was also spilled and stays retrievable
through the query tools; otherwise the summary is all that remains (re-run the tool for the
full output).
"""
recover = f'{retrieval_hint(handle)}' if handle is not None else 're-run the tool for the full output'
return f'[Tool output too large ({size_desc}); summarized by harness. {recover}. The summary follows.]'
def missing_handle_message(handle: str) -> str:
"""Guidance returned (not raised) when a handle does not resolve to a stored payload.
+15 -4
View File
@@ -538,7 +538,7 @@ class TestSummarize:
bands=[Band(over=5, action=Summarize(summarize=lambda name, text: f'{name}:{len(text)}'))]
)
out = await _run(cap, 'x' * 100)
assert out == 'big_tool:100'
assert out.endswith('big_tool:100')
async def test_custom_async_summarizer(self):
async def summ(name: str, text: str) -> str:
@@ -548,14 +548,25 @@ class TestSummarize:
bands=[Band(over=5, action=Summarize(summarize=summ))]
)
out = await _run(cap, 'x' * 100)
assert out == 'async:100'
assert out.endswith('async:100')
async def test_summary_carries_explicit_marker(self):
# The summary replaces the real tool output, so it must be marked as a harness
# stand-in the same way every other elision path is -- not passed off as the result.
cap: OverflowingToolOutput[object] = OverflowingToolOutput(
bands=[Band(over=5, action=Summarize(summarize=lambda name, text: 'THE GIST'))]
)
out = await _run(cap, 'x' * 100)
assert out.startswith('[Tool output too large (100 chars); summarized by harness.')
assert 're-run the tool for the full output' in out
assert out.endswith('\nTHE GIST')
async def test_inherited_model_and_usage(self):
# model=None resolves to ctx.model, and the call threads usage=ctx.usage.
ctx = _make_ctx(model=_fixed_model('THE SUMMARY'))
cap: OverflowingToolOutput[object] = OverflowingToolOutput(bands=[Band(over=5, action=Summarize())])
out = await _run(cap, 'x' * 100, ctx=ctx)
assert out == 'THE SUMMARY'
assert out.endswith('THE SUMMARY')
assert ctx.usage.requests == 1
async def test_explicit_model_overrides_ctx(self):
@@ -564,7 +575,7 @@ class TestSummarize:
bands=[Band(over=5, action=Summarize(model=_fixed_model('FROM EXPLICIT MODEL')))]
)
out = await _run(cap, 'x' * 100, ctx=ctx)
assert out == 'FROM EXPLICIT MODEL'
assert out.endswith('FROM EXPLICIT MODEL')
assert ctx.usage.requests == 1
async def test_binary_summarize_falls_back(self):