mirror of
https://github.com/pydantic/pydantic-ai-harness.git
synced 2026-07-21 10:55:35 +00:00
fix(overflow): mark the Summarize band's output as a harness stand-in
The `Summarize` band replaced a tool's real content with an LLM summary carrying no marker, so the model (and every downstream model that reads the history) saw the summary as the tool's genuine output -- inconsistent with this PR's own uniform-explicit-elision-marker goal, which every other path (spill preview, truncation, read-back cap) already meets. Add `summary_header` to `_markers.py` (same bracketed, timestamp-free convention as `spill_header`/`elision_marker`) and prefix it onto the summary: it states the original size, that the harness summarized it, and how to recover the original (a retrieval hint when spilled, else re-run the tool). `Summarize` does not spill, so the recovery clause is the re-run form. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
279336dd59
commit
80c9d2827a
@@ -29,6 +29,7 @@ from pydantic_ai_harness.experimental.overflow._markers import (
|
||||
elision_marker,
|
||||
missing_handle_message,
|
||||
spill_header,
|
||||
summary_header,
|
||||
)
|
||||
from pydantic_ai_harness.experimental.overflow._payload import (
|
||||
is_binary,
|
||||
@@ -437,7 +438,14 @@ class OverflowingToolOutput(AbstractCapability[AgentDepsT]):
|
||||
summary = await self._summarize(ctx, call, action, unit.text)
|
||||
except Exception:
|
||||
return await self._fallback(ctx, call, action.then, unit)
|
||||
return summary, None
|
||||
# Mark the summary the same way every other elision path is marked: an explicit
|
||||
# bracketed header so the model (and any downstream model) knows this body is a
|
||||
# harness-generated stand-in, not the tool's real output. `Summarize` does not spill
|
||||
# the original, so there is no retrieval handle.
|
||||
size_unit = 'tokens' if self.over_tokens else 'chars'
|
||||
amount = measure(unit.text, over_tokens=self.over_tokens, tokenizer=self.tokenizer)
|
||||
header = summary_header(size_desc=f'{amount:,} {size_unit}', handle=None)
|
||||
return f'{header}\n{summary}', None
|
||||
|
||||
async def _summarize(
|
||||
self,
|
||||
|
||||
@@ -48,6 +48,19 @@ def elision_marker(*, omitted: str, handle: str | None) -> str:
|
||||
return f'[... {omitted} omitted; not stored, re-run the tool for the full output ...]'
|
||||
|
||||
|
||||
def summary_header(*, size_desc: str, handle: str | None) -> str:
|
||||
"""Lead line of a summarized tool return: what was replaced, by whom, and how to recover it.
|
||||
|
||||
Like `spill_header`/`elision_marker`, this makes the elision explicit: the model learns the
|
||||
body below is a harness-generated summary standing in for the real tool output, not the tool
|
||||
result itself. When `handle` is set the original was also spilled and stays retrievable
|
||||
through the query tools; otherwise the summary is all that remains (re-run the tool for the
|
||||
full output).
|
||||
"""
|
||||
recover = f'{retrieval_hint(handle)}' if handle is not None else 're-run the tool for the full output'
|
||||
return f'[Tool output too large ({size_desc}); summarized by harness. {recover}. The summary follows.]'
|
||||
|
||||
|
||||
def missing_handle_message(handle: str) -> str:
|
||||
"""Guidance returned (not raised) when a handle does not resolve to a stored payload.
|
||||
|
||||
|
||||
@@ -538,7 +538,7 @@ class TestSummarize:
|
||||
bands=[Band(over=5, action=Summarize(summarize=lambda name, text: f'{name}:{len(text)}'))]
|
||||
)
|
||||
out = await _run(cap, 'x' * 100)
|
||||
assert out == 'big_tool:100'
|
||||
assert out.endswith('big_tool:100')
|
||||
|
||||
async def test_custom_async_summarizer(self):
|
||||
async def summ(name: str, text: str) -> str:
|
||||
@@ -548,14 +548,25 @@ class TestSummarize:
|
||||
bands=[Band(over=5, action=Summarize(summarize=summ))]
|
||||
)
|
||||
out = await _run(cap, 'x' * 100)
|
||||
assert out == 'async:100'
|
||||
assert out.endswith('async:100')
|
||||
|
||||
async def test_summary_carries_explicit_marker(self):
|
||||
# The summary replaces the real tool output, so it must be marked as a harness
|
||||
# stand-in the same way every other elision path is -- not passed off as the result.
|
||||
cap: OverflowingToolOutput[object] = OverflowingToolOutput(
|
||||
bands=[Band(over=5, action=Summarize(summarize=lambda name, text: 'THE GIST'))]
|
||||
)
|
||||
out = await _run(cap, 'x' * 100)
|
||||
assert out.startswith('[Tool output too large (100 chars); summarized by harness.')
|
||||
assert 're-run the tool for the full output' in out
|
||||
assert out.endswith('\nTHE GIST')
|
||||
|
||||
async def test_inherited_model_and_usage(self):
|
||||
# model=None resolves to ctx.model, and the call threads usage=ctx.usage.
|
||||
ctx = _make_ctx(model=_fixed_model('THE SUMMARY'))
|
||||
cap: OverflowingToolOutput[object] = OverflowingToolOutput(bands=[Band(over=5, action=Summarize())])
|
||||
out = await _run(cap, 'x' * 100, ctx=ctx)
|
||||
assert out == 'THE SUMMARY'
|
||||
assert out.endswith('THE SUMMARY')
|
||||
assert ctx.usage.requests == 1
|
||||
|
||||
async def test_explicit_model_overrides_ctx(self):
|
||||
@@ -564,7 +575,7 @@ class TestSummarize:
|
||||
bands=[Band(over=5, action=Summarize(model=_fixed_model('FROM EXPLICIT MODEL')))]
|
||||
)
|
||||
out = await _run(cap, 'x' * 100, ctx=ctx)
|
||||
assert out == 'FROM EXPLICIT MODEL'
|
||||
assert out.endswith('FROM EXPLICIT MODEL')
|
||||
assert ctx.usage.requests == 1
|
||||
|
||||
async def test_binary_summarize_falls_back(self):
|
||||
|
||||
Reference in New Issue
Block a user