mirror of
https://github.com/pydantic/pydantic-ai-harness.git
synced 2026-07-21 02:45:34 +00:00
refactor(cross_model_label): graduate to top-level layout, add docs-site page
Merge `main` and relocate from `experimental/cross_model_label` to top-level `pydantic_ai_harness/cross_model_label/`, matching #347/#354. Drops the experimental import warning, fixes import paths, moves tests. Adds `docs/cross-model-label.md` (nav.json, index.md, parity gate) and a README matrix row. No behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
d718138aa1
commit
e93a64352f
@@ -158,6 +158,7 @@ We studied leading coding agents, agent frameworks, and Claw-style assistants to
|
||||
| | **Limit warnings** | Warn agent before hitting context/iteration limits | :white_check_mark: [Docs](pydantic_ai_harness/compaction/) | [summarization-pydantic-ai](https://github.com/vstorm-co/summarization-pydantic-ai) (vstorm‑co) |
|
||||
| | **Tool output management** | Truncate, summarize, or spill large tool outputs | :white_check_mark: [Docs](pydantic_ai_harness/overflowing_tool_output/) | |
|
||||
| | **System reminders** | Inject periodic reminders to counteract instruction drift | :construction: [PR #181](https://github.com/pydantic/pydantic-ai-harness/pull/181) | |
|
||||
| | **Cross-model labeling** | Tell the model when earlier turns in the history came from a different model | :white_check_mark: [Docs](pydantic_ai_harness/cross_model_label/) | |
|
||||
| **Memory & persistence** | **Memory** | Persistent, namespaced notebook with bounded prompt injection, on-demand search, and concurrency-safe stores | :white_check_mark: [Docs](pydantic_ai_harness/memory/) | [pydantic-deep](https://github.com/vstorm-co/pydantic-deepagents) (vstorm‑co) |
|
||||
| | **Session persistence** | Save and restore full conversation state | :white_check_mark: [Docs](pydantic_ai_harness/step_persistence/) | |
|
||||
| | **Checkpointing** | Snapshot, resume (`continue_run`), and fork (`fork_run`) a run | :white_check_mark: [Docs](pydantic_ai_harness/step_persistence/) | [pydantic-deep](https://github.com/vstorm-co/pydantic-deepagents) (vstorm‑co) |
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
---
|
||||
title: Cross-Model Label
|
||||
description: Tell the model when earlier assistant turns came from a different model.
|
||||
---
|
||||
|
||||
# Cross-Model Label
|
||||
|
||||
Tell the model when earlier assistant turns came from a different model.
|
||||
|
||||
> [!NOTE]
|
||||
> Import this capability from its submodule. It is not re-exported from `pydantic_ai_harness`:
|
||||
>
|
||||
> ```python
|
||||
> from pydantic_ai_harness.cross_model_label import CrossModelHistoryLabel
|
||||
> ```
|
||||
|
||||
Cross-Model Label is a released, non-experimental capability. Pydantic AI Harness is still on 0.x releases, so the API may change between minor releases. See the repository [version policy](https://github.com/pydantic/pydantic-ai-harness#version-policy).
|
||||
|
||||
When a run continues a history whose assistant turns were produced by a *different* model than
|
||||
the one now serving -- a `FallbackModel` failover, a model swap between runs, an A/B handoff, a
|
||||
takeover -- the serving model otherwise reads those turns as its own. It defends claims it never
|
||||
made and keeps commitments it cannot verify. `CrossModelHistoryLabel` detects the mismatch and
|
||||
contributes one short line naming the other model, so the serving model treats the earlier turns
|
||||
as inherited context.
|
||||
|
||||
```
|
||||
Note: assistant responses before this point were produced by a different model (gpt-5.2).
|
||||
Treat their claims and commitments as inherited context, not your own output.
|
||||
```
|
||||
|
||||
## Cache safety and provenance
|
||||
|
||||
The line is contributed through the capability `get_instructions` channel, so it is *ephemeral*:
|
||||
instructions are rebuilt for every request and are never stored as a message part in the run
|
||||
history. That is the cache-safe channel (a note that changes with the history must not move the
|
||||
cached message prefix), and it is the correct provenance channel besides: a note *about* the
|
||||
history must not itself become history that a later model reads back as fact. A test asserts the
|
||||
line is never a persisted message part.
|
||||
|
||||
## Minimal usage
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent
|
||||
from pydantic_ai_harness.cross_model_label import CrossModelHistoryLabel
|
||||
|
||||
agent = Agent('anthropic:claude-sonnet-4-5', capabilities=[CrossModelHistoryLabel()])
|
||||
# `history` was produced by a different model earlier:
|
||||
await agent.run('continue', message_history=history)
|
||||
```
|
||||
|
||||
## Family-level comparison
|
||||
|
||||
Identity is compared at the *family* level by default: `gpt-5.2-mini` and `gpt-5.2` are the same
|
||||
family and do not trigger the label, while `gpt-5.2` and `claude-sonnet-4-5` do. A smaller sibling
|
||||
of the same model does not carry a foreign voice, so it is not worth a note.
|
||||
|
||||
Pydantic AI profiles do not expose a first-class family key, so the default resolver derives one
|
||||
heuristically from the model name: it strips a leading provider segment, a trailing dated snapshot
|
||||
(`-2024-08-06`, `-20241022`), trailing alias markers (`-latest`, `-preview`), and size-tier
|
||||
suffixes (`-mini`, `-nano`, `-small`, `-lite`, `-tiny`). This is best-effort, not an authoritative
|
||||
taxonomy. When you need exact control, set `granularity='exact'` (compare full normalized names) or
|
||||
pass a `(model_name, provider_name) -> key` callable. The default resolver is exported as
|
||||
`model_family` so a callable can wrap it.
|
||||
|
||||
## `threshold`: handoff nudge vs provenance banner
|
||||
|
||||
- `'recent'` (default) fires only when the immediately preceding response (the most recent one
|
||||
carrying a `model_name`) is a different family. It acts as a one-shot handoff nudge: it fires on
|
||||
the first request after the model changes, then goes quiet once the serving model has added its
|
||||
own turn (its own turn becomes the most recent).
|
||||
- A float in `(0, 1]` fires when at least that fraction of all prior responses (that carry a
|
||||
`model_name`) are a different family. It acts as a persistent provenance banner: it keeps firing
|
||||
for as long as the history stays majority-foreign. When several other families are present it
|
||||
names the most common one (ties broken by most recent).
|
||||
|
||||
## FallbackModel
|
||||
|
||||
Under a `FallbackModel` the serving member is not known until it answers, so the current identity
|
||||
is taken from the most recent response's `model_name`, which records who actually served (Pydantic
|
||||
AI core [#6338](https://github.com/pydantic/pydantic-ai/pull/6338)). Before any response, it falls
|
||||
back to the wrapper's first candidate. So a mid-history failover -- early turns from candidate A,
|
||||
later turns from candidate B -- is detected against B, the model now serving.
|
||||
|
||||
## Options
|
||||
|
||||
- `granularity` (default `'family'`): `'family'`, `'exact'`, or a `(model_name, provider_name) ->
|
||||
key` callable.
|
||||
- `threshold` (default `'recent'`): `'recent'`, or a float in `(0, 1]`.
|
||||
- `format` (default `None`): override the line. Receives the run context and a `CrossModelHistory`
|
||||
summary (current family, other family, raw other name, differing/known counts); returns the line,
|
||||
or `None` to contribute nothing this request.
|
||||
|
||||
## Scope
|
||||
|
||||
- **Stateless.** It reads only `ctx.model` and `ctx.messages` each request; one instance is reusable
|
||||
across runs.
|
||||
- **Detect and disclose only.** It never mutates the message history, tool availability, or model
|
||||
settings. Unlabeled prior responses (no `model_name`) are skipped, never guessed.
|
||||
- **One line at most.** It contributes a single line per request, or nothing.
|
||||
|
||||
## Further reading
|
||||
|
||||
- [`pydantic_ai_harness.cross_model_label` source](https://github.com/pydantic/pydantic-ai-harness/tree/main/pydantic_ai_harness/cross_model_label/)
|
||||
- [Pydantic AI capabilities](/ai/core-concepts/capabilities/)
|
||||
@@ -118,6 +118,7 @@ Each capability is a self-contained battery you drop into an agent's `capabiliti
|
||||
| [Pydantic AI Docs](pydantic-ai-docs.md) | An on-demand `read_pyai_docs` tool that pulls Pydantic AI documentation into the run when the agent needs it, instead of preloading it. | -- |
|
||||
| [Compaction](compaction.md) | Keeps a run within token limits: sliding-window trimming, LLM-powered summarization of older messages, and warnings before the context or iteration ceiling is hit. | -- |
|
||||
| [Overflowing Tool Output](overflowing-tool-output.md) | Reduces an oversized tool return when it is produced -- truncate, spill to a queryable file, or summarize -- so a large payload does not persist in history and get re-sent every request. | -- |
|
||||
| [Cross-Model Label](cross-model-label.md) | Tells the model when earlier turns in the conversation history came from a different model. | -- |
|
||||
| [Step Persistence](step-persistence.md) | Saves and restores full conversation state; snapshot, resume (`continue_run`), and fork (`fork_run`) a run. | -- |
|
||||
| [Media](media.md) | Offloads large `BinaryContent` to content-addressed stores (local or S3) so big media does not bloat message history. | -- |
|
||||
| [Subagents](subagents.md) | Delegates subtasks to specialized child agents through a delegate tool. | -- |
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
{ "label": "Pydantic AI Docs", "slug": "pydantic-ai-docs" },
|
||||
{ "label": "Compaction", "slug": "compaction" },
|
||||
{ "label": "Overflowing Tool Output", "slug": "overflowing-tool-output" },
|
||||
{ "label": "Cross-Model Label", "slug": "cross-model-label" },
|
||||
{ "label": "Step Persistence", "slug": "step-persistence" },
|
||||
{ "label": "Media", "slug": "media" },
|
||||
{ "label": "Subagents", "slug": "subagents" },
|
||||
|
||||
+15
-6
@@ -1,7 +1,16 @@
|
||||
# CrossModelHistoryLabel
|
||||
# Cross-Model Label
|
||||
|
||||
Tell the model when earlier assistant turns came from a different model.
|
||||
|
||||
> [!NOTE]
|
||||
> Import this capability from its submodule. It is not re-exported from `pydantic_ai_harness`:
|
||||
>
|
||||
> ```python
|
||||
> from pydantic_ai_harness.cross_model_label import CrossModelHistoryLabel
|
||||
> ```
|
||||
|
||||
Cross-Model Label is a released, non-experimental capability. Pydantic AI Harness is still on 0.x releases, so the API may change between minor releases. See the repository [version policy](https://github.com/pydantic/pydantic-ai-harness#version-policy).
|
||||
|
||||
When a run continues a history whose assistant turns were produced by a *different* model than
|
||||
the one now serving -- a `FallbackModel` failover, a model swap between runs, an A/B handoff, a
|
||||
takeover -- the serving model otherwise reads those turns as its own. It defends claims it never
|
||||
@@ -14,10 +23,6 @@ Note: assistant responses before this point were produced by a different model (
|
||||
Treat their claims and commitments as inherited context, not your own output.
|
||||
```
|
||||
|
||||
> This capability is experimental and private. It is not re-exported from
|
||||
> `pydantic_ai_harness`; import it from its own module. Its API may change or be removed in any
|
||||
> release.
|
||||
|
||||
## Cache safety and provenance
|
||||
|
||||
The line is contributed through the capability `get_instructions` channel, so it is *ephemeral*:
|
||||
@@ -31,7 +36,7 @@ line is never a persisted message part.
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent
|
||||
from pydantic_ai_harness.experimental.cross_model_label import CrossModelHistoryLabel
|
||||
from pydantic_ai_harness.cross_model_label import CrossModelHistoryLabel
|
||||
|
||||
agent = Agent('anthropic:claude-sonnet-4-5', capabilities=[CrossModelHistoryLabel()])
|
||||
# `history` was produced by a different model earlier:
|
||||
@@ -87,3 +92,7 @@ later turns from candidate B -- is detected against B, the model now serving.
|
||||
- **Detect and disclose only.** It never mutates the message history, tool availability, or model
|
||||
settings. Unlabeled prior responses (no `model_name`) are skipped, never guessed.
|
||||
- **One line at most.** It contributes a single line per request, or nothing.
|
||||
|
||||
## Further reading
|
||||
|
||||
- [`pydantic_ai_harness.cross_model_label` source](https://github.com/pydantic/pydantic-ai-harness/tree/main/pydantic_ai_harness/cross_model_label/)
|
||||
+1
-4
@@ -1,7 +1,6 @@
|
||||
"""Tell the model when earlier turns came from a different model (private, not re-exported at top level)."""
|
||||
|
||||
from pydantic_ai_harness.experimental._warn import warn_experimental
|
||||
from pydantic_ai_harness.experimental.cross_model_label._capability import (
|
||||
from pydantic_ai_harness.cross_model_label._capability import (
|
||||
CrossModelFormatter,
|
||||
CrossModelHistory,
|
||||
CrossModelHistoryLabel,
|
||||
@@ -11,8 +10,6 @@ from pydantic_ai_harness.experimental.cross_model_label._capability import (
|
||||
normalize_model_name,
|
||||
)
|
||||
|
||||
warn_experimental('cross_model_label')
|
||||
|
||||
__all__ = [
|
||||
'CrossModelFormatter',
|
||||
'CrossModelHistory',
|
||||
+1
-1
@@ -134,7 +134,7 @@ class CrossModelHistoryLabel(AbstractCapability[AgentDepsT]):
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent
|
||||
from pydantic_ai_harness.experimental.cross_model_label import CrossModelHistoryLabel
|
||||
from pydantic_ai_harness.cross_model_label import CrossModelHistoryLabel
|
||||
|
||||
agent = Agent('anthropic:claude-sonnet-4-5', capabilities=[CrossModelHistoryLabel()])
|
||||
# `message_history` was produced by a different model earlier:
|
||||
+2
-2
@@ -28,13 +28,13 @@ from pydantic_ai.messages import (
|
||||
from pydantic_ai.models.fallback import FallbackModel
|
||||
from pydantic_ai.models.function import AgentInfo, FunctionModel
|
||||
|
||||
from pydantic_ai_harness.experimental.cross_model_label import (
|
||||
from pydantic_ai_harness.cross_model_label import (
|
||||
CrossModelHistory,
|
||||
CrossModelHistoryLabel,
|
||||
model_family,
|
||||
normalize_model_name,
|
||||
)
|
||||
from pydantic_ai_harness.experimental.cross_model_label._capability import _DEFAULT_FORMAT
|
||||
from pydantic_ai_harness.cross_model_label._capability import _DEFAULT_FORMAT
|
||||
|
||||
pytestmark = pytest.mark.anyio
|
||||
|
||||
@@ -120,6 +120,7 @@ _CAPABILITY_PAGE_META = {
|
||||
'pydantic-ai-docs.md': ('docs', 'Pydantic AI Docs'),
|
||||
'compaction.md': ('compaction', 'Compaction'),
|
||||
'overflowing-tool-output.md': ('overflowing_tool_output', 'Overflowing Tool Output'),
|
||||
'cross-model-label.md': ('cross_model_label', 'Cross-Model Label'),
|
||||
'step-persistence.md': ('step_persistence', 'Step Persistence'),
|
||||
'media.md': ('media', 'Media Externalization'),
|
||||
'subagents.md': ('subagents', 'Subagents'),
|
||||
|
||||
Reference in New Issue
Block a user