mirror of
https://github.com/pydantic/pydantic-ai-harness.git
synced 2026-07-20 10:25:35 +00:00
Public launch infrastructure: docs, GitHub infra, template (#165)
This commit is contained in:
David
@@ -0,0 +1 @@
|
||||
* @dsfaccini @DouweM @samuelcolvin @dmontagu @adtyavrdhn @Kludex
|
||||
@@ -0,0 +1,41 @@
|
||||
name: Bug Report
|
||||
description: Report a bug in an existing capability
|
||||
labels: ['bug', 'needs:triage']
|
||||
body:
|
||||
- type: input
|
||||
id: capability
|
||||
attributes:
|
||||
label: Capability
|
||||
description: Which capability is affected?
|
||||
placeholder: e.g., Guardrails, Memory
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Bug Description
|
||||
description: What happened? What did you expect?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
attributes:
|
||||
label: Minimal Reproduction
|
||||
description: Code that reproduces the issue
|
||||
render: python
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: pydantic-harness version
|
||||
placeholder: e.g., 0.1.0
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: pydantic-ai-version
|
||||
attributes:
|
||||
label: pydantic-ai version
|
||||
placeholder: e.g., 1.76.0
|
||||
validations:
|
||||
required: false
|
||||
@@ -0,0 +1,45 @@
|
||||
name: Capability Request
|
||||
description: Propose a new capability for pydantic-harness
|
||||
labels: ['capability', 'needs:triage']
|
||||
body:
|
||||
- type: input
|
||||
id: name
|
||||
attributes:
|
||||
label: Capability Name
|
||||
description: A short, descriptive name (e.g., "SlidingWindow", "ToolErrorRecovery")
|
||||
placeholder: MyCapability
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Description
|
||||
description: What does this capability do? What problem does it solve?
|
||||
placeholder: This capability allows agents to...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: use-case
|
||||
attributes:
|
||||
label: Use Case
|
||||
description: Describe a concrete scenario where this capability is useful
|
||||
placeholder: When building an agent that needs to...
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: hooks
|
||||
attributes:
|
||||
label: Hooks / Integration Points
|
||||
description: Which capability hooks would this use? (e.g., before_model_request, wrap_run, prepare_tools). See [hooks docs](https://ai.pydantic.dev/hooks/)
|
||||
placeholder: |
|
||||
- before_model_request: to inject context
|
||||
- after_run: to persist state
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: prior-art
|
||||
attributes:
|
||||
label: Prior Art / References
|
||||
description: Links to similar implementations, papers, or discussions
|
||||
validations:
|
||||
required: false
|
||||
@@ -0,0 +1,18 @@
|
||||
## Summary
|
||||
|
||||
<!-- Brief description of the changes -->
|
||||
|
||||
## Linked Issue
|
||||
|
||||
<!-- REQUIRED: Every PR must have a linked issue. Open one first if it doesn't exist. -->
|
||||
<!-- Use: Fixes #... or Closes #... -->
|
||||
|
||||
Fixes #
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] Linked issue exists and is referenced above
|
||||
- [ ] Tests added/updated for new behavior
|
||||
- [ ] `make lint && make typecheck && make test` passes locally (don't stress about CI -- we'll help)
|
||||
- [ ] No changes to `pyproject.toml` or `uv.lock` (dependency changes require a separate issue)
|
||||
- [ ] Docstrings use single backticks (not RST double backticks)
|
||||
@@ -0,0 +1,44 @@
|
||||
name: Guard Dependencies
|
||||
|
||||
on:
|
||||
pull_request_target: # zizmor: ignore[dangerous-triggers] -- This workflow only reads context.payload metadata, never checks out PR code
|
||||
branches: [main]
|
||||
paths:
|
||||
- pyproject.toml
|
||||
- uv.lock
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
check-author:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check if author is a team member
|
||||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
|
||||
with:
|
||||
script: |
|
||||
const teamMembers = ['DouweM', 'samuelcolvin', 'dmontagu', 'dsfaccini', 'adtyavrdhn', 'Kludex'];
|
||||
const author = context.payload.pull_request.user.login;
|
||||
|
||||
if (!teamMembers.includes(author)) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.payload.pull_request.number,
|
||||
body: `Claude here: This PR modifies dependency files (\`pyproject.toml\` or \`uv.lock\`), which is restricted to team members.\n\nIf you need a dependency change, please [open an issue](https://github.com/${context.repo.owner}/${context.repo.repo}/issues/new) describing what you need and why.\n\nClosing this PR automatically.`
|
||||
});
|
||||
|
||||
await github.rest.pulls.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: context.payload.pull_request.number,
|
||||
state: 'closed'
|
||||
});
|
||||
|
||||
core.setFailed('Dependency changes are restricted to team members.');
|
||||
} else {
|
||||
console.log(`Author ${author} is a team member. Allowing dependency changes.`);
|
||||
}
|
||||
@@ -28,7 +28,13 @@ jobs:
|
||||
enable-cache: true # zizmor: ignore[cache-poisoning] -- Job does not produce release artifacts and does not have sensitive permissions
|
||||
|
||||
- run: uv sync --frozen --all-groups
|
||||
- run: make lint
|
||||
|
||||
- uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # v3.0.0
|
||||
with:
|
||||
extra_args: --all-files --verbose
|
||||
env:
|
||||
SKIP: no-commit-to-branch
|
||||
|
||||
- run: make typecheck
|
||||
|
||||
test:
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
.env*
|
||||
.mcp.json
|
||||
.DS_Store
|
||||
.agents/settings.local.json
|
||||
CLAUDE.local.md
|
||||
LOCAL_WORKTREES.md
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
|
||||
@@ -17,3 +24,4 @@ wheels/
|
||||
|
||||
# Hypothesis
|
||||
.hypothesis/
|
||||
.vscode/
|
||||
|
||||
@@ -7,6 +7,15 @@ repos:
|
||||
- id: check-toml
|
||||
- id: end-of-file-fixer
|
||||
- id: trailing-whitespace
|
||||
- id: check-added-large-files
|
||||
args: [--maxkb=1024]
|
||||
exclude: uv.lock
|
||||
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.1
|
||||
hooks:
|
||||
- id: codespell
|
||||
additional_dependencies: [tomli]
|
||||
|
||||
- repo: https://github.com/google/yamlfmt
|
||||
rev: v0.21.0
|
||||
|
||||
@@ -12,4 +12,87 @@ Use this file only to persist information about the specific workstation you are
|
||||
Capabilities branch is checked out at: <absolute path to the capabilities branch checkout on this system>
|
||||
```
|
||||
|
||||
Fill in the absolute path where the `capabilities` branch of `pydantic-ai` is checked out on your system. If unknown, ask the user.
|
||||
Fill in the absolute path where the `capabilities` branch of `pydantic-ai` is checked out on your system. If unknown, ask the user.
|
||||
|
||||
## Vocabulary
|
||||
|
||||
- **Capability**: an `AbstractCapability` subclass that bundles tools, hooks, instructions, and model settings into a reusable unit. This is the core abstraction of pydantic-harness
|
||||
- **Hook**: a lifecycle method on `AbstractCapability` that intercepts agent graph execution (e.g. `before_model_request`, `wrap_run`, `after_tool_execute`)
|
||||
- **Toolset**: a collection of tools that a capability can provide to the agent
|
||||
- **Guard**: a type of capability that validates inputs/outputs or controls tool access (e.g. `InputGuardrail`, `CostGuard`)
|
||||
- **Harness**: this package -- a collection of pre-made capabilities for pydantic-ai
|
||||
- **AICA**: AI Code Assistant -- the automated agent that implements issues, reviews plans, and handles PR feedback
|
||||
- **Ralph loop**: the state-machine-based workflow that drives AICA through phases (TRIAGE -> GOALS -> PLAN -> CODE -> VERIFY -> REVIEW -> PUBLISH)
|
||||
- **DDD+ protocol**: classification system for PR review comments (do, dismiss, discuss, waiting, done)
|
||||
|
||||
## Capabilities API reference
|
||||
|
||||
When implementing a new capability, reference these docs in the pydantic-ai repo (path in `CLAUDE.local.md`):
|
||||
|
||||
- `docs/capabilities.md` -- main capabilities documentation, usage patterns, built-in capabilities
|
||||
- `docs/hooks.md` -- lifecycle hooks reference, hook ordering, all hook categories
|
||||
- `docs/extensibility.md` -- publishing capabilities as packages, spec serialization
|
||||
- `docs/toolsets.md` -- toolset abstraction, building tools for capabilities
|
||||
- `docs/tools-advanced.md` -- tool hooks, prepare_tools, tool validation
|
||||
- `docs/agent.md` -- agent configuration, instructions, model settings
|
||||
- `pydantic_ai_slim/pydantic_ai/capabilities/abstract.py` -- the `AbstractCapability` base class (all hook methods)
|
||||
- `pydantic_ai_slim/pydantic_ai/capabilities/hooks.py` -- decorator-based `Hooks` capability
|
||||
- `pydantic_ai_slim/pydantic_ai/capabilities/combined.py` -- `CombinedCapability` for composition
|
||||
|
||||
## Coding standards
|
||||
|
||||
- Python 3.10+ (target version for pyright and ruff)
|
||||
- **pyright strict** mode -- no `Any` types, full type annotations
|
||||
- **ruff**: line-length=120, single quotes, max-complexity=15
|
||||
- **100% branch coverage** required (enforced by `make testcov`)
|
||||
- docstrings use single backticks (markdown), not RST double backticks
|
||||
- no typecasting (`as` in TypeScript, `cast()` in Python) -- use type narrowing instead
|
||||
- prefer the most generic input types possible (reduce dependency chains)
|
||||
- don't add comments that restate what the code does
|
||||
|
||||
## Package management
|
||||
|
||||
- Use `uv` for all dependency operations
|
||||
- Never edit `pyproject.toml` or `uv.lock` directly -- use `uv add`, `uv remove`
|
||||
- External PRs that change dependencies are auto-closed by CI
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
make format # ruff format
|
||||
make lint # ruff check
|
||||
make typecheck # pyright strict
|
||||
make test # pytest
|
||||
make testcov # pytest with branch coverage
|
||||
```
|
||||
|
||||
Always run `make lint && make typecheck && make test` before committing.
|
||||
|
||||
## File structure
|
||||
|
||||
```
|
||||
src/pydantic_harness/
|
||||
__init__.py # public API re-exports
|
||||
guardrails.py # guardrail capabilities (InputGuardrail, OutputGuardrail, CostGuard, ToolGuard, AsyncGuardrail)
|
||||
<capability>.py # each capability gets its own module
|
||||
tests/
|
||||
conftest.py # shared fixtures (TestModel, test_agent)
|
||||
test_<capability>.py # tests mirror source modules
|
||||
examples/
|
||||
<capability>/ # runnable examples per capability
|
||||
```
|
||||
|
||||
## Testing patterns
|
||||
|
||||
- Use `pydantic_ai.models.TestModel` for all tests (no real API calls)
|
||||
- `ALLOW_MODEL_REQUESTS = False` is set globally in `conftest.py`
|
||||
- Tests use `pytest-anyio` for async support
|
||||
- Each capability test class follows: `TestCapabilityName` with methods `test_<scenario>`
|
||||
|
||||
## Contributing rules for AICAs
|
||||
|
||||
- Never change `pyproject.toml` or `uv.lock` -- if a dependency is needed, open an issue
|
||||
- Always link sources for any claims made during research
|
||||
- Run `make lint && make typecheck && make test` before every commit
|
||||
- Commit messages should summarize the "why", not the "what"
|
||||
- All GitHub comments must start with "Claude here: "
|
||||
|
||||
@@ -1,17 +1,30 @@
|
||||
# Pydantic Harness
|
||||
|
||||
Composable, reusable capabilities for [Pydantic AI](https://ai.pydantic.dev/) agents.
|
||||
[](https://github.com/pydantic/pydantic-harness/actions/workflows/main.yml?query=branch%3Amain)
|
||||
[](https://pypi.python.org/pypi/pydantic-harness)
|
||||
[](https://github.com/pydantic/pydantic-harness)
|
||||
[](https://github.com/pydantic/pydantic-harness/blob/main/LICENSE)
|
||||
|
||||
## What is it?
|
||||
**The batteries for your [Pydantic AI](https://ai.pydantic.dev/) agent.**
|
||||
|
||||
Pydantic Harness provides a library of **capabilities** -- self-contained bundles of system prompts, tools, and lifecycle hooks -- that you can attach to any Pydantic AI agent to give it new powers without writing boilerplate.
|
||||
---
|
||||
|
||||
Each capability is an [`AbstractCapability`](https://ai.pydantic.dev/capabilities/) subclass that plugs into the agent loop via Pydantic AI's capabilities API.
|
||||
Pydantic AI's [capabilities](https://ai.pydantic.dev/capabilities/) and [hooks](https://ai.pydantic.dev/hooks/) API is how you give an agent its harness -- bundles of tools, lifecycle hooks, instructions, and model settings that extend what the agent can do without any framework changes.
|
||||
|
||||
**Pydantic Harness** is the official capability library for Pydantic AI, maintained by the [Pydantic AI](https://github.com/pydantic/pydantic-ai) team. Pydantic AI core ships capabilities that require model or framework support, and capabilities fundamental to every agent -- [web search](https://ai.pydantic.dev/capabilities/#provider-adaptive-tools), [tool search](https://ai.pydantic.dev/deferred-tools/), [thinking](https://ai.pydantic.dev/capabilities/#thinking). Everything else lives here: standalone building blocks you pick and choose to turn your agent into a coding agent, a research assistant, or anything else. This is also where new capabilities start -- as they stabilize and prove themselves broadly essential, they can graduate into core.
|
||||
|
||||
The [capability matrix](#capability-matrix) tracks where we are. [Tell us what to prioritize.](#help-us-prioritize)
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install pydantic-harness
|
||||
uv add pydantic-harness
|
||||
```
|
||||
|
||||
Extras for specific capabilities:
|
||||
|
||||
```bash
|
||||
uv add "pydantic-harness[code-mode]" # CodeMode (adds the Monty sandbox)
|
||||
```
|
||||
|
||||
Requires Python 3.10+ and `pydantic-ai-slim>=1.76.0`.
|
||||
@@ -20,56 +33,106 @@ Requires Python 3.10+ and `pydantic-ai-slim>=1.76.0`.
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent
|
||||
from pydantic_harness import Memory, Skills, Compaction
|
||||
from pydantic_ai.capabilities import MCP # from the core pydantic-ai package
|
||||
from pydantic_harness import CodeMode
|
||||
|
||||
agent = Agent(
|
||||
'openai:gpt-4o',
|
||||
capabilities=[Memory(), Skills(), Compaction()],
|
||||
'anthropic:claude-sonnet-4-6',
|
||||
capabilities=[
|
||||
MCP('https://api.githubcopilot.com/mcp/'),
|
||||
CodeMode(),
|
||||
],
|
||||
)
|
||||
|
||||
result = agent.run_sync('Remember that my favourite colour is blue.')
|
||||
result = agent.run_sync('Rank the open PRs on pydantic/pydantic-harness by thumbs-up reactions. Which 5 should we merge first?')
|
||||
```
|
||||
|
||||
## Available capabilities
|
||||
[`MCP`](https://ai.pydantic.dev/capabilities/#provider-adaptive-tools) (from the core `pydantic-ai` package) connects your agent to any MCP server -- here, [GitHub's official MCP server](https://github.com/github/github-mcp-server).
|
||||
|
||||
| Capability | Description |
|
||||
|---|---|
|
||||
| AdaptiveReasoning | Dynamically adjust reasoning effort based on task complexity |
|
||||
| Approval | Require human approval before executing sensitive operations |
|
||||
| Compaction | Compress conversation history to stay within context limits |
|
||||
| FileSystem | Read, write, and navigate the local filesystem |
|
||||
| Guardrails | Validate inputs/outputs and enforce cost and tool constraints |
|
||||
| KnowsCurrentTime | Inject the current date and time into the system prompt |
|
||||
| Memory | Persistent key-value memory across agent sessions |
|
||||
| Planning | Break complex tasks into plans before execution |
|
||||
| RepoContextInjection | Inject repository structure and context into the system prompt |
|
||||
| SecretMasking | Detect and redact secrets in agent inputs and outputs |
|
||||
| SessionPersistence | Save and restore full conversation sessions |
|
||||
| Shell | Execute shell commands with safety controls |
|
||||
| Skills | Progressive tool loading via search and activate |
|
||||
| SlidingWindow | Keep conversation history within a sliding token window |
|
||||
| StuckLoopDetection | Detect and break out of repetitive agent loops |
|
||||
| SubAgent | Delegate subtasks to specialised child agents |
|
||||
| SystemReminders | Inject periodic reminders into the conversation |
|
||||
| ToolErrorRecovery | Automatically retry or recover from tool execution errors |
|
||||
| ToolOrphanRepair | Repair orphaned tool calls in conversation history |
|
||||
| ToolOutputManagement | Control and format tool output for the model |
|
||||
[`CodeMode`](code_mode/) wraps all tools into a single `run_code` tool powered by our [Monty](https://github.com/pydantic/monty) sandbox, so the model can orchestrate multiple tool calls with Python code instead of one model round-trip per call.
|
||||
|
||||
## Documentation
|
||||
## Capability matrix
|
||||
|
||||
- [Pydantic AI docs](https://ai.pydantic.dev/)
|
||||
- [Capabilities API](https://ai.pydantic.dev/capabilities/)
|
||||
We studied the leading coding agents and AI harnesses -- [Claude Code](https://docs.anthropic.com/en/docs/claude-code), [OpenClaw](https://github.com/open-claw/open-claw), [Goose](https://block.github.io/goose/), [Hermes](https://github.com/NousResearch/hermes-agent), [OpenAI Codex](https://openai.com/index/codex/), [SWE-agent](https://github.com/SWE-agent/SWE-agent), [OpenHands](https://github.com/All-Hands-AI/OpenHands), and [Mastra](https://mastra.ai/) -- to map every capability area that matters for production agents. Each one is tracked as an [issue](https://github.com/pydantic/pydantic-harness/issues) in this repo.
|
||||
|
||||
## Development
|
||||
**Vote on whatever is linked in the Status column** -- PRs if we're actively building it, issues if it's planned -- to help us decide what to work on next.
|
||||
|
||||
| Category | Capability | Description | Status | Community alternatives |
|
||||
|---|---|---|---|---|
|
||||
| **Tools & execution** | **Code mode** | Sandboxed Python execution via [Monty](https://github.com/pydantic/monty) -- one `run_code` call replaces N tool calls | :white_check_mark: [Docs](code_mode/) | |
|
||||
| | **Tool search** | Progressive tool discovery for large tool sets | :white_check_mark: [Pydantic AI](https://ai.pydantic.dev/deferred-tools/) | |
|
||||
| | **File system** | Read, write, edit, search files with path traversal prevention | :construction: [PR #139](https://github.com/pydantic/pydantic-harness/pull/139) | [pydantic-ai-backend](https://github.com/vstorm-co/pydantic-ai-backend) (vstorm‑co) |
|
||||
| | **Shell** | Execute commands with allowlists, denylists, and timeouts | :construction: [PR #139](https://github.com/pydantic/pydantic-harness/pull/139) | [pydantic-ai-backend](https://github.com/vstorm-co/pydantic-ai-backend) (vstorm‑co) |
|
||||
| | **Repo context injection** | Auto-load CLAUDE.md/AGENTS.md and repo structure | :construction: [PR #154](https://github.com/pydantic/pydantic-harness/pull/154) | [pydantic-deep](https://github.com/vstorm-co/pydantic-deepagents) (vstorm‑co) |
|
||||
| | **Verification loop** | Run tests after edits, auto-fix failures | :construction: [PR #160](https://github.com/pydantic/pydantic-harness/pull/160) | |
|
||||
| **Context management** | **Sliding window** | Trim conversation history to stay within token limits | :construction: [PR #140](https://github.com/pydantic/pydantic-harness/pull/140) | [summarization-pydantic-ai](https://github.com/vstorm-co/summarization-pydantic-ai) (vstorm‑co) |
|
||||
| | **Context compaction** | LLM-powered summarization of older messages | :construction: [PR #140](https://github.com/pydantic/pydantic-harness/pull/140) | [summarization-pydantic-ai](https://github.com/vstorm-co/summarization-pydantic-ai) (vstorm‑co) |
|
||||
| | **Limit warnings** | Warn agent before hitting context/iteration limits | :construction: [PR #140](https://github.com/pydantic/pydantic-harness/pull/140) | [summarization-pydantic-ai](https://github.com/vstorm-co/summarization-pydantic-ai) (vstorm‑co) |
|
||||
| | **Tool output management** | Truncate, summarize, or spill large tool outputs | :construction: [PR #131](https://github.com/pydantic/pydantic-harness/pull/131) | |
|
||||
| | **System reminders** | Inject periodic reminders to counteract instruction drift | :construction: [PR #135](https://github.com/pydantic/pydantic-harness/pull/135) | |
|
||||
| **Memory & persistence** | **Memory** | Persistent key-value memory across sessions | :construction: [PR #137](https://github.com/pydantic/pydantic-harness/pull/137) | [memv](https://github.com/vstorm-co/memv) (vstorm‑co) |
|
||||
| | **Session persistence** | Save and restore full conversation state | :construction: [PR #153](https://github.com/pydantic/pydantic-harness/pull/153) | |
|
||||
| **Agent orchestration** | **Sub-agents** | Delegate subtasks to specialized child agents | :construction: [PR #138](https://github.com/pydantic/pydantic-harness/pull/138) | [subagents-pydantic-ai](https://github.com/vstorm-co/subagents-pydantic-ai) (vstorm‑co) |
|
||||
| | **Skills** | Progressive tool loading -- search, activate, deactivate | :construction: [PR #133](https://github.com/pydantic/pydantic-harness/pull/133) | [pydantic-ai-skills](https://github.com/DougTrajano/pydantic-ai-skills) (DougTrajano) |
|
||||
| | **Planning** | Break complex tasks into structured plans before execution | :construction: [PR #136](https://github.com/pydantic/pydantic-harness/pull/136) | |
|
||||
| | **Task tracking** | Track tasks, subtasks, and dependencies | :memo: [#65](https://github.com/pydantic/pydantic-harness/issues/65) | [pydantic-ai-todo](https://github.com/vstorm-co/pydantic-ai-todo) (vstorm‑co) |
|
||||
| **Safety & guardrails** | **Input guardrails** | Validate user input before the agent run starts | :construction: [PR #134](https://github.com/pydantic/pydantic-harness/pull/134) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) |
|
||||
| | **Output guardrails** | Validate model output after the run completes | :construction: [PR #134](https://github.com/pydantic/pydantic-harness/pull/134) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) |
|
||||
| | **Cost/token budgets** | Enforce token and cost limits per run | :construction: [PR #134](https://github.com/pydantic/pydantic-harness/pull/134) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) |
|
||||
| | **Tool access control** | Block tools or require approval before execution | :construction: [PR #134](https://github.com/pydantic/pydantic-harness/pull/134) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) |
|
||||
| | **Async guardrails** | Run validation concurrently with model requests | :construction: [PR #134](https://github.com/pydantic/pydantic-harness/pull/134) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) |
|
||||
| | **Secret masking** | Detect and redact secrets in agent I/O | :construction: [PR #157](https://github.com/pydantic/pydantic-harness/pull/157) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) |
|
||||
| | **Approval workflows** | Require human approval for sensitive operations | :construction: [PR #156](https://github.com/pydantic/pydantic-harness/pull/156) | |
|
||||
| | **Tool budget** | Limit total tool calls or cost per run | :construction: [PR #161](https://github.com/pydantic/pydantic-harness/pull/161) | |
|
||||
| **Reliability** | **Stuck loop detection** | Detect and break out of repetitive agent loops | :construction: [PR #130](https://github.com/pydantic/pydantic-harness/pull/130) | |
|
||||
| | **Tool error recovery** | Retry failed tool calls with backoff and budget | :construction: [PR #158](https://github.com/pydantic/pydantic-harness/pull/158) | |
|
||||
| | **Tool orphan repair** | Fix orphaned tool calls in conversation history | :construction: [PR #132](https://github.com/pydantic/pydantic-harness/pull/132) | |
|
||||
| **Reasoning** | **Adaptive reasoning** | Adjust thinking effort based on task complexity | :construction: [PR #155](https://github.com/pydantic/pydantic-harness/pull/155) | |
|
||||
| | **Current time** | Inject current date/time into system prompt | :construction: [PR #159](https://github.com/pydantic/pydantic-harness/pull/159) | |
|
||||
|
||||
> Packages by [vstorm-co](https://github.com/vstorm-co) are endorsed by the Pydantic AI team. We're working with them to upstream some of their implementations into this repo.
|
||||
|
||||
## Help us prioritize
|
||||
|
||||
**Vote on whatever is linked in the Status column above.** If there's a PR, vote on the PR -- it means we're actively building it. If there's only an issue, vote on the issue.
|
||||
|
||||
Want something that's not on the list? [Open a capability request](https://github.com/pydantic/pydantic-harness/issues/new?template=capability-request.yml).
|
||||
|
||||
## Build your own
|
||||
|
||||
[Capabilities](https://ai.pydantic.dev/capabilities/#building-custom-capabilities) are the primary extension point for Pydantic AI. Any of the existing capabilities in this repo can serve as a reference for building your own.
|
||||
|
||||
**Publishing as a standalone package?** Use the `pydantic-ai-<name>` naming convention. See [Publishing capability packages](https://ai.pydantic.dev/extensibility/#publishing-capability-packages).
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome capability contributions. Here's how:
|
||||
|
||||
1. **Start with an issue.** [Open a capability request](https://github.com/pydantic/pydantic-harness/issues/new?template=capability-request.yml) describing the behavior you want. This lets us discuss the approach and priority before code is written -- we can close an approach without closing the problem.
|
||||
2. **Then open a PR.** Once the issue exists, you're welcome to open a PR with an implementation. Link the issue in your PR. We review based on community interest -- upvotes on both the issue and PR count.
|
||||
3. **Don't chase green CI.** Get the approach working, then let us know. We'll take it from there -- we may push to your branch, rewrite, or open a follow-up PR. You'll be credited as the original author. (See the [Pydantic AI contributing guide](https://github.com/pydantic/pydantic-ai/blob/main/CONTRIBUTING.md).)
|
||||
|
||||
> **Note**: PRs that modify `pyproject.toml` or `uv.lock` from non-team members are auto-closed by CI to prevent supply chain risk. If you need a new dependency, [open an issue](https://github.com/pydantic/pydantic-harness/issues/new).
|
||||
|
||||
### Development
|
||||
|
||||
```bash
|
||||
make install # install dependencies
|
||||
make lint # ruff format check + lint
|
||||
make format # ruff format
|
||||
make lint # ruff check
|
||||
make typecheck # pyright strict
|
||||
make test # pytest
|
||||
make testcov # pytest with coverage
|
||||
make testcov # pytest with 100% branch coverage
|
||||
```
|
||||
|
||||
## Pydantic AI references
|
||||
|
||||
- [Capabilities](https://ai.pydantic.dev/capabilities/) -- what capabilities are, built-in capabilities, building your own
|
||||
- [Hooks](https://ai.pydantic.dev/hooks/) -- lifecycle hooks reference, ordering, error handling
|
||||
- [Extensibility](https://ai.pydantic.dev/extensibility/) -- publishing packages, third-party ecosystem
|
||||
- [Toolsets](https://ai.pydantic.dev/toolsets/) -- building tools for capabilities
|
||||
- [API reference](https://ai.pydantic.dev/api/capabilities/) -- full API docs
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
MIT -- see [LICENSE](LICENSE).
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
# Code Mode
|
||||
|
||||
Replace individual tool calls with a single sandboxed Python execution environment.
|
||||
|
||||
## The problem
|
||||
|
||||
Standard tool calling requires one model round-trip per tool call. An agent that needs to fetch 10 items and process each one makes 11+ model calls -- slow, expensive, and context-heavy.
|
||||
|
||||
## The solution
|
||||
|
||||
`CodeMode` wraps your tools into a single `run_code` tool. The model writes Python code that calls multiple tools with loops, conditionals, variables, and `asyncio.gather` -- all inside a sandboxed [Monty](https://github.com/pydantic/monty) runtime.
|
||||
|
||||
| Standard tool calling | Code mode |
|
||||
|---|---|
|
||||
| 1 model call per tool | 1 model call for N tools |
|
||||
| Sequential by default | Parallel via `asyncio.gather` |
|
||||
| No local computation | Filter, transform, aggregate in code |
|
||||
| Large conversation history | Compact -- fewer messages |
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent
|
||||
from pydantic_harness import CodeMode
|
||||
|
||||
agent = Agent('anthropic:claude-sonnet-4-6', capabilities=[CodeMode()])
|
||||
|
||||
@agent.tool_plain
|
||||
def get_weather(city: str) -> dict:
|
||||
"""Get current weather for a city."""
|
||||
return {'city': city, 'temp_f': 72, 'condition': 'sunny'}
|
||||
|
||||
@agent.tool_plain
|
||||
def convert_temp(fahrenheit: float) -> float:
|
||||
"""Convert Fahrenheit to Celsius."""
|
||||
return round((fahrenheit - 32) * 5 / 9, 1)
|
||||
|
||||
result = agent.run_sync("What's the weather in Paris and Tokyo, in Celsius?")
|
||||
```
|
||||
|
||||
The model writes code like:
|
||||
|
||||
```python
|
||||
paris, tokyo = await asyncio.gather(
|
||||
get_weather(city='Paris'),
|
||||
get_weather(city='Tokyo'),
|
||||
)
|
||||
paris_c = await convert_temp(fahrenheit=paris['temp_f'])
|
||||
tokyo_c = await convert_temp(fahrenheit=tokyo['temp_f'])
|
||||
{'paris': paris_c, 'tokyo': tokyo_c}
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
Code mode requires the Monty sandbox:
|
||||
|
||||
```bash
|
||||
uv add "pydantic-harness[code-mode]"
|
||||
```
|
||||
|
||||
## Selective tool sandboxing
|
||||
|
||||
By default, `CodeMode(tools='all')` sandboxes every tool. You can control which tools go through the sandbox:
|
||||
|
||||
```python
|
||||
# By name -- only these tools are available inside run_code
|
||||
CodeMode(tools=['search', 'fetch'])
|
||||
|
||||
# By predicate
|
||||
CodeMode(tools=lambda ctx, td: td.name != 'dangerous_tool')
|
||||
|
||||
# By metadata -- combine with SetToolMetadata or .with_metadata()
|
||||
CodeMode(tools={'code_mode': True})
|
||||
```
|
||||
|
||||
Tools that match the selector are wrapped inside `run_code`. Non-matching tools remain available as regular tool calls.
|
||||
|
||||
### Metadata-based selection
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent
|
||||
from pydantic_ai.toolsets import FunctionToolset
|
||||
from pydantic_harness import CodeMode
|
||||
|
||||
search_tools = FunctionToolset(tools=[search, fetch]).with_metadata(code_mode=True)
|
||||
|
||||
agent = Agent(
|
||||
'anthropic:claude-sonnet-4-6',
|
||||
toolsets=[search_tools],
|
||||
capabilities=[CodeMode(tools={'code_mode': True})],
|
||||
)
|
||||
```
|
||||
|
||||
## Return values
|
||||
|
||||
The last expression in the code snippet is automatically captured as the return value -- the model does not need to `print()`.
|
||||
|
||||
| Scenario | Return |
|
||||
|---|---|
|
||||
| No print output | Last expression value |
|
||||
| With print output | `{"output": "<printed text>", "result": <last expression>}` |
|
||||
| Multimodal content (e.g. images) | Returned natively for model processing |
|
||||
|
||||
## REPL state
|
||||
|
||||
State persists between `run_code` calls within the same agent run -- variables, imports, and function definitions carry over. Pass `restart: true` in the tool call to reset state.
|
||||
|
||||
## Observability
|
||||
|
||||
Nested tool calls inside `run_code` produce their own spans when instrumented with [Logfire](https://pydantic.dev/logfire) or any OpenTelemetry backend. The `run_code` tool return includes metadata with all nested calls:
|
||||
|
||||
```python
|
||||
for msg in result.all_messages():
|
||||
for part in msg.parts:
|
||||
if isinstance(part, ToolReturnPart) and part.tool_name == 'run_code':
|
||||
tool_calls = part.metadata['tool_calls'] # dict[str, ToolCallPart]
|
||||
tool_returns = part.metadata['tool_returns'] # dict[str, ToolReturnPart]
|
||||
```
|
||||
|
||||
## Sandbox restrictions
|
||||
|
||||
Code runs inside [Monty](https://github.com/pydantic/monty), a sandboxed Python subset. Key restrictions:
|
||||
|
||||
- No class definitions
|
||||
- No third-party imports (allowed stdlib: `sys`, `typing`, `asyncio`, `math`, `json`, `re`, `datetime`, `os`, `pathlib`)
|
||||
- No `import *`
|
||||
- Tools requiring approval or with deferred execution are excluded from the sandbox
|
||||
|
||||
## API
|
||||
|
||||
```python
|
||||
CodeMode(
|
||||
tools: ToolSelector = 'all', # 'all', list[str], callable, or dict
|
||||
max_retries: int = 3, # retries on sandbox execution errors
|
||||
)
|
||||
```
|
||||
|
||||
## Agent spec (YAML/JSON)
|
||||
|
||||
CodeMode works with Pydantic AI's [agent spec](https://ai.pydantic.dev/agent-spec/) feature for defining agents in YAML:
|
||||
|
||||
```yaml
|
||||
# agent.yaml
|
||||
model: anthropic:claude-sonnet-4-6
|
||||
capabilities:
|
||||
- CodeMode: {}
|
||||
```
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent
|
||||
from pydantic_harness import CodeMode
|
||||
|
||||
agent = Agent.from_file('agent.yaml', custom_capability_types=[CodeMode])
|
||||
```
|
||||
|
||||
Pass `custom_capability_types` so the spec loader knows how to instantiate `CodeMode`. You can also pass arguments in the YAML:
|
||||
|
||||
```yaml
|
||||
capabilities:
|
||||
- CodeMode:
|
||||
tools: ['search', 'fetch']
|
||||
max_retries: 5
|
||||
```
|
||||
|
||||
## Further reading
|
||||
|
||||
- [Tool use via code](https://www.anthropic.com/engineering/code-execution-with-mcp) (Anthropic)
|
||||
- [Code mode in production](https://blog.cloudflare.com/code-mode/) (Cloudflare)
|
||||
- [Pydantic AI capabilities](https://ai.pydantic.dev/capabilities/)
|
||||
@@ -1,29 +0,0 @@
|
||||
# Capabilities
|
||||
|
||||
Each capability is an `AbstractCapability` subclass that can be attached to any
|
||||
Pydantic AI agent via the `capabilities` parameter.
|
||||
|
||||
| Capability | Description |
|
||||
|---|---|
|
||||
| AdaptiveReasoning | Dynamically adjust reasoning effort based on task complexity |
|
||||
| Approval | Require human approval before executing sensitive operations |
|
||||
| Compaction | Compress conversation history to stay within context limits |
|
||||
| FileSystem | Read, write, and navigate the local filesystem |
|
||||
| Guardrails | Validate inputs/outputs and enforce cost and tool constraints |
|
||||
| KnowsCurrentTime | Inject the current date and time into the system prompt |
|
||||
| Memory | Persistent key-value memory across agent sessions |
|
||||
| Planning | Break complex tasks into plans before execution |
|
||||
| RepoContextInjection | Inject repository structure and context into the system prompt |
|
||||
| SecretMasking | Detect and redact secrets in agent inputs and outputs |
|
||||
| SessionPersistence | Save and restore full conversation sessions |
|
||||
| Shell | Execute shell commands with safety controls |
|
||||
| Skills | Progressive tool loading via search and activate |
|
||||
| SlidingWindow | Keep conversation history within a sliding token window |
|
||||
| StuckLoopDetection | Detect and break out of repetitive agent loops |
|
||||
| SubAgent | Delegate subtasks to specialised child agents |
|
||||
| SystemReminders | Inject periodic reminders into the conversation |
|
||||
| ToolErrorRecovery | Automatically retry or recover from tool execution errors |
|
||||
| ToolOrphanRepair | Repair orphaned tool calls in conversation history |
|
||||
| ToolOutputManagement | Control and format tool output for the model |
|
||||
|
||||
Detailed documentation for each capability will be added as they are merged.
|
||||
@@ -1,36 +0,0 @@
|
||||
# Pydantic Harness
|
||||
|
||||
Composable, reusable capabilities for [Pydantic AI](https://ai.pydantic.dev/) agents.
|
||||
|
||||
## What is it?
|
||||
|
||||
Pydantic Harness provides a library of **capabilities** -- self-contained bundles of
|
||||
system prompts, tools, and lifecycle hooks -- that you can attach to any Pydantic AI
|
||||
agent to give it new powers without writing boilerplate.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install pydantic-harness
|
||||
```
|
||||
|
||||
Or with `uv`:
|
||||
|
||||
```bash
|
||||
uv add pydantic-harness
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
```python
|
||||
from pydantic_ai import Agent
|
||||
from pydantic_harness import Memory, Skills
|
||||
|
||||
agent = Agent('openai:gpt-4o', capabilities=[Memory(), Skills()])
|
||||
```
|
||||
|
||||
## Learn more
|
||||
|
||||
- [Available capabilities](capabilities/index.md)
|
||||
- [Pydantic AI documentation](https://ai.pydantic.dev/)
|
||||
- [GitHub repository](https://github.com/pydantic/pydantic-harness)
|
||||
@@ -57,6 +57,7 @@ packages = ['src/pydantic_harness']
|
||||
[tool.ruff]
|
||||
line-length = 120
|
||||
target-version = 'py310'
|
||||
exclude = ['template']
|
||||
|
||||
[tool.ruff.lint]
|
||||
extend-select = ['Q', 'RUF100', 'C90', 'UP', 'I', 'D', 'TID251']
|
||||
@@ -79,11 +80,13 @@ quote-style = 'single'
|
||||
[tool.pyright]
|
||||
pythonVersion = '3.10'
|
||||
typeCheckingMode = 'strict'
|
||||
exclude = ['template', '.venv']
|
||||
executionEnvironments = [
|
||||
{ root = 'tests', reportPrivateUsage = false },
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ['tests']
|
||||
xfail_strict = true
|
||||
filterwarnings = ['error']
|
||||
anyio_mode = 'auto'
|
||||
|
||||
Reference in New Issue
Block a user