diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..2548a97 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @dsfaccini @DouweM @samuelcolvin @dmontagu @adtyavrdhn @Kludex diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000..e583c91 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,41 @@ +name: Bug Report +description: Report a bug in an existing capability +labels: ['bug', 'needs:triage'] +body: + - type: input + id: capability + attributes: + label: Capability + description: Which capability is affected? + placeholder: e.g., Guardrails, Memory + validations: + required: true + - type: textarea + id: description + attributes: + label: Bug Description + description: What happened? What did you expect? + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Minimal Reproduction + description: Code that reproduces the issue + render: python + validations: + required: true + - type: input + id: version + attributes: + label: pydantic-harness version + placeholder: e.g., 0.1.0 + validations: + required: true + - type: input + id: pydantic-ai-version + attributes: + label: pydantic-ai version + placeholder: e.g., 1.76.0 + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/capability-request.yml b/.github/ISSUE_TEMPLATE/capability-request.yml new file mode 100644 index 0000000..c49a2e6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/capability-request.yml @@ -0,0 +1,45 @@ +name: Capability Request +description: Propose a new capability for pydantic-harness +labels: ['capability', 'needs:triage'] +body: + - type: input + id: name + attributes: + label: Capability Name + description: A short, descriptive name (e.g., "SlidingWindow", "ToolErrorRecovery") + placeholder: MyCapability + validations: + required: true + - type: textarea + id: description + attributes: + label: Description + description: What does this capability do? What problem does it solve? + placeholder: This capability allows agents to... + validations: + required: true + - type: textarea + id: use-case + attributes: + label: Use Case + description: Describe a concrete scenario where this capability is useful + placeholder: When building an agent that needs to... + validations: + required: true + - type: textarea + id: hooks + attributes: + label: Hooks / Integration Points + description: Which capability hooks would this use? (e.g., before_model_request, wrap_run, prepare_tools). See [hooks docs](https://ai.pydantic.dev/hooks/) + placeholder: | + - before_model_request: to inject context + - after_run: to persist state + validations: + required: false + - type: textarea + id: prior-art + attributes: + label: Prior Art / References + description: Links to similar implementations, papers, or discussions + validations: + required: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..bb94f4c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,18 @@ +## Summary + + + +## Linked Issue + + + + +Fixes # + +## Checklist + +- [ ] Linked issue exists and is referenced above +- [ ] Tests added/updated for new behavior +- [ ] `make lint && make typecheck && make test` passes locally (don't stress about CI -- we'll help) +- [ ] No changes to `pyproject.toml` or `uv.lock` (dependency changes require a separate issue) +- [ ] Docstrings use single backticks (not RST double backticks) diff --git a/.github/workflows/guard-dependencies.yml b/.github/workflows/guard-dependencies.yml new file mode 100644 index 0000000..e68de9f --- /dev/null +++ b/.github/workflows/guard-dependencies.yml @@ -0,0 +1,44 @@ +name: Guard Dependencies + +on: + pull_request_target: # zizmor: ignore[dangerous-triggers] -- This workflow only reads context.payload metadata, never checks out PR code + branches: [main] + paths: + - pyproject.toml + - uv.lock + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + check-author: + runs-on: ubuntu-latest + steps: + - name: Check if author is a team member + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + const teamMembers = ['DouweM', 'samuelcolvin', 'dmontagu', 'dsfaccini', 'adtyavrdhn', 'Kludex']; + const author = context.payload.pull_request.user.login; + + if (!teamMembers.includes(author)) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: `Claude here: This PR modifies dependency files (\`pyproject.toml\` or \`uv.lock\`), which is restricted to team members.\n\nIf you need a dependency change, please [open an issue](https://github.com/${context.repo.owner}/${context.repo.repo}/issues/new) describing what you need and why.\n\nClosing this PR automatically.` + }); + + await github.rest.pulls.update({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + state: 'closed' + }); + + core.setFailed('Dependency changes are restricted to team members.'); + } else { + console.log(`Author ${author} is a team member. Allowing dependency changes.`); + } diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1a7ea54..627b46e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,13 @@ jobs: enable-cache: true # zizmor: ignore[cache-poisoning] -- Job does not produce release artifacts and does not have sensitive permissions - run: uv sync --frozen --all-groups - - run: make lint + + - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # v3.0.0 + with: + extra_args: --all-files --verbose + env: + SKIP: no-commit-to-branch + - run: make typecheck test: diff --git a/.gitignore b/.gitignore index 00e73a6..a54049e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +.env* +.mcp.json +.DS_Store +.agents/settings.local.json +CLAUDE.local.md +LOCAL_WORKTREES.md + # IDE .idea/ @@ -17,3 +24,4 @@ wheels/ # Hypothesis .hypothesis/ +.vscode/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b2c4e3c..73a1daf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,15 @@ repos: - id: check-toml - id: end-of-file-fixer - id: trailing-whitespace + - id: check-added-large-files + args: [--maxkb=1024] + exclude: uv.lock + + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + additional_dependencies: [tomli] - repo: https://github.com/google/yamlfmt rev: v0.21.0 diff --git a/AGENTS.md b/AGENTS.md index 7555a28..7fd41db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,4 +12,87 @@ Use this file only to persist information about the specific workstation you are Capabilities branch is checked out at: ``` -Fill in the absolute path where the `capabilities` branch of `pydantic-ai` is checked out on your system. If unknown, ask the user. \ No newline at end of file +Fill in the absolute path where the `capabilities` branch of `pydantic-ai` is checked out on your system. If unknown, ask the user. + +## Vocabulary + +- **Capability**: an `AbstractCapability` subclass that bundles tools, hooks, instructions, and model settings into a reusable unit. This is the core abstraction of pydantic-harness +- **Hook**: a lifecycle method on `AbstractCapability` that intercepts agent graph execution (e.g. `before_model_request`, `wrap_run`, `after_tool_execute`) +- **Toolset**: a collection of tools that a capability can provide to the agent +- **Guard**: a type of capability that validates inputs/outputs or controls tool access (e.g. `InputGuardrail`, `CostGuard`) +- **Harness**: this package -- a collection of pre-made capabilities for pydantic-ai +- **AICA**: AI Code Assistant -- the automated agent that implements issues, reviews plans, and handles PR feedback +- **Ralph loop**: the state-machine-based workflow that drives AICA through phases (TRIAGE -> GOALS -> PLAN -> CODE -> VERIFY -> REVIEW -> PUBLISH) +- **DDD+ protocol**: classification system for PR review comments (do, dismiss, discuss, waiting, done) + +## Capabilities API reference + +When implementing a new capability, reference these docs in the pydantic-ai repo (path in `CLAUDE.local.md`): + +- `docs/capabilities.md` -- main capabilities documentation, usage patterns, built-in capabilities +- `docs/hooks.md` -- lifecycle hooks reference, hook ordering, all hook categories +- `docs/extensibility.md` -- publishing capabilities as packages, spec serialization +- `docs/toolsets.md` -- toolset abstraction, building tools for capabilities +- `docs/tools-advanced.md` -- tool hooks, prepare_tools, tool validation +- `docs/agent.md` -- agent configuration, instructions, model settings +- `pydantic_ai_slim/pydantic_ai/capabilities/abstract.py` -- the `AbstractCapability` base class (all hook methods) +- `pydantic_ai_slim/pydantic_ai/capabilities/hooks.py` -- decorator-based `Hooks` capability +- `pydantic_ai_slim/pydantic_ai/capabilities/combined.py` -- `CombinedCapability` for composition + +## Coding standards + +- Python 3.10+ (target version for pyright and ruff) +- **pyright strict** mode -- no `Any` types, full type annotations +- **ruff**: line-length=120, single quotes, max-complexity=15 +- **100% branch coverage** required (enforced by `make testcov`) +- docstrings use single backticks (markdown), not RST double backticks +- no typecasting (`as` in TypeScript, `cast()` in Python) -- use type narrowing instead +- prefer the most generic input types possible (reduce dependency chains) +- don't add comments that restate what the code does + +## Package management + +- Use `uv` for all dependency operations +- Never edit `pyproject.toml` or `uv.lock` directly -- use `uv add`, `uv remove` +- External PRs that change dependencies are auto-closed by CI + +## Commands + +```bash +make format # ruff format +make lint # ruff check +make typecheck # pyright strict +make test # pytest +make testcov # pytest with branch coverage +``` + +Always run `make lint && make typecheck && make test` before committing. + +## File structure + +``` +src/pydantic_harness/ + __init__.py # public API re-exports + guardrails.py # guardrail capabilities (InputGuardrail, OutputGuardrail, CostGuard, ToolGuard, AsyncGuardrail) + .py # each capability gets its own module +tests/ + conftest.py # shared fixtures (TestModel, test_agent) + test_.py # tests mirror source modules +examples/ + / # runnable examples per capability +``` + +## Testing patterns + +- Use `pydantic_ai.models.TestModel` for all tests (no real API calls) +- `ALLOW_MODEL_REQUESTS = False` is set globally in `conftest.py` +- Tests use `pytest-anyio` for async support +- Each capability test class follows: `TestCapabilityName` with methods `test_` + +## Contributing rules for AICAs + +- Never change `pyproject.toml` or `uv.lock` -- if a dependency is needed, open an issue +- Always link sources for any claims made during research +- Run `make lint && make typecheck && make test` before every commit +- Commit messages should summarize the "why", not the "what" +- All GitHub comments must start with "Claude here: " diff --git a/README.md b/README.md index f0ba88f..4230d2e 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,30 @@ # Pydantic Harness -Composable, reusable capabilities for [Pydantic AI](https://ai.pydantic.dev/) agents. +[![CI](https://github.com/pydantic/pydantic-harness/actions/workflows/main.yml/badge.svg?event=push)](https://github.com/pydantic/pydantic-harness/actions/workflows/main.yml?query=branch%3Amain) +[![PyPI](https://img.shields.io/pypi/v/pydantic-harness.svg)](https://pypi.python.org/pypi/pydantic-harness) +[![versions](https://img.shields.io/pypi/pyversions/pydantic-harness.svg)](https://github.com/pydantic/pydantic-harness) +[![license](https://img.shields.io/github/license/pydantic/pydantic-harness.svg)](https://github.com/pydantic/pydantic-harness/blob/main/LICENSE) -## What is it? +**The batteries for your [Pydantic AI](https://ai.pydantic.dev/) agent.** -Pydantic Harness provides a library of **capabilities** -- self-contained bundles of system prompts, tools, and lifecycle hooks -- that you can attach to any Pydantic AI agent to give it new powers without writing boilerplate. +--- -Each capability is an [`AbstractCapability`](https://ai.pydantic.dev/capabilities/) subclass that plugs into the agent loop via Pydantic AI's capabilities API. +Pydantic AI's [capabilities](https://ai.pydantic.dev/capabilities/) and [hooks](https://ai.pydantic.dev/hooks/) API is how you give an agent its harness -- bundles of tools, lifecycle hooks, instructions, and model settings that extend what the agent can do without any framework changes. + +**Pydantic Harness** is the official capability library for Pydantic AI, maintained by the [Pydantic AI](https://github.com/pydantic/pydantic-ai) team. Pydantic AI core ships capabilities that require model or framework support, and capabilities fundamental to every agent -- [web search](https://ai.pydantic.dev/capabilities/#provider-adaptive-tools), [tool search](https://ai.pydantic.dev/deferred-tools/), [thinking](https://ai.pydantic.dev/capabilities/#thinking). Everything else lives here: standalone building blocks you pick and choose to turn your agent into a coding agent, a research assistant, or anything else. This is also where new capabilities start -- as they stabilize and prove themselves broadly essential, they can graduate into core. + +The [capability matrix](#capability-matrix) tracks where we are. [Tell us what to prioritize.](#help-us-prioritize) ## Installation ```bash -pip install pydantic-harness +uv add pydantic-harness +``` + +Extras for specific capabilities: + +```bash +uv add "pydantic-harness[code-mode]" # CodeMode (adds the Monty sandbox) ``` Requires Python 3.10+ and `pydantic-ai-slim>=1.76.0`. @@ -20,56 +33,106 @@ Requires Python 3.10+ and `pydantic-ai-slim>=1.76.0`. ```python from pydantic_ai import Agent -from pydantic_harness import Memory, Skills, Compaction +from pydantic_ai.capabilities import MCP # from the core pydantic-ai package +from pydantic_harness import CodeMode agent = Agent( - 'openai:gpt-4o', - capabilities=[Memory(), Skills(), Compaction()], + 'anthropic:claude-sonnet-4-6', + capabilities=[ + MCP('https://api.githubcopilot.com/mcp/'), + CodeMode(), + ], ) -result = agent.run_sync('Remember that my favourite colour is blue.') +result = agent.run_sync('Rank the open PRs on pydantic/pydantic-harness by thumbs-up reactions. Which 5 should we merge first?') ``` -## Available capabilities +[`MCP`](https://ai.pydantic.dev/capabilities/#provider-adaptive-tools) (from the core `pydantic-ai` package) connects your agent to any MCP server -- here, [GitHub's official MCP server](https://github.com/github/github-mcp-server). -| Capability | Description | -|---|---| -| AdaptiveReasoning | Dynamically adjust reasoning effort based on task complexity | -| Approval | Require human approval before executing sensitive operations | -| Compaction | Compress conversation history to stay within context limits | -| FileSystem | Read, write, and navigate the local filesystem | -| Guardrails | Validate inputs/outputs and enforce cost and tool constraints | -| KnowsCurrentTime | Inject the current date and time into the system prompt | -| Memory | Persistent key-value memory across agent sessions | -| Planning | Break complex tasks into plans before execution | -| RepoContextInjection | Inject repository structure and context into the system prompt | -| SecretMasking | Detect and redact secrets in agent inputs and outputs | -| SessionPersistence | Save and restore full conversation sessions | -| Shell | Execute shell commands with safety controls | -| Skills | Progressive tool loading via search and activate | -| SlidingWindow | Keep conversation history within a sliding token window | -| StuckLoopDetection | Detect and break out of repetitive agent loops | -| SubAgent | Delegate subtasks to specialised child agents | -| SystemReminders | Inject periodic reminders into the conversation | -| ToolErrorRecovery | Automatically retry or recover from tool execution errors | -| ToolOrphanRepair | Repair orphaned tool calls in conversation history | -| ToolOutputManagement | Control and format tool output for the model | +[`CodeMode`](code_mode/) wraps all tools into a single `run_code` tool powered by our [Monty](https://github.com/pydantic/monty) sandbox, so the model can orchestrate multiple tool calls with Python code instead of one model round-trip per call. -## Documentation +## Capability matrix -- [Pydantic AI docs](https://ai.pydantic.dev/) -- [Capabilities API](https://ai.pydantic.dev/capabilities/) +We studied the leading coding agents and AI harnesses -- [Claude Code](https://docs.anthropic.com/en/docs/claude-code), [OpenClaw](https://github.com/open-claw/open-claw), [Goose](https://block.github.io/goose/), [Hermes](https://github.com/NousResearch/hermes-agent), [OpenAI Codex](https://openai.com/index/codex/), [SWE-agent](https://github.com/SWE-agent/SWE-agent), [OpenHands](https://github.com/All-Hands-AI/OpenHands), and [Mastra](https://mastra.ai/) -- to map every capability area that matters for production agents. Each one is tracked as an [issue](https://github.com/pydantic/pydantic-harness/issues) in this repo. -## Development +**Vote on whatever is linked in the Status column** -- PRs if we're actively building it, issues if it's planned -- to help us decide what to work on next. + +| Category | Capability | Description | Status | Community alternatives | +|---|---|---|---|---| +| **Tools & execution** | **Code mode** | Sandboxed Python execution via [Monty](https://github.com/pydantic/monty) -- one `run_code` call replaces N tool calls | :white_check_mark: [Docs](code_mode/) | | +| | **Tool search** | Progressive tool discovery for large tool sets | :white_check_mark: [Pydantic AI](https://ai.pydantic.dev/deferred-tools/) | | +| | **File system** | Read, write, edit, search files with path traversal prevention | :construction: [PR #139](https://github.com/pydantic/pydantic-harness/pull/139) | [pydantic-ai-backend](https://github.com/vstorm-co/pydantic-ai-backend) (vstorm‑co) | +| | **Shell** | Execute commands with allowlists, denylists, and timeouts | :construction: [PR #139](https://github.com/pydantic/pydantic-harness/pull/139) | [pydantic-ai-backend](https://github.com/vstorm-co/pydantic-ai-backend) (vstorm‑co) | +| | **Repo context injection** | Auto-load CLAUDE.md/AGENTS.md and repo structure | :construction: [PR #154](https://github.com/pydantic/pydantic-harness/pull/154) | [pydantic-deep](https://github.com/vstorm-co/pydantic-deepagents) (vstorm‑co) | +| | **Verification loop** | Run tests after edits, auto-fix failures | :construction: [PR #160](https://github.com/pydantic/pydantic-harness/pull/160) | | +| **Context management** | **Sliding window** | Trim conversation history to stay within token limits | :construction: [PR #140](https://github.com/pydantic/pydantic-harness/pull/140) | [summarization-pydantic-ai](https://github.com/vstorm-co/summarization-pydantic-ai) (vstorm‑co) | +| | **Context compaction** | LLM-powered summarization of older messages | :construction: [PR #140](https://github.com/pydantic/pydantic-harness/pull/140) | [summarization-pydantic-ai](https://github.com/vstorm-co/summarization-pydantic-ai) (vstorm‑co) | +| | **Limit warnings** | Warn agent before hitting context/iteration limits | :construction: [PR #140](https://github.com/pydantic/pydantic-harness/pull/140) | [summarization-pydantic-ai](https://github.com/vstorm-co/summarization-pydantic-ai) (vstorm‑co) | +| | **Tool output management** | Truncate, summarize, or spill large tool outputs | :construction: [PR #131](https://github.com/pydantic/pydantic-harness/pull/131) | | +| | **System reminders** | Inject periodic reminders to counteract instruction drift | :construction: [PR #135](https://github.com/pydantic/pydantic-harness/pull/135) | | +| **Memory & persistence** | **Memory** | Persistent key-value memory across sessions | :construction: [PR #137](https://github.com/pydantic/pydantic-harness/pull/137) | [memv](https://github.com/vstorm-co/memv) (vstorm‑co) | +| | **Session persistence** | Save and restore full conversation state | :construction: [PR #153](https://github.com/pydantic/pydantic-harness/pull/153) | | +| **Agent orchestration** | **Sub-agents** | Delegate subtasks to specialized child agents | :construction: [PR #138](https://github.com/pydantic/pydantic-harness/pull/138) | [subagents-pydantic-ai](https://github.com/vstorm-co/subagents-pydantic-ai) (vstorm‑co) | +| | **Skills** | Progressive tool loading -- search, activate, deactivate | :construction: [PR #133](https://github.com/pydantic/pydantic-harness/pull/133) | [pydantic-ai-skills](https://github.com/DougTrajano/pydantic-ai-skills) (DougTrajano) | +| | **Planning** | Break complex tasks into structured plans before execution | :construction: [PR #136](https://github.com/pydantic/pydantic-harness/pull/136) | | +| | **Task tracking** | Track tasks, subtasks, and dependencies | :memo: [#65](https://github.com/pydantic/pydantic-harness/issues/65) | [pydantic-ai-todo](https://github.com/vstorm-co/pydantic-ai-todo) (vstorm‑co) | +| **Safety & guardrails** | **Input guardrails** | Validate user input before the agent run starts | :construction: [PR #134](https://github.com/pydantic/pydantic-harness/pull/134) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) | +| | **Output guardrails** | Validate model output after the run completes | :construction: [PR #134](https://github.com/pydantic/pydantic-harness/pull/134) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) | +| | **Cost/token budgets** | Enforce token and cost limits per run | :construction: [PR #134](https://github.com/pydantic/pydantic-harness/pull/134) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) | +| | **Tool access control** | Block tools or require approval before execution | :construction: [PR #134](https://github.com/pydantic/pydantic-harness/pull/134) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) | +| | **Async guardrails** | Run validation concurrently with model requests | :construction: [PR #134](https://github.com/pydantic/pydantic-harness/pull/134) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) | +| | **Secret masking** | Detect and redact secrets in agent I/O | :construction: [PR #157](https://github.com/pydantic/pydantic-harness/pull/157) | [pydantic-ai-shields](https://github.com/vstorm-co/pydantic-ai-shields) (vstorm‑co) | +| | **Approval workflows** | Require human approval for sensitive operations | :construction: [PR #156](https://github.com/pydantic/pydantic-harness/pull/156) | | +| | **Tool budget** | Limit total tool calls or cost per run | :construction: [PR #161](https://github.com/pydantic/pydantic-harness/pull/161) | | +| **Reliability** | **Stuck loop detection** | Detect and break out of repetitive agent loops | :construction: [PR #130](https://github.com/pydantic/pydantic-harness/pull/130) | | +| | **Tool error recovery** | Retry failed tool calls with backoff and budget | :construction: [PR #158](https://github.com/pydantic/pydantic-harness/pull/158) | | +| | **Tool orphan repair** | Fix orphaned tool calls in conversation history | :construction: [PR #132](https://github.com/pydantic/pydantic-harness/pull/132) | | +| **Reasoning** | **Adaptive reasoning** | Adjust thinking effort based on task complexity | :construction: [PR #155](https://github.com/pydantic/pydantic-harness/pull/155) | | +| | **Current time** | Inject current date/time into system prompt | :construction: [PR #159](https://github.com/pydantic/pydantic-harness/pull/159) | | + +> Packages by [vstorm-co](https://github.com/vstorm-co) are endorsed by the Pydantic AI team. We're working with them to upstream some of their implementations into this repo. + +## Help us prioritize + +**Vote on whatever is linked in the Status column above.** If there's a PR, vote on the PR -- it means we're actively building it. If there's only an issue, vote on the issue. + +Want something that's not on the list? [Open a capability request](https://github.com/pydantic/pydantic-harness/issues/new?template=capability-request.yml). + +## Build your own + +[Capabilities](https://ai.pydantic.dev/capabilities/#building-custom-capabilities) are the primary extension point for Pydantic AI. Any of the existing capabilities in this repo can serve as a reference for building your own. + +**Publishing as a standalone package?** Use the `pydantic-ai-` naming convention. See [Publishing capability packages](https://ai.pydantic.dev/extensibility/#publishing-capability-packages). + +## Contributing + +We welcome capability contributions. Here's how: + +1. **Start with an issue.** [Open a capability request](https://github.com/pydantic/pydantic-harness/issues/new?template=capability-request.yml) describing the behavior you want. This lets us discuss the approach and priority before code is written -- we can close an approach without closing the problem. +2. **Then open a PR.** Once the issue exists, you're welcome to open a PR with an implementation. Link the issue in your PR. We review based on community interest -- upvotes on both the issue and PR count. +3. **Don't chase green CI.** Get the approach working, then let us know. We'll take it from there -- we may push to your branch, rewrite, or open a follow-up PR. You'll be credited as the original author. (See the [Pydantic AI contributing guide](https://github.com/pydantic/pydantic-ai/blob/main/CONTRIBUTING.md).) + +> **Note**: PRs that modify `pyproject.toml` or `uv.lock` from non-team members are auto-closed by CI to prevent supply chain risk. If you need a new dependency, [open an issue](https://github.com/pydantic/pydantic-harness/issues/new). + +### Development ```bash make install # install dependencies -make lint # ruff format check + lint +make format # ruff format +make lint # ruff check make typecheck # pyright strict make test # pytest -make testcov # pytest with coverage +make testcov # pytest with 100% branch coverage ``` +## Pydantic AI references + +- [Capabilities](https://ai.pydantic.dev/capabilities/) -- what capabilities are, built-in capabilities, building your own +- [Hooks](https://ai.pydantic.dev/hooks/) -- lifecycle hooks reference, ordering, error handling +- [Extensibility](https://ai.pydantic.dev/extensibility/) -- publishing packages, third-party ecosystem +- [Toolsets](https://ai.pydantic.dev/toolsets/) -- building tools for capabilities +- [API reference](https://ai.pydantic.dev/api/capabilities/) -- full API docs + ## License -MIT +MIT -- see [LICENSE](LICENSE). diff --git a/code_mode/README.md b/code_mode/README.md new file mode 100644 index 0000000..a0eaa06 --- /dev/null +++ b/code_mode/README.md @@ -0,0 +1,169 @@ +# Code Mode + +Replace individual tool calls with a single sandboxed Python execution environment. + +## The problem + +Standard tool calling requires one model round-trip per tool call. An agent that needs to fetch 10 items and process each one makes 11+ model calls -- slow, expensive, and context-heavy. + +## The solution + +`CodeMode` wraps your tools into a single `run_code` tool. The model writes Python code that calls multiple tools with loops, conditionals, variables, and `asyncio.gather` -- all inside a sandboxed [Monty](https://github.com/pydantic/monty) runtime. + +| Standard tool calling | Code mode | +|---|---| +| 1 model call per tool | 1 model call for N tools | +| Sequential by default | Parallel via `asyncio.gather` | +| No local computation | Filter, transform, aggregate in code | +| Large conversation history | Compact -- fewer messages | + +## Usage + +```python +from pydantic_ai import Agent +from pydantic_harness import CodeMode + +agent = Agent('anthropic:claude-sonnet-4-6', capabilities=[CodeMode()]) + +@agent.tool_plain +def get_weather(city: str) -> dict: + """Get current weather for a city.""" + return {'city': city, 'temp_f': 72, 'condition': 'sunny'} + +@agent.tool_plain +def convert_temp(fahrenheit: float) -> float: + """Convert Fahrenheit to Celsius.""" + return round((fahrenheit - 32) * 5 / 9, 1) + +result = agent.run_sync("What's the weather in Paris and Tokyo, in Celsius?") +``` + +The model writes code like: + +```python +paris, tokyo = await asyncio.gather( + get_weather(city='Paris'), + get_weather(city='Tokyo'), +) +paris_c = await convert_temp(fahrenheit=paris['temp_f']) +tokyo_c = await convert_temp(fahrenheit=tokyo['temp_f']) +{'paris': paris_c, 'tokyo': tokyo_c} +``` + +## Installation + +Code mode requires the Monty sandbox: + +```bash +uv add "pydantic-harness[code-mode]" +``` + +## Selective tool sandboxing + +By default, `CodeMode(tools='all')` sandboxes every tool. You can control which tools go through the sandbox: + +```python +# By name -- only these tools are available inside run_code +CodeMode(tools=['search', 'fetch']) + +# By predicate +CodeMode(tools=lambda ctx, td: td.name != 'dangerous_tool') + +# By metadata -- combine with SetToolMetadata or .with_metadata() +CodeMode(tools={'code_mode': True}) +``` + +Tools that match the selector are wrapped inside `run_code`. Non-matching tools remain available as regular tool calls. + +### Metadata-based selection + +```python +from pydantic_ai import Agent +from pydantic_ai.toolsets import FunctionToolset +from pydantic_harness import CodeMode + +search_tools = FunctionToolset(tools=[search, fetch]).with_metadata(code_mode=True) + +agent = Agent( + 'anthropic:claude-sonnet-4-6', + toolsets=[search_tools], + capabilities=[CodeMode(tools={'code_mode': True})], +) +``` + +## Return values + +The last expression in the code snippet is automatically captured as the return value -- the model does not need to `print()`. + +| Scenario | Return | +|---|---| +| No print output | Last expression value | +| With print output | `{"output": "", "result": }` | +| Multimodal content (e.g. images) | Returned natively for model processing | + +## REPL state + +State persists between `run_code` calls within the same agent run -- variables, imports, and function definitions carry over. Pass `restart: true` in the tool call to reset state. + +## Observability + +Nested tool calls inside `run_code` produce their own spans when instrumented with [Logfire](https://pydantic.dev/logfire) or any OpenTelemetry backend. The `run_code` tool return includes metadata with all nested calls: + +```python +for msg in result.all_messages(): + for part in msg.parts: + if isinstance(part, ToolReturnPart) and part.tool_name == 'run_code': + tool_calls = part.metadata['tool_calls'] # dict[str, ToolCallPart] + tool_returns = part.metadata['tool_returns'] # dict[str, ToolReturnPart] +``` + +## Sandbox restrictions + +Code runs inside [Monty](https://github.com/pydantic/monty), a sandboxed Python subset. Key restrictions: + +- No class definitions +- No third-party imports (allowed stdlib: `sys`, `typing`, `asyncio`, `math`, `json`, `re`, `datetime`, `os`, `pathlib`) +- No `import *` +- Tools requiring approval or with deferred execution are excluded from the sandbox + +## API + +```python +CodeMode( + tools: ToolSelector = 'all', # 'all', list[str], callable, or dict + max_retries: int = 3, # retries on sandbox execution errors +) +``` + +## Agent spec (YAML/JSON) + +CodeMode works with Pydantic AI's [agent spec](https://ai.pydantic.dev/agent-spec/) feature for defining agents in YAML: + +```yaml +# agent.yaml +model: anthropic:claude-sonnet-4-6 +capabilities: + - CodeMode: {} +``` + +```python +from pydantic_ai import Agent +from pydantic_harness import CodeMode + +agent = Agent.from_file('agent.yaml', custom_capability_types=[CodeMode]) +``` + +Pass `custom_capability_types` so the spec loader knows how to instantiate `CodeMode`. You can also pass arguments in the YAML: + +```yaml +capabilities: + - CodeMode: + tools: ['search', 'fetch'] + max_retries: 5 +``` + +## Further reading + +- [Tool use via code](https://www.anthropic.com/engineering/code-execution-with-mcp) (Anthropic) +- [Code mode in production](https://blog.cloudflare.com/code-mode/) (Cloudflare) +- [Pydantic AI capabilities](https://ai.pydantic.dev/capabilities/) diff --git a/docs/capabilities/index.md b/docs/capabilities/index.md deleted file mode 100644 index b12b547..0000000 --- a/docs/capabilities/index.md +++ /dev/null @@ -1,29 +0,0 @@ -# Capabilities - -Each capability is an `AbstractCapability` subclass that can be attached to any -Pydantic AI agent via the `capabilities` parameter. - -| Capability | Description | -|---|---| -| AdaptiveReasoning | Dynamically adjust reasoning effort based on task complexity | -| Approval | Require human approval before executing sensitive operations | -| Compaction | Compress conversation history to stay within context limits | -| FileSystem | Read, write, and navigate the local filesystem | -| Guardrails | Validate inputs/outputs and enforce cost and tool constraints | -| KnowsCurrentTime | Inject the current date and time into the system prompt | -| Memory | Persistent key-value memory across agent sessions | -| Planning | Break complex tasks into plans before execution | -| RepoContextInjection | Inject repository structure and context into the system prompt | -| SecretMasking | Detect and redact secrets in agent inputs and outputs | -| SessionPersistence | Save and restore full conversation sessions | -| Shell | Execute shell commands with safety controls | -| Skills | Progressive tool loading via search and activate | -| SlidingWindow | Keep conversation history within a sliding token window | -| StuckLoopDetection | Detect and break out of repetitive agent loops | -| SubAgent | Delegate subtasks to specialised child agents | -| SystemReminders | Inject periodic reminders into the conversation | -| ToolErrorRecovery | Automatically retry or recover from tool execution errors | -| ToolOrphanRepair | Repair orphaned tool calls in conversation history | -| ToolOutputManagement | Control and format tool output for the model | - -Detailed documentation for each capability will be added as they are merged. diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index a386193..0000000 --- a/docs/index.md +++ /dev/null @@ -1,36 +0,0 @@ -# Pydantic Harness - -Composable, reusable capabilities for [Pydantic AI](https://ai.pydantic.dev/) agents. - -## What is it? - -Pydantic Harness provides a library of **capabilities** -- self-contained bundles of -system prompts, tools, and lifecycle hooks -- that you can attach to any Pydantic AI -agent to give it new powers without writing boilerplate. - -## Installation - -```bash -pip install pydantic-harness -``` - -Or with `uv`: - -```bash -uv add pydantic-harness -``` - -## Quick start - -```python -from pydantic_ai import Agent -from pydantic_harness import Memory, Skills - -agent = Agent('openai:gpt-4o', capabilities=[Memory(), Skills()]) -``` - -## Learn more - -- [Available capabilities](capabilities/index.md) -- [Pydantic AI documentation](https://ai.pydantic.dev/) -- [GitHub repository](https://github.com/pydantic/pydantic-harness) diff --git a/pyproject.toml b/pyproject.toml index 0d573a0..77c42de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ packages = ['src/pydantic_harness'] [tool.ruff] line-length = 120 target-version = 'py310' +exclude = ['template'] [tool.ruff.lint] extend-select = ['Q', 'RUF100', 'C90', 'UP', 'I', 'D', 'TID251'] @@ -79,11 +80,13 @@ quote-style = 'single' [tool.pyright] pythonVersion = '3.10' typeCheckingMode = 'strict' +exclude = ['template', '.venv'] executionEnvironments = [ { root = 'tests', reportPrivateUsage = false }, ] [tool.pytest.ini_options] +testpaths = ['tests'] xfail_strict = true filterwarnings = ['error'] anyio_mode = 'auto'