mirror of
https://github.com/pydantic/pydantic-ai-harness.git
synced 2026-07-21 10:55:35 +00:00
test: statically validate doc snippets exist and parse
Every Python snippet in the capability READMEs and docs/*.md pages is now
checked for the two failures a reader hits immediately: it does not parse
(syntax), or it imports a pydantic_ai_harness symbol that does not exist (stale
module path or renamed name -- the class of bug behind the experimental.* import
drift). Static only: no model/network execution, so it needs no mocking. The
four illustrative API-signature blocks opt out with a {test="skip"} fence
(read by pytest-examples, stripped-safe for the unified-docs render).
This commit is contained in:
+1
-1
@@ -211,7 +211,7 @@ A model id is any string a Pydantic AI model accepts, so newer models not yet in
|
||||
|
||||
## API
|
||||
|
||||
```python
|
||||
```python {test="skip"}
|
||||
run_acp_stdio( # async; serve until the client disconnects
|
||||
agent,
|
||||
*,
|
||||
|
||||
@@ -248,7 +248,7 @@ Code runs inside [Monty](https://github.com/pydantic/monty), a sandboxed Python
|
||||
|
||||
## API
|
||||
|
||||
```python
|
||||
```python {test="skip"}
|
||||
CodeMode(
|
||||
tools: ToolSelector = 'all', # 'all', list[str], callable, or dict
|
||||
max_retries: int = 3, # retries on sandbox execution errors
|
||||
|
||||
@@ -225,7 +225,7 @@ Each completed turn reports its token counts (input/output/total, plus cached to
|
||||
|
||||
## API
|
||||
|
||||
```python
|
||||
```python {test="skip"}
|
||||
run_acp_stdio( # async; serve until the client disconnects
|
||||
agent,
|
||||
*,
|
||||
|
||||
@@ -160,7 +160,7 @@ Any exception raised by the guard propagates as-is -- use `InputBlocked` / `Outp
|
||||
|
||||
## API
|
||||
|
||||
```python
|
||||
```python {test="skip"}
|
||||
@dataclass
|
||||
class GuardResult:
|
||||
action: Literal['allow', 'block', 'replace', 'retry']
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Static validation of the code snippets shown in the docs.
|
||||
|
||||
Every Python snippet in a capability `README.md` (GitHub/PyPI) and in the flat
|
||||
`docs/<capability>.md` pages (the unified docs site) is checked for the two
|
||||
failure modes a reader hits immediately:
|
||||
|
||||
- **it does not parse** -- a syntax error means the snippet cannot run at all;
|
||||
- **it imports a harness symbol that does not exist** -- a stale module path or a
|
||||
renamed/removed name (e.g. a snippet still importing from
|
||||
`pydantic_ai_harness.experimental.<graduated>`).
|
||||
|
||||
This is the *static* half of doc-snippet testing. It deliberately does not
|
||||
execute the snippets -- most build an `Agent` and call `.run()`, which needs a
|
||||
model -- so it stays fast and needs no mocking. Running snippets against a mocked
|
||||
model is a separate concern (see `test_readme_quick_start.py` for that shape).
|
||||
|
||||
Illustrative signature blocks (API-reference pseudo-code with type annotations or
|
||||
a bare `*`, which is not runnable Python) opt out with a `{test="skip"}` fence
|
||||
directive.
|
||||
"""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
import ast
|
||||
import importlib
|
||||
import os
|
||||
import warnings
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from _pytest.mark import ParameterSet
|
||||
from pytest_examples import CodeExample, find_examples
|
||||
|
||||
_ROOT = Path(__file__).parent.parent
|
||||
_HARNESS = 'pydantic_ai_harness'
|
||||
|
||||
|
||||
def _harness_import_targets(tree: ast.AST) -> Iterable[tuple[str, str | None]]:
|
||||
"""`(module, name)` for every `pydantic_ai_harness` symbol a snippet imports.
|
||||
|
||||
`name` is `None` for a plain `import pydantic_ai_harness.x` or a star import,
|
||||
where only the module's existence can be checked.
|
||||
"""
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
module = node.module or ''
|
||||
if module == _HARNESS or module.startswith(f'{_HARNESS}.'):
|
||||
for alias in node.names:
|
||||
yield module, None if alias.name == '*' else alias.name
|
||||
elif isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
if alias.name == _HARNESS or alias.name.startswith(f'{_HARNESS}.'):
|
||||
yield alias.name, None
|
||||
|
||||
|
||||
def _snippet_problem(source: str) -> str | None:
|
||||
"""Return why a snippet is invalid, or `None` if it parses and its harness imports resolve."""
|
||||
try:
|
||||
tree = ast.parse(source)
|
||||
except SyntaxError as exc:
|
||||
return f'does not parse: {exc.msg} (line {exc.lineno})'
|
||||
|
||||
for module, name in _harness_import_targets(tree):
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore') # a deprecated shim path still resolves; existence is what we check
|
||||
imported = importlib.import_module(module)
|
||||
except ImportError as exc:
|
||||
return f'imports `{module}`, which does not exist: {exc}'
|
||||
if name is not None and not hasattr(imported, name):
|
||||
return f'imports `{name}` from `{module}`, but that name does not exist'
|
||||
return None
|
||||
|
||||
|
||||
def _doc_snippets() -> Iterable[ParameterSet]:
|
||||
# `find_examples` yields only Python fenced blocks and wants paths relative to
|
||||
# the cwd, so pin it to the repo root (matches `test_skill_examples.py`).
|
||||
os.chdir(_ROOT)
|
||||
readmes = sorted(str(p.relative_to(_ROOT)) for p in _ROOT.glob(f'{_HARNESS}/**/README.md'))
|
||||
for ex in find_examples(*readmes, 'docs'):
|
||||
yield pytest.param(ex, id=f'{ex.path}:{ex.start_line}')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('example', _doc_snippets())
|
||||
def test_doc_snippet_valid(example: CodeExample) -> None:
|
||||
if example.prefix_settings().get('test', '').startswith('skip'):
|
||||
pytest.skip('illustrative signature block; not runnable Python')
|
||||
problem = _snippet_problem(example.source)
|
||||
assert problem is None, (
|
||||
f'{example.path}:{example.start_line} {problem}. '
|
||||
'Fix the snippet, or mark the fence `{test="skip"}` if it is illustrative signature pseudo-code.'
|
||||
)
|
||||
|
||||
|
||||
def test_doc_snippets_discovered() -> None:
|
||||
# Guard against a discovery break silently making the check vacuous.
|
||||
assert sum(1 for _ in _doc_snippets()) >= 100
|
||||
|
||||
|
||||
def test_snippet_problem_detects_each_failure_mode() -> None:
|
||||
# Valid: harness imports that resolve, star imports, plain imports, and non-harness imports.
|
||||
assert _snippet_problem('from pydantic_ai_harness import CodeMode') is None
|
||||
assert _snippet_problem('import pydantic_ai_harness.code_mode') is None
|
||||
assert _snippet_problem('from pydantic_ai_harness.overflowing_tool_output import *') is None
|
||||
assert _snippet_problem('from os import path\nimport sys') is None
|
||||
# Invalid: syntax, a module that does not exist, and a name that does not exist.
|
||||
assert 'does not parse' in (_snippet_problem('def (:') or '')
|
||||
assert 'does not exist' in (_snippet_problem('from pydantic_ai_harness.nope import X') or '')
|
||||
assert 'does not exist' in (_snippet_problem('from pydantic_ai_harness import NoSuchCapability') or '')
|
||||
Reference in New Issue
Block a user