test: statically validate doc snippets exist and parse

Every Python snippet in the capability READMEs and docs/*.md pages is now
checked for the two failures a reader hits immediately: it does not parse
(syntax), or it imports a pydantic_ai_harness symbol that does not exist (stale
module path or renamed name -- the class of bug behind the experimental.* import
drift). Static only: no model/network execution, so it needs no mocking. The
four illustrative API-signature blocks opt out with a {test="skip"} fence
(read by pytest-examples, stripped-safe for the unified-docs render).
This commit is contained in:
David SF
2026-07-12 14:29:26 -05:00
parent f15f69197e
commit a79c13de0d
5 changed files with 114 additions and 4 deletions
+1 -1
View File
@@ -211,7 +211,7 @@ A model id is any string a Pydantic AI model accepts, so newer models not yet in
## API
```python
```python {test="skip"}
run_acp_stdio( # async; serve until the client disconnects
agent,
*,
+1 -1
View File
@@ -248,7 +248,7 @@ Code runs inside [Monty](https://github.com/pydantic/monty), a sandboxed Python
## API
```python
```python {test="skip"}
CodeMode(
tools: ToolSelector = 'all', # 'all', list[str], callable, or dict
max_retries: int = 3, # retries on sandbox execution errors
@@ -225,7 +225,7 @@ Each completed turn reports its token counts (input/output/total, plus cached to
## API
```python
```python {test="skip"}
run_acp_stdio( # async; serve until the client disconnects
agent,
*,
+1 -1
View File
@@ -160,7 +160,7 @@ Any exception raised by the guard propagates as-is -- use `InputBlocked` / `Outp
## API
```python
```python {test="skip"}
@dataclass
class GuardResult:
action: Literal['allow', 'block', 'replace', 'retry']
+110
View File
@@ -0,0 +1,110 @@
"""Static validation of the code snippets shown in the docs.
Every Python snippet in a capability `README.md` (GitHub/PyPI) and in the flat
`docs/<capability>.md` pages (the unified docs site) is checked for the two
failure modes a reader hits immediately:
- **it does not parse** -- a syntax error means the snippet cannot run at all;
- **it imports a harness symbol that does not exist** -- a stale module path or a
renamed/removed name (e.g. a snippet still importing from
`pydantic_ai_harness.experimental.<graduated>`).
This is the *static* half of doc-snippet testing. It deliberately does not
execute the snippets -- most build an `Agent` and call `.run()`, which needs a
model -- so it stays fast and needs no mocking. Running snippets against a mocked
model is a separate concern (see `test_readme_quick_start.py` for that shape).
Illustrative signature blocks (API-reference pseudo-code with type annotations or
a bare `*`, which is not runnable Python) opt out with a `{test="skip"}` fence
directive.
"""
from __future__ import annotations as _annotations
import ast
import importlib
import os
import warnings
from collections.abc import Iterable
from pathlib import Path
import pytest
from _pytest.mark import ParameterSet
from pytest_examples import CodeExample, find_examples
_ROOT = Path(__file__).parent.parent
_HARNESS = 'pydantic_ai_harness'
def _harness_import_targets(tree: ast.AST) -> Iterable[tuple[str, str | None]]:
"""`(module, name)` for every `pydantic_ai_harness` symbol a snippet imports.
`name` is `None` for a plain `import pydantic_ai_harness.x` or a star import,
where only the module's existence can be checked.
"""
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom):
module = node.module or ''
if module == _HARNESS or module.startswith(f'{_HARNESS}.'):
for alias in node.names:
yield module, None if alias.name == '*' else alias.name
elif isinstance(node, ast.Import):
for alias in node.names:
if alias.name == _HARNESS or alias.name.startswith(f'{_HARNESS}.'):
yield alias.name, None
def _snippet_problem(source: str) -> str | None:
"""Return why a snippet is invalid, or `None` if it parses and its harness imports resolve."""
try:
tree = ast.parse(source)
except SyntaxError as exc:
return f'does not parse: {exc.msg} (line {exc.lineno})'
for module, name in _harness_import_targets(tree):
try:
with warnings.catch_warnings():
warnings.simplefilter('ignore') # a deprecated shim path still resolves; existence is what we check
imported = importlib.import_module(module)
except ImportError as exc:
return f'imports `{module}`, which does not exist: {exc}'
if name is not None and not hasattr(imported, name):
return f'imports `{name}` from `{module}`, but that name does not exist'
return None
def _doc_snippets() -> Iterable[ParameterSet]:
# `find_examples` yields only Python fenced blocks and wants paths relative to
# the cwd, so pin it to the repo root (matches `test_skill_examples.py`).
os.chdir(_ROOT)
readmes = sorted(str(p.relative_to(_ROOT)) for p in _ROOT.glob(f'{_HARNESS}/**/README.md'))
for ex in find_examples(*readmes, 'docs'):
yield pytest.param(ex, id=f'{ex.path}:{ex.start_line}')
@pytest.mark.parametrize('example', _doc_snippets())
def test_doc_snippet_valid(example: CodeExample) -> None:
if example.prefix_settings().get('test', '').startswith('skip'):
pytest.skip('illustrative signature block; not runnable Python')
problem = _snippet_problem(example.source)
assert problem is None, (
f'{example.path}:{example.start_line} {problem}. '
'Fix the snippet, or mark the fence `{test="skip"}` if it is illustrative signature pseudo-code.'
)
def test_doc_snippets_discovered() -> None:
# Guard against a discovery break silently making the check vacuous.
assert sum(1 for _ in _doc_snippets()) >= 100
def test_snippet_problem_detects_each_failure_mode() -> None:
# Valid: harness imports that resolve, star imports, plain imports, and non-harness imports.
assert _snippet_problem('from pydantic_ai_harness import CodeMode') is None
assert _snippet_problem('import pydantic_ai_harness.code_mode') is None
assert _snippet_problem('from pydantic_ai_harness.overflowing_tool_output import *') is None
assert _snippet_problem('from os import path\nimport sys') is None
# Invalid: syntax, a module that does not exist, and a name that does not exist.
assert 'does not parse' in (_snippet_problem('def (:') or '')
assert 'does not exist' in (_snippet_problem('from pydantic_ai_harness.nope import X') or '')
assert 'does not exist' in (_snippet_problem('from pydantic_ai_harness import NoSuchCapability') or '')