mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-07-21 10:15:47 +00:00
Daoyuan LiandGitHub
d075be0277
* fix(browserless): surface target-page error status in web_fetch_tool Browserless returns HTTP 200 for the render request itself even when the target page responded with a 4xx/5xx or served an anti-bot block page, tagging the real outcome on X-Response-Code/X-Response-Status headers. capture_screenshot/web_capture_tool already reads these headers and surfaces a warning via _target_status_warning. fetch_html only logged them at debug level and web_fetch_tool returned the block/error page's raw text as if it were a normal successful fetch, with no indication anything was wrong. fetch_html now returns a BrowserlessFetchResult carrying the rendered HTML plus the target-status headers (mirroring BrowserlessScreenshotResult), and web_fetch_tool appends the same _target_status_warning used by web_capture_tool when the target page errored. Legitimate 200-target fetches are unaffected. * fix(browserless): keep fetch_html() returning a plain string BrowserlessClient is re-exported from deerflow.community.browserless.__all__, so fetch_html() is public harness API with an established str-only contract: the rendered HTML on success, or an "Error: ..." string on failure. Changing its return type to BrowserlessFetchResult broke that contract for any caller that treats the result as a string (.lower(), concatenation, passing to a parser), even when the fetch itself succeeded. fetch_html() is now a thin wrapper that always unwraps back to the original str contract. The richer, status-aware result (needed to tell a genuine 200 apart from a render-succeeded-but-target-errored response) moves to a new fetch_html_with_status() method, which web_fetch_tool now calls instead so it keeps surfacing the target-page-error warning. Tests: retarget the tool-level mocks onto fetch_html_with_status, add a direct regression asserting fetch_html() returns str on success - including when the target page itself errored under a 200 render response - and keep the status-aware coverage on the new method.
766 lines
36 KiB
Python
766 lines
36 KiB
Python
"""Tests for Browserless community tools."""
|
|
|
|
import ipaddress
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from deerflow.community.browserless import tools
|
|
from deerflow.community.browserless.browserless_client import BrowserlessClient, BrowserlessFetchResult, BrowserlessScreenshotResult
|
|
|
|
|
|
class AsyncMock(MagicMock):
|
|
"""Mock that supports async call."""
|
|
|
|
async def __call__(self, *args, **kwargs):
|
|
return super().__call__(*args, **kwargs)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestBrowserlessClient:
|
|
"""Tests for the BrowserlessClient class."""
|
|
|
|
async def test_fetch_html_success(self):
|
|
"""fetch_html returns the rendered HTML as a plain string on success.
|
|
|
|
Regression guard for the fetch_html() public string contract:
|
|
BrowserlessClient is re-exported from deerflow.community.browserless.__all__,
|
|
so harness consumers that call string methods, compare the result, or pass
|
|
it directly to a parser must keep getting a str back. Status-aware callers
|
|
(e.g. web_fetch_tool) use fetch_html_with_status() instead - see
|
|
test_fetch_html_with_status_surfaces_target_status_headers below.
|
|
"""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html><body>Page content</body></html>"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.fetch_html("https://example.com")
|
|
|
|
assert isinstance(result, str)
|
|
assert result == "<html><body>Page content</body></html>"
|
|
call_kwargs = mock_ctx.post.call_args.kwargs
|
|
assert call_kwargs["json"]["url"] == "https://example.com"
|
|
assert "waitUntil" not in call_kwargs["json"]
|
|
assert "gotoTimeout" not in call_kwargs["json"]
|
|
assert "bestAttempt" not in call_kwargs["json"]
|
|
|
|
async def test_fetch_html_returns_plain_string_even_when_target_page_errored(self):
|
|
"""fetch_html stays a plain string even when the target page itself errored.
|
|
|
|
Browserless returns HTTP 200 for the render request itself even when the
|
|
target page responded with a 404 (or an anti-bot block page). Before this
|
|
fix, a successful fetch_html() call started returning a BrowserlessFetchResult
|
|
object instead of a string in exactly this case, breaking existing callers
|
|
(.lower(), string concatenation, parsers expecting str) even though their
|
|
fetch technically succeeded. fetch_html() must keep unwrapping to the plain
|
|
HTML string regardless of the target status; only fetch_html_with_status()
|
|
exposes the richer result.
|
|
"""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html><body>Access Denied</body></html>"
|
|
mock_resp.headers = {
|
|
"X-Response-Code": "404",
|
|
"X-Response-Status": "Not Found",
|
|
}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.fetch_html("https://example.com/blocked")
|
|
|
|
assert isinstance(result, str)
|
|
assert result == "<html><body>Access Denied</body></html>"
|
|
|
|
async def test_fetch_html_with_status_surfaces_target_status_headers(self):
|
|
"""fetch_html_with_status carries the target page's real status headers on the result.
|
|
|
|
Browserless returns HTTP 200 for the render request itself even when the
|
|
target page responded with a 404 (or an anti-bot block page), so status-aware
|
|
callers need the X-Response-Code/X-Response-Status headers to tell the two
|
|
apart. This richer result is opt-in via fetch_html_with_status(); plain
|
|
fetch_html() stays a str (see test_fetch_html_success above).
|
|
"""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html><body>Access Denied</body></html>"
|
|
mock_resp.headers = {
|
|
"X-Response-Code": "404",
|
|
"X-Response-Status": "Not Found",
|
|
}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.fetch_html_with_status("https://example.com/blocked")
|
|
|
|
assert isinstance(result, BrowserlessFetchResult)
|
|
assert result.html == "<html><body>Access Denied</body></html>"
|
|
assert result.target_status_code == "404"
|
|
assert result.target_status == "Not Found"
|
|
|
|
async def test_fetch_html_empty_response(self):
|
|
"""fetch_html returns error for empty response."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = " "
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.fetch_html("https://example.com")
|
|
assert result == "Error: Browserless returned empty response"
|
|
|
|
async def test_fetch_html_http_error(self):
|
|
"""fetch_html returns error for non-200 status."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 500
|
|
mock_resp.text = "Internal error"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.fetch_html("https://example.com")
|
|
assert "Error: Browserless HTTP 500" in result
|
|
|
|
async def test_fetch_html_timeout(self):
|
|
"""fetch_html returns timeout error."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
import httpx
|
|
|
|
mock_ctx.post = AsyncMock(side_effect=httpx.TimeoutException("Timed out"))
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000", timeout_s=10)
|
|
result = await client.fetch_html("https://example.com")
|
|
assert "timed out" in result.lower() or "timeout" in result.lower()
|
|
|
|
async def test_fetch_html_with_token(self):
|
|
"""fetch_html includes token in payload when set."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html>OK</html>"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000", token="my-token")
|
|
await client.fetch_html("https://example.com")
|
|
|
|
payload = mock_ctx.post.call_args.kwargs["json"]
|
|
assert payload["token"] == "my-token"
|
|
|
|
async def test_fetch_html_with_wait_for_selector(self):
|
|
"""fetch_html sends waitForSelector when selector is set."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html>OK</html>"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
await client.fetch_html("https://example.com", wait_for_selector="article")
|
|
|
|
payload = mock_ctx.post.call_args.kwargs["json"]
|
|
assert payload["waitForSelector"]["selector"] == "article"
|
|
|
|
async def test_fetch_html_with_reject_params(self):
|
|
"""fetch_html sends reject params when set."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html>OK</html>"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
await client.fetch_html(
|
|
"https://example.com",
|
|
reject_resource_types=["image"],
|
|
reject_request_pattern=[r"\.css$"],
|
|
)
|
|
|
|
payload = mock_ctx.post.call_args.kwargs["json"]
|
|
assert payload["rejectResourceTypes"] == ["image"]
|
|
assert payload["rejectRequestPattern"] == [r"\.css$"]
|
|
|
|
async def test_capture_screenshot_success(self):
|
|
"""capture_screenshot posts to /screenshot and returns image bytes."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.content = b"\x89PNG\r\n\x1a\nimage"
|
|
mock_resp.text = "<binary>"
|
|
mock_resp.headers = {
|
|
"Content-Type": "image/png",
|
|
"X-Response-Code": "200",
|
|
"X-Response-Status": "OK",
|
|
"X-Response-URL": "https://example.com/final",
|
|
}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000", token="secret-token")
|
|
result = await client.capture_screenshot(
|
|
"https://example.com",
|
|
full_page=True,
|
|
output_format="png",
|
|
viewport={"width": 1280, "height": 720},
|
|
wait_for_selector="main",
|
|
wait_for_selector_timeout_ms=3000,
|
|
wait_for_timeout_ms=500,
|
|
best_attempt=True,
|
|
)
|
|
|
|
assert isinstance(result, BrowserlessScreenshotResult)
|
|
assert result.content == b"\x89PNG\r\n\x1a\nimage"
|
|
assert result.content_type == "image/png"
|
|
assert result.target_status_code == "200"
|
|
assert result.target_status == "OK"
|
|
assert result.final_url == "https://example.com/final"
|
|
|
|
call = mock_ctx.post.call_args
|
|
assert call.args == ("http://browserless:3000/screenshot",)
|
|
payload = call.kwargs["json"]
|
|
assert payload["url"] == "https://example.com"
|
|
assert payload["options"] == {
|
|
"fullPage": True,
|
|
"type": "png",
|
|
}
|
|
assert call.kwargs["params"] == {"token": "secret-token"}
|
|
assert payload["viewport"] == {"width": 1280, "height": 720}
|
|
assert payload["waitForSelector"] == {"selector": "main", "timeout": 3000}
|
|
assert payload["waitForTimeout"] == 500
|
|
assert payload["bestAttempt"] is True
|
|
|
|
async def test_capture_screenshot_http_error(self):
|
|
"""capture_screenshot returns a bounded error on non-200 responses."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 500
|
|
mock_resp.text = "Internal browserless error"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.capture_screenshot("https://example.com")
|
|
|
|
assert isinstance(result, str)
|
|
assert "Error: Browserless HTTP 500" in result
|
|
assert "Internal browserless error" in result
|
|
|
|
async def test_capture_screenshot_empty_response(self):
|
|
"""capture_screenshot returns a clear error for empty binary content."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.content = b""
|
|
mock_resp.text = ""
|
|
mock_resp.headers = {"Content-Type": "image/png"}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.capture_screenshot("https://example.com")
|
|
|
|
assert result == "Error: Browserless returned empty screenshot response"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestBrowserlessTools:
|
|
"""Tests for the Browserless tool functions."""
|
|
|
|
async def test_get_browserless_client_uses_env_token_fallback(self):
|
|
"""Browserless tools use BROWSERLESS_TOKEN when config omits token."""
|
|
with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg:
|
|
mock_cfg.return_value = {"base_url": "https://production-sfo.browserless.io"}
|
|
with patch.dict("os.environ", {"BROWSERLESS_TOKEN": "env-token"}, clear=True):
|
|
client = tools._get_browserless_client("web_capture")
|
|
|
|
assert client.token == "env-token"
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_success(self, mock_get_client):
|
|
"""web_fetch_tool successfully fetches and extracts content."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><h1>Title</h1><p>Content</p></article></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com/article")
|
|
|
|
assert "Error:" not in result
|
|
assert "warning:" not in result
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_error(self, mock_get_client):
|
|
"""web_fetch_tool returns error when fetch fails."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(return_value="Error: Browserless returned empty response")
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com")
|
|
|
|
assert result.startswith("Error:")
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_exception(self, mock_get_client):
|
|
"""web_fetch_tool returns error when client raises exception."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(side_effect=Exception("Unexpected error"))
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com")
|
|
|
|
assert result.startswith("Error:")
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_rejects_metadata_ip(self, mock_get_client):
|
|
"""web_fetch_tool blocks the cloud-metadata link-local endpoint."""
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("http://169.254.169.254/latest/meta-data/")
|
|
|
|
assert "private, loopback, or metadata" in result
|
|
mock_get_client.assert_not_called()
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_rejects_dns_resolving_to_private(self, mock_get_client):
|
|
"""web_fetch_tool blocks hostnames that resolve to internal IPs."""
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("10.0.0.5")],
|
|
):
|
|
result = await tools.web_fetch_tool.ainvoke("https://internal.example.com/")
|
|
|
|
assert "private, loopback, or metadata" in result
|
|
mock_get_client.assert_not_called()
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_allows_private_when_opted_in(self, mock_get_client):
|
|
"""web_fetch_tool allows internal targets only when explicitly configured."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><p>internal</p></article></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value={"allow_private_addresses": True}):
|
|
result = await tools.web_fetch_tool.ainvoke("http://10.0.0.5/dashboard")
|
|
|
|
assert "Error:" not in result
|
|
mock_client.fetch_html_with_status.assert_called_once()
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_warns_on_target_error_status(self, mock_get_client):
|
|
"""web_fetch_tool surfaces a warning when the fetched page itself errored.
|
|
|
|
Mirrors test_web_capture_tool_warns_on_target_error_status below:
|
|
Browserless returns HTTP 200 for the render request even when the target
|
|
page is a 404 or an anti-bot block page, so fetch_html_with_status's
|
|
target-status headers must produce the same visible warning
|
|
web_capture_tool already gives via _target_status_warning.
|
|
"""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><h1>Not Found</h1><p>The page does not exist.</p></article></body></html>",
|
|
target_status_code="404",
|
|
target_status="Not Found",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com/missing")
|
|
|
|
assert "Error:" not in result
|
|
assert "warning: target page responded 404 Not Found" in result
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_no_warning_for_normal_target_status(self, mock_get_client):
|
|
"""web_fetch_tool does not warn when the target page responded normally.
|
|
|
|
Regression guard: a legitimate 200-target-page fetch must be unaffected
|
|
by the target-status warning added for error/blocked pages.
|
|
"""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><h1>Title</h1><p>Content</p></article></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com/article")
|
|
|
|
assert "Error:" not in result
|
|
assert "warning:" not in result
|
|
|
|
async def test_web_fetch_and_web_capture_tools_agree_on_target_error_warning(self, tmp_path):
|
|
"""web_fetch_tool and web_capture_tool surface the identical warning for identical target-error headers.
|
|
|
|
Both tools sit on top of the same Browserless target-status headers
|
|
(X-Response-Code / X-Response-Status). Before this fix, only
|
|
web_capture_tool (via capture_screenshot + _target_status_warning)
|
|
surfaced them; web_fetch_tool (via fetch_html_with_status) silently
|
|
returned the blocked page's raw content with no indication anything was
|
|
wrong. This drives both real tool functions with the same headers and
|
|
asserts they now agree.
|
|
"""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
fetch_client = MagicMock()
|
|
fetch_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><h1>Blocked</h1><p>Access denied.</p></article></body></html>",
|
|
target_status_code="404",
|
|
target_status="Not Found",
|
|
)
|
|
)
|
|
|
|
capture_client = MagicMock()
|
|
capture_client.capture_screenshot = AsyncMock(
|
|
return_value=BrowserlessScreenshotResult(
|
|
content=b"\x89PNG\r\n\x1a\nimage",
|
|
content_type="image/png",
|
|
target_status_code="404",
|
|
target_status="Not Found",
|
|
final_url="https://example.com/missing",
|
|
)
|
|
)
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client", return_value=fetch_client):
|
|
fetch_result = await tools.web_fetch_tool.ainvoke("https://example.com/missing")
|
|
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client", return_value=capture_client):
|
|
capture_command = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com/missing",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
capture_message = capture_command.update["messages"][0].content
|
|
|
|
assert "warning: target page responded 404 Not Found" in fetch_result
|
|
assert "warning: target page responded 404 Not Found" in capture_message
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_capture_tool_writes_artifact(self, mock_get_client, tmp_path):
|
|
"""web_capture_tool writes screenshots into thread outputs and presents the artifact."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
mock_client = MagicMock()
|
|
mock_client.capture_screenshot = AsyncMock(
|
|
return_value=BrowserlessScreenshotResult(
|
|
content=b"\x89PNG\r\n\x1a\nimage",
|
|
content_type="image/png",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
final_url="https://example.com/final",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg:
|
|
mock_cfg.side_effect = lambda name: {
|
|
"web_capture": {
|
|
"full_page": False,
|
|
"output_format": "png",
|
|
"viewport_width": 1024,
|
|
"viewport_height": 768,
|
|
"wait_for_selector": "main",
|
|
"wait_for_selector_timeout_ms": 4000,
|
|
"wait_for_timeout_ms": 250,
|
|
"best_attempt": True,
|
|
}
|
|
}.get(name)
|
|
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com/dashboard",
|
|
tool_call_id="tool-1",
|
|
filename="Dashboard Capture.png",
|
|
)
|
|
|
|
artifact_path = result.update["artifacts"][0]
|
|
assert artifact_path == "/mnt/user-data/outputs/Dashboard_Capture.png"
|
|
written = outputs_dir / "Dashboard_Capture.png"
|
|
assert written.read_bytes() == b"\x89PNG\r\n\x1a\nimage"
|
|
assert result.update["messages"][0].content == "Captured screenshot: /mnt/user-data/outputs/Dashboard_Capture.png"
|
|
mock_cfg.assert_any_call("web_capture")
|
|
mock_client.capture_screenshot.assert_called_once_with(
|
|
url="https://example.com/dashboard",
|
|
full_page=False,
|
|
output_format="png",
|
|
quality=None,
|
|
viewport={"width": 1024, "height": 768},
|
|
wait_for_selector="main",
|
|
wait_for_selector_timeout_ms=4000,
|
|
wait_for_timeout_ms=250,
|
|
best_attempt=True,
|
|
)
|
|
|
|
async def test_web_capture_tool_rejects_non_http_url(self, tmp_path):
|
|
"""web_capture_tool only accepts explicit http(s) URLs."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client:
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="file:///etc/passwd",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "Only http:// and https:// URLs are supported" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|
|
assert list(outputs_dir.iterdir()) == []
|
|
|
|
async def test_web_capture_tool_rejects_loopback_url(self, tmp_path):
|
|
"""web_capture_tool blocks loopback/localhost targets to prevent SSRF."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client:
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="http://localhost:8080/admin",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "private or loopback" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|
|
assert list(outputs_dir.iterdir()) == []
|
|
|
|
async def test_web_capture_tool_rejects_metadata_ip(self, tmp_path):
|
|
"""web_capture_tool blocks the cloud-metadata link-local endpoint."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client:
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="http://169.254.169.254/latest/meta-data/",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "private, loopback, or metadata" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|
|
|
|
async def test_web_capture_tool_rejects_dns_resolving_to_private(self, tmp_path):
|
|
"""web_capture_tool blocks a public hostname that resolves to an internal IP."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("10.0.0.5")],
|
|
):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client:
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://internal.example.com/",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "private, loopback, or metadata" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|
|
|
|
async def test_web_capture_tool_allows_private_when_opted_in(self, tmp_path):
|
|
"""web_capture_tool honors allow_private_addresses for internal targets."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
mock_client = MagicMock()
|
|
mock_client.capture_screenshot = AsyncMock(
|
|
return_value=BrowserlessScreenshotResult(
|
|
content=b"\x89PNG\r\n\x1a\nimage",
|
|
content_type="image/png",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
final_url="http://10.0.0.5/final",
|
|
)
|
|
)
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value={"allow_private_addresses": True}):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client", return_value=mock_client):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="http://10.0.0.5/dashboard",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert result.update["artifacts"][0].startswith("/mnt/user-data/outputs/")
|
|
assert (outputs_dir / result.update["artifacts"][0].split("/")[-1]).read_bytes() == b"\x89PNG\r\n\x1a\nimage"
|
|
mock_client.capture_screenshot.assert_called_once()
|
|
|
|
async def test_web_capture_tool_warns_on_target_error_status(self, tmp_path):
|
|
"""web_capture_tool surfaces a warning when the captured page itself errored."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
mock_client = MagicMock()
|
|
mock_client.capture_screenshot = AsyncMock(
|
|
return_value=BrowserlessScreenshotResult(
|
|
content=b"\x89PNG\r\n\x1a\nimage",
|
|
content_type="image/png",
|
|
target_status_code="404",
|
|
target_status="Not Found",
|
|
final_url="https://example.com/missing",
|
|
)
|
|
)
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client", return_value=mock_client):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com/missing",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
message = result.update["messages"][0].content
|
|
assert "warning: target page responded 404 Not Found" in message
|
|
assert result.update["artifacts"]
|
|
|
|
async def test_web_capture_tool_dedupes_existing_filename(self, tmp_path):
|
|
"""web_capture_tool appends a suffix instead of overwriting an existing capture."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
(outputs_dir / "report.png").write_bytes(b"existing")
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
mock_client = MagicMock()
|
|
mock_client.capture_screenshot = AsyncMock(
|
|
return_value=BrowserlessScreenshotResult(
|
|
content=b"\x89PNG\r\n\x1a\nnew",
|
|
content_type="image/png",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
final_url="https://example.com/",
|
|
)
|
|
)
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client", return_value=mock_client):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com/",
|
|
tool_call_id="tool-1",
|
|
filename="report.png",
|
|
)
|
|
|
|
assert result.update["artifacts"] == ["/mnt/user-data/outputs/report-1.png"]
|
|
assert (outputs_dir / "report.png").read_bytes() == b"existing"
|
|
assert (outputs_dir / "report-1.png").read_bytes() == b"\x89PNG\r\n\x1a\nnew"
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_capture_tool_missing_outputs_path_returns_error(self, mock_get_client):
|
|
"""web_capture_tool requires ThreadDataMiddleware outputs_path."""
|
|
runtime = SimpleNamespace(state={"thread_data": {}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "Thread outputs path is not available" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|