mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-07-21 10:15:47 +00:00
Revert "fix(firecrawl): align tools with installed firecrawl-py v2 SDK surfac…" (#4165)
This reverts commit b7b4b49bfe.
This commit is contained in:
Willem Jiang
@@ -1,18 +1,17 @@
|
||||
import json
|
||||
|
||||
from firecrawl import Firecrawl
|
||||
from firecrawl.v2.types import ScrapeOptions
|
||||
from firecrawl import FirecrawlApp
|
||||
from langchain.tools import tool
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
|
||||
def _get_firecrawl_client(tool_name: str = "web_search") -> Firecrawl:
|
||||
def _get_firecrawl_client(tool_name: str = "web_search") -> FirecrawlApp:
|
||||
config = get_app_config().get_tool_config(tool_name)
|
||||
api_key = None
|
||||
if config is not None and "api_key" in config.model_extra:
|
||||
api_key = config.model_extra.get("api_key")
|
||||
return Firecrawl(api_key=api_key) # type: ignore[arg-type]
|
||||
return FirecrawlApp(api_key=api_key) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@tool("web_search", parse_docstring=True)
|
||||
@@ -59,15 +58,8 @@ def web_fetch_tool(url: str) -> str:
|
||||
url: The URL to fetch the contents of.
|
||||
"""
|
||||
try:
|
||||
config = get_app_config().get_tool_config("web_fetch")
|
||||
max_chars = 16384
|
||||
formats = ["markdown"]
|
||||
if config is not None:
|
||||
max_chars = config.model_extra.get("max_chars", max_chars)
|
||||
formats = config.model_extra.get("formats", formats)
|
||||
|
||||
client = _get_firecrawl_client("web_fetch")
|
||||
result = client.scrape(url, formats=formats)
|
||||
result = client.scrape(url, formats=["markdown"])
|
||||
|
||||
markdown_content = result.markdown or ""
|
||||
metadata = result.metadata
|
||||
@@ -78,220 +70,4 @@ def web_fetch_tool(url: str) -> str:
|
||||
except Exception as e:
|
||||
return f"Error: {str(e)}"
|
||||
|
||||
return f"# {title}\n\n{markdown_content[:max_chars]}"
|
||||
|
||||
|
||||
@tool("web_map", parse_docstring=True)
|
||||
def web_map_tool(url: str) -> str:
|
||||
"""Discover URLs on a website. Acts like a sitemap generator — given a URL,
|
||||
returns a list of URLs found on that site.
|
||||
|
||||
Args:
|
||||
url: The website URL to map (e.g. https://example.com).
|
||||
"""
|
||||
try:
|
||||
config = get_app_config().get_tool_config("web_map")
|
||||
limit = 100
|
||||
include_subdomains = True
|
||||
ignore_sitemap = False
|
||||
if config is not None:
|
||||
limit = config.model_extra.get("limit", limit)
|
||||
ignore_sitemap = config.model_extra.get("ignore_sitemap", ignore_sitemap)
|
||||
include_subdomains = config.model_extra.get("include_subdomains", include_subdomains)
|
||||
|
||||
client = _get_firecrawl_client("web_map")
|
||||
result = client.map(
|
||||
url,
|
||||
limit=limit,
|
||||
include_subdomains=include_subdomains,
|
||||
sitemap="skip" if ignore_sitemap else None,
|
||||
)
|
||||
|
||||
urls = getattr(result, "links", None) or []
|
||||
normalized = {
|
||||
"urls": urls,
|
||||
"total_count": len(urls),
|
||||
}
|
||||
return json.dumps(normalized, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"Error: {str(e)}"
|
||||
|
||||
|
||||
@tool("web_crawl", parse_docstring=True)
|
||||
def web_crawl_tool(url: str) -> str:
|
||||
"""Crawl a website starting from a given URL.
|
||||
|
||||
Args:
|
||||
url: The starting URL to crawl from.
|
||||
"""
|
||||
try:
|
||||
config = get_app_config().get_tool_config("web_crawl")
|
||||
max_discovery_depth = 2
|
||||
limit = 10
|
||||
allow_subdomains = True
|
||||
scrape_formats = ["markdown"]
|
||||
if config is not None:
|
||||
max_discovery_depth = config.model_extra.get("max_depth", max_discovery_depth)
|
||||
limit = config.model_extra.get("limit", limit)
|
||||
allow_subdomains = config.model_extra.get("allow_subdomains", allow_subdomains)
|
||||
scrape_formats = config.model_extra.get("scrape_formats", scrape_formats)
|
||||
|
||||
client = _get_firecrawl_client("web_crawl")
|
||||
result = client.crawl(
|
||||
url,
|
||||
max_discovery_depth=max_discovery_depth,
|
||||
limit=limit,
|
||||
allow_subdomains=allow_subdomains,
|
||||
scrape_options=ScrapeOptions(formats=scrape_formats), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
pages = []
|
||||
for page in result.data or []:
|
||||
page_url = ""
|
||||
page_md = ""
|
||||
if isinstance(page, dict):
|
||||
page_url = page.get("url", "") or ""
|
||||
page_md = page.get("markdown", "") or ""
|
||||
else:
|
||||
page_url = getattr(page, "url", None) or ""
|
||||
page_md = getattr(page, "markdown", None) or ""
|
||||
pages.append(
|
||||
{
|
||||
"url": page_url,
|
||||
"markdown": page_md[:8192],
|
||||
}
|
||||
)
|
||||
|
||||
return json.dumps(pages, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"Error: {str(e)}"
|
||||
|
||||
|
||||
@tool("web_interact", parse_docstring=True)
|
||||
def web_interact_tool(url: str, actions: str) -> str:
|
||||
"""Interact with a web page using browser automation. Opens a browser, performs
|
||||
the described actions, and returns the final page content.
|
||||
|
||||
Use this tool to fill forms, click buttons, navigate dynamic pages, or any
|
||||
other browser-based interaction that requires simulating user behavior.
|
||||
|
||||
``actions`` must be a JSON array of action objects. Each object has a ``type``
|
||||
field and type-specific fields:
|
||||
|
||||
- ``{"type": "wait", "milliseconds": 2000}`` — wait N ms
|
||||
- ``{"type": "click", "selector": "#login-btn"}`` — click an element
|
||||
- ``{"type": "write", "selector": "#search", "text": "hello"}`` — type text
|
||||
- ``{"type": "press", "key": "Enter"}`` — press a key
|
||||
- ``{"type": "scroll", "direction": "down", "amount": 300}`` — scroll
|
||||
- ``{"type": "screenshot", "full_page": true}`` — take a screenshot
|
||||
- ``{"type": "execute_javascript", "code": "..."}`` — run JS
|
||||
- ``{"type": "scrape", "selector": "body"}`` — extract HTML from a selector
|
||||
|
||||
Args:
|
||||
url: The starting URL to open in the browser.
|
||||
actions: JSON array of browser action objects to perform on the page.
|
||||
"""
|
||||
try:
|
||||
config = get_app_config().get_tool_config("web_interact")
|
||||
timeout = 30000
|
||||
wait_for = 2000
|
||||
formats = ["markdown"]
|
||||
if config is not None:
|
||||
timeout = config.model_extra.get("timeout", timeout)
|
||||
wait_for = config.model_extra.get("wait_for", wait_for)
|
||||
formats = config.model_extra.get("formats", formats)
|
||||
|
||||
# Parse the action objects from JSON
|
||||
action_dicts = json.loads(actions)
|
||||
if not isinstance(action_dicts, list):
|
||||
return "Error: actions must be a JSON array of action objects"
|
||||
|
||||
# Build structured action objects
|
||||
parsed_actions = []
|
||||
for act in action_dicts:
|
||||
act_type = act.get("type", "")
|
||||
if act_type == "wait":
|
||||
parsed_actions.append({"type": "wait", "milliseconds": act.get("milliseconds", 1000)})
|
||||
elif act_type == "click":
|
||||
parsed_actions.append({"type": "click", "selector": act["selector"]})
|
||||
elif act_type == "write":
|
||||
parsed_actions.append({"type": "write", "selector": act["selector"], "text": act["text"]})
|
||||
elif act_type == "press":
|
||||
parsed_actions.append({"type": "press", "key": act["key"]})
|
||||
elif act_type == "scroll":
|
||||
parsed_actions.append({"type": "scroll", "direction": act.get("direction", "down"), "amount": act.get("amount", 300)})
|
||||
elif act_type == "screenshot":
|
||||
parsed_actions.append({"type": "screenshot", "full_page": act.get("full_page", False)})
|
||||
elif act_type == "execute_javascript":
|
||||
parsed_actions.append({"type": "executeJavascript", "code": act["code"]})
|
||||
elif act_type == "scrape":
|
||||
parsed_actions.append({"type": "scrape", "selector": act.get("selector", "body")})
|
||||
else:
|
||||
return f"Error: unknown action type '{act_type}'"
|
||||
|
||||
client = _get_firecrawl_client("web_interact")
|
||||
result = client.scrape(
|
||||
url,
|
||||
formats=formats, # type: ignore[arg-type]
|
||||
actions=parsed_actions, # type: ignore[arg-type]
|
||||
timeout=timeout,
|
||||
wait_for=wait_for,
|
||||
)
|
||||
|
||||
markdown_content = getattr(result, "markdown", None) or ""
|
||||
if not markdown_content:
|
||||
return "Error: No content returned"
|
||||
|
||||
return markdown_content
|
||||
except json.JSONDecodeError:
|
||||
return "Error: actions must be valid JSON"
|
||||
except Exception as e:
|
||||
return f"Error: {str(e)}"
|
||||
|
||||
|
||||
@tool("structured_extract", parse_docstring=True)
|
||||
def structured_extract_tool(
|
||||
urls: list[str],
|
||||
prompt: str,
|
||||
schema: dict | None = None,
|
||||
) -> str:
|
||||
"""Extract structured data from web pages using an LLM-powered schema.
|
||||
|
||||
Provide a list of URLs and a prompt describing what data to extract. Optionally
|
||||
supply a JSON schema dict to enforce a specific output structure; if omitted,
|
||||
Firecrawl will infer the schema from the prompt.
|
||||
|
||||
Args:
|
||||
urls: List of URLs to extract data from.
|
||||
prompt: Natural-language description of the data to extract (e.g. "Extract product name, price, and rating for each product").
|
||||
schema: Optional JSON schema dict defining the expected output structure.
|
||||
"""
|
||||
try:
|
||||
config = get_app_config().get_tool_config("structured_extract")
|
||||
max_urls = 10
|
||||
if config is not None:
|
||||
max_urls = config.model_extra.get("max_urls", max_urls)
|
||||
|
||||
client = _get_firecrawl_client("structured_extract")
|
||||
|
||||
trimmed_urls = urls[:max_urls]
|
||||
kwargs = {
|
||||
"urls": trimmed_urls,
|
||||
"prompt": prompt,
|
||||
}
|
||||
if schema is not None:
|
||||
kwargs["schema"] = schema
|
||||
|
||||
result = client.extract(**kwargs)
|
||||
|
||||
data = getattr(result, "data", None)
|
||||
if data is None:
|
||||
return json.dumps(
|
||||
{"error": "No data returned from extraction", "success": False},
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
return json.dumps(data, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"Error: {str(e)}"
|
||||
return f"# {title}\n\n{markdown_content[:4096]}"
|
||||
|
||||
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestWebSearchTool:
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.FirecrawlApp")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_search_uses_web_search_config(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
search_config = MagicMock()
|
||||
@@ -35,15 +35,11 @@ class TestWebSearchTool:
|
||||
|
||||
|
||||
class TestWebFetchTool:
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.FirecrawlApp")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_fetch_uses_web_fetch_config(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
fetch_config = MagicMock()
|
||||
fetch_config.model_extra = {
|
||||
"api_key": "firecrawl-fetch-key",
|
||||
"max_chars": 50,
|
||||
"formats": ["markdown", "html"],
|
||||
}
|
||||
fetch_config.model_extra = {"api_key": "firecrawl-fetch-key"}
|
||||
|
||||
def get_tool_config(name):
|
||||
if name == "web_fetch":
|
||||
@@ -52,9 +48,8 @@ class TestWebFetchTool:
|
||||
|
||||
mock_get_app_config.return_value.get_tool_config.side_effect = get_tool_config
|
||||
|
||||
long_content = "This is a much longer markdown content that exceeds fifty characters."
|
||||
mock_scrape_result = MagicMock()
|
||||
mock_scrape_result.markdown = long_content
|
||||
mock_scrape_result.markdown = "Fetched markdown"
|
||||
mock_scrape_result.metadata = MagicMock(title="Fetched Page")
|
||||
mock_firecrawl_cls.return_value.scrape.return_value = mock_scrape_result
|
||||
|
||||
@@ -62,482 +57,10 @@ class TestWebFetchTool:
|
||||
|
||||
result = web_fetch_tool.invoke({"url": "https://example.com"})
|
||||
|
||||
# Content should be truncated to max_chars (50)
|
||||
expected = "# Fetched Page\n\n" + long_content[:50]
|
||||
assert result == expected
|
||||
assert len(result) == len("# Fetched Page\n\n") + 50
|
||||
assert result == "# Fetched Page\n\nFetched markdown"
|
||||
mock_get_app_config.return_value.get_tool_config.assert_any_call("web_fetch")
|
||||
mock_firecrawl_cls.assert_called_once_with(api_key="firecrawl-fetch-key")
|
||||
mock_firecrawl_cls.return_value.scrape.assert_called_once_with(
|
||||
"https://example.com",
|
||||
formats=["markdown", "html"],
|
||||
formats=["markdown"],
|
||||
)
|
||||
|
||||
|
||||
class TestWebMapTool:
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_map_success(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""map returns a list of URLs — they should be serialized as JSON."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {"api_key": "map-key"}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.links = [
|
||||
"https://example.com",
|
||||
"https://example.com/about",
|
||||
"https://example.com/contact",
|
||||
]
|
||||
mock_firecrawl_cls.return_value.map.return_value = mock_result
|
||||
|
||||
from deerflow.community.firecrawl.tools import web_map_tool
|
||||
|
||||
result = web_map_tool.invoke({"url": "https://example.com"})
|
||||
|
||||
parsed = json.loads(result)
|
||||
assert parsed["urls"] == [
|
||||
"https://example.com",
|
||||
"https://example.com/about",
|
||||
"https://example.com/contact",
|
||||
]
|
||||
assert parsed["total_count"] == 3
|
||||
mock_firecrawl_cls.return_value.map.assert_called_once_with(
|
||||
"https://example.com",
|
||||
limit=100,
|
||||
include_subdomains=True,
|
||||
sitemap=None,
|
||||
)
|
||||
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_map_respects_config(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""Config values for limit, ignore_sitemap, include_subdomains pass through to map."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {
|
||||
"api_key": "cfg-key",
|
||||
"limit": 42,
|
||||
"ignore_sitemap": True,
|
||||
"include_subdomains": False,
|
||||
}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.links = ["https://example.com/page"]
|
||||
mock_firecrawl_cls.return_value.map.return_value = mock_result
|
||||
|
||||
from deerflow.community.firecrawl.tools import web_map_tool
|
||||
|
||||
web_map_tool.invoke({"url": "https://example.com"})
|
||||
|
||||
mock_get_app_config.return_value.get_tool_config.assert_called_with("web_map")
|
||||
mock_firecrawl_cls.return_value.map.assert_called_once_with(
|
||||
"https://example.com",
|
||||
limit=42,
|
||||
include_subdomains=False,
|
||||
sitemap="skip",
|
||||
)
|
||||
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_map_empty_site(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""Empty site — no URLs found — returns empty list with count 0."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {"api_key": "map-key"}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.links = []
|
||||
mock_firecrawl_cls.return_value.map.return_value = mock_result
|
||||
|
||||
from deerflow.community.firecrawl.tools import web_map_tool
|
||||
|
||||
result = web_map_tool.invoke({"url": "https://empty.example.com"})
|
||||
|
||||
parsed = json.loads(result)
|
||||
assert parsed["urls"] == []
|
||||
assert parsed["total_count"] == 0
|
||||
|
||||
|
||||
class TestWebCrawlTool:
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_crawl_success(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""crawl returns a list of pages — they should be serialized as JSON with url + markdown."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {"api_key": "crawl-key"}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
mock_doc1 = MagicMock()
|
||||
mock_doc1.url = "https://example.com"
|
||||
mock_doc1.markdown = "Page 1 content"
|
||||
mock_doc2 = MagicMock()
|
||||
mock_doc2.url = "https://example.com/page2"
|
||||
mock_doc2.markdown = "Page 2 content"
|
||||
mock_result = MagicMock()
|
||||
mock_result.data = [mock_doc1, mock_doc2]
|
||||
mock_firecrawl_cls.return_value.crawl.return_value = mock_result
|
||||
|
||||
from deerflow.community.firecrawl.tools import web_crawl_tool
|
||||
|
||||
result = web_crawl_tool.invoke({"url": "https://example.com"})
|
||||
|
||||
parsed = json.loads(result)
|
||||
assert len(parsed) == 2
|
||||
assert parsed[0]["url"] == "https://example.com"
|
||||
assert parsed[0]["markdown"] == "Page 1 content"
|
||||
assert parsed[1]["url"] == "https://example.com/page2"
|
||||
assert parsed[1]["markdown"] == "Page 2 content"
|
||||
|
||||
@patch("deerflow.community.firecrawl.tools.ScrapeOptions")
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_crawl_respects_config(self, mock_get_app_config, mock_firecrawl_cls, mock_scrape_options):
|
||||
"""Config values for max_depth, limit, allow_subdomains, scrape_formats pass through to crawl."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {
|
||||
"api_key": "cfg-key",
|
||||
"max_depth": 3,
|
||||
"limit": 20,
|
||||
"allow_subdomains": False,
|
||||
"scrape_formats": ["html"],
|
||||
}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.data = []
|
||||
mock_firecrawl_cls.return_value.crawl.return_value = mock_result
|
||||
mock_scrape_options.return_value = "scrape_opts_mock"
|
||||
|
||||
from deerflow.community.firecrawl.tools import web_crawl_tool
|
||||
|
||||
web_crawl_tool.invoke({"url": "https://example.com"})
|
||||
|
||||
mock_get_app_config.return_value.get_tool_config.assert_called_with("web_crawl")
|
||||
mock_scrape_options.assert_called_once_with(formats=["html"])
|
||||
mock_firecrawl_cls.return_value.crawl.assert_called_once_with(
|
||||
"https://example.com",
|
||||
max_discovery_depth=3,
|
||||
limit=20,
|
||||
allow_subdomains=False,
|
||||
scrape_options="scrape_opts_mock",
|
||||
)
|
||||
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_crawl_error(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""Exceptions from crawl are caught and returned as 'Error: ...' string."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {"api_key": "crawl-key"}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
mock_firecrawl_cls.return_value.crawl.side_effect = RuntimeError("Crawl failed")
|
||||
|
||||
from deerflow.community.firecrawl.tools import web_crawl_tool
|
||||
|
||||
result = web_crawl_tool.invoke({"url": "https://example.com"})
|
||||
|
||||
assert result == "Error: Crawl failed"
|
||||
|
||||
|
||||
class TestWebInteractTool:
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_interact_success(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""scrape() with browser actions returns markdown content — it should be passed through as-is."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {"api_key": "interact-key"}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.markdown = "# Final Page\n\nContent after interaction."
|
||||
mock_firecrawl_cls.return_value.scrape.return_value = mock_result
|
||||
|
||||
from deerflow.community.firecrawl.tools import web_interact_tool
|
||||
|
||||
actions_json = json.dumps(
|
||||
[
|
||||
{"type": "click", "selector": "#login-btn"},
|
||||
{"type": "wait", "milliseconds": 1000},
|
||||
]
|
||||
)
|
||||
result = web_interact_tool.invoke(
|
||||
{
|
||||
"url": "https://example.com",
|
||||
"actions": actions_json,
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "# Final Page\n\nContent after interaction."
|
||||
mock_get_app_config.return_value.get_tool_config.assert_called_with("web_interact")
|
||||
mock_firecrawl_cls.return_value.scrape.assert_called_once()
|
||||
call_args, call_kwargs = mock_firecrawl_cls.return_value.scrape.call_args
|
||||
assert call_args[0] == "https://example.com"
|
||||
assert call_kwargs["formats"] == ["markdown"]
|
||||
assert call_kwargs["timeout"] == 30000
|
||||
assert call_kwargs["wait_for"] == 2000
|
||||
# Verify actions were parsed into dicts
|
||||
assert len(call_kwargs["actions"]) == 2
|
||||
assert call_kwargs["actions"][0] == {"type": "click", "selector": "#login-btn"}
|
||||
assert call_kwargs["actions"][1] == {"type": "wait", "milliseconds": 1000}
|
||||
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_interact_actions_pass_through(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""The actions JSON should be parsed into action dicts and passed to scrape()."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {"api_key": "interact-key"}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.markdown = "Done."
|
||||
mock_firecrawl_cls.return_value.scrape.return_value = mock_result
|
||||
|
||||
from deerflow.community.firecrawl.tools import web_interact_tool
|
||||
|
||||
actions_json = json.dumps(
|
||||
[
|
||||
{"type": "write", "selector": "#search", "text": "hello"},
|
||||
{"type": "press", "key": "Enter"},
|
||||
]
|
||||
)
|
||||
web_interact_tool.invoke(
|
||||
{
|
||||
"url": "https://example.com",
|
||||
"actions": actions_json,
|
||||
}
|
||||
)
|
||||
|
||||
call_kwargs = mock_firecrawl_cls.return_value.scrape.call_args.kwargs
|
||||
assert call_kwargs["actions"] == [
|
||||
{"type": "write", "selector": "#search", "text": "hello"},
|
||||
{"type": "press", "key": "Enter"},
|
||||
]
|
||||
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_interact_error(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""Exceptions from scrape() are caught and returned as 'Error: ...' string."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {"api_key": "interact-key"}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
mock_firecrawl_cls.return_value.scrape.side_effect = RuntimeError("Browser interaction timed out")
|
||||
|
||||
from deerflow.community.firecrawl.tools import web_interact_tool
|
||||
|
||||
actions_json = json.dumps([{"type": "click", "selector": "#btn"}])
|
||||
result = web_interact_tool.invoke(
|
||||
{
|
||||
"url": "https://example.com",
|
||||
"actions": actions_json,
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "Error: Browser interaction timed out"
|
||||
|
||||
|
||||
class TestStructuredExtractTool:
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_extract_with_schema(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""extract() with a schema returns the structured data as JSON."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {"api_key": "extract-key"}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
extracted_data = {
|
||||
"products": [
|
||||
{"name": "Widget", "price": 9.99, "rating": 4.5},
|
||||
{"name": "Gadget", "price": 19.99, "rating": 4.2},
|
||||
]
|
||||
}
|
||||
mock_result = MagicMock()
|
||||
mock_result.data = extracted_data
|
||||
mock_firecrawl_cls.return_value.extract.return_value = mock_result
|
||||
|
||||
from deerflow.community.firecrawl.tools import structured_extract_tool
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"products": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"price": {"type": "number"},
|
||||
"rating": {"type": "number"},
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
result = structured_extract_tool.invoke(
|
||||
{
|
||||
"urls": ["https://shop.example.com"],
|
||||
"prompt": "Extract product name, price, and rating",
|
||||
"schema": schema,
|
||||
}
|
||||
)
|
||||
|
||||
parsed = json.loads(result)
|
||||
assert parsed == extracted_data
|
||||
mock_get_app_config.return_value.get_tool_config.assert_called_with("structured_extract")
|
||||
mock_firecrawl_cls.return_value.extract.assert_called_once_with(
|
||||
urls=["https://shop.example.com"],
|
||||
prompt="Extract product name, price, and rating",
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_extract_no_schema(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""extract() without a schema (prompt-only) infers structure from the prompt."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {"api_key": "extract-key"}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
extracted_data = {"title": "Example", "summary": "A test summary"}
|
||||
mock_result = MagicMock()
|
||||
mock_result.data = extracted_data
|
||||
mock_firecrawl_cls.return_value.extract.return_value = mock_result
|
||||
|
||||
from deerflow.community.firecrawl.tools import structured_extract_tool
|
||||
|
||||
result = structured_extract_tool.invoke(
|
||||
{
|
||||
"urls": ["https://example.com"],
|
||||
"prompt": "Extract the title and summary",
|
||||
}
|
||||
)
|
||||
|
||||
parsed = json.loads(result)
|
||||
assert parsed == extracted_data
|
||||
# Schema should NOT be passed when omitted
|
||||
mock_firecrawl_cls.return_value.extract.assert_called_once_with(
|
||||
urls=["https://example.com"],
|
||||
prompt="Extract the title and summary",
|
||||
)
|
||||
|
||||
@patch("deerflow.community.firecrawl.tools.Firecrawl")
|
||||
@patch("deerflow.community.firecrawl.tools.get_app_config")
|
||||
def test_extract_error(self, mock_get_app_config, mock_firecrawl_cls):
|
||||
"""Exceptions from extract() are caught and returned as 'Error: ...' string."""
|
||||
config = MagicMock()
|
||||
config.model_extra = {"api_key": "extract-key"}
|
||||
mock_get_app_config.return_value.get_tool_config.return_value = config
|
||||
|
||||
mock_firecrawl_cls.return_value.extract.side_effect = ValueError("Invalid URL format")
|
||||
|
||||
from deerflow.community.firecrawl.tools import structured_extract_tool
|
||||
|
||||
result = structured_extract_tool.invoke(
|
||||
{
|
||||
"urls": ["not-a-url"],
|
||||
"prompt": "Extract data",
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "Error: Invalid URL format"
|
||||
|
||||
|
||||
class TestRealFirecrawlAppContract:
|
||||
"""Verify that the methods called by tools.py actually exist on the real
|
||||
firecrawl-py FirecrawlApp with the parameter names we expect.
|
||||
|
||||
This guards against SDK upgrades changing signatures without our tools
|
||||
being updated.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def test_client_methods_exist():
|
||||
"""All methods used by tools.py must be present on a Firecrawl instance."""
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
client = Firecrawl(api_key="test-contract-key")
|
||||
|
||||
expected_methods = ["map", "crawl", "scrape", "search", "extract"]
|
||||
for name in expected_methods:
|
||||
method = getattr(client, name, None)
|
||||
assert method is not None, f"Firecrawl instance is missing '{name}' method — tools.py will fail at runtime"
|
||||
|
||||
@staticmethod
|
||||
def test_map_signature():
|
||||
"""map() must accept url, limit, include_subdomains, sitemap."""
|
||||
import inspect
|
||||
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
client = Firecrawl(api_key="test-contract-key")
|
||||
sig = inspect.signature(client.map)
|
||||
params = sig.parameters
|
||||
|
||||
assert "url" in params, f"map() missing 'url' param; sig={sig}"
|
||||
assert "limit" in params, f"map() missing 'limit' param; sig={sig}"
|
||||
assert "include_subdomains" in params, f"map() missing 'include_subdomains' param; sig={sig}"
|
||||
assert "sitemap" in params, f"map() missing 'sitemap' param; sig={sig}"
|
||||
|
||||
@staticmethod
|
||||
def test_crawl_signature():
|
||||
"""crawl() must accept url, max_discovery_depth, limit, allow_subdomains, scrape_options."""
|
||||
import inspect
|
||||
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
client = Firecrawl(api_key="test-contract-key")
|
||||
sig = inspect.signature(client.crawl)
|
||||
params = sig.parameters
|
||||
|
||||
assert "url" in params, f"crawl() missing 'url' param; sig={sig}"
|
||||
assert "max_discovery_depth" in params, f"crawl() missing 'max_discovery_depth' param; sig={sig}"
|
||||
assert "limit" in params, f"crawl() missing 'limit' param; sig={sig}"
|
||||
assert "allow_subdomains" in params, f"crawl() missing 'allow_subdomains' param; sig={sig}"
|
||||
assert "scrape_options" in params, f"crawl() missing 'scrape_options' param; sig={sig}"
|
||||
|
||||
@staticmethod
|
||||
def test_scrape_signature():
|
||||
"""scrape() must accept url, formats, actions, timeout, wait_for."""
|
||||
import inspect
|
||||
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
client = Firecrawl(api_key="test-contract-key")
|
||||
sig = inspect.signature(client.scrape)
|
||||
params = sig.parameters
|
||||
|
||||
assert "url" in params, f"scrape() missing 'url' param; sig={sig}"
|
||||
assert "formats" in params, f"scrape() missing 'formats' param; sig={sig}"
|
||||
assert "actions" in params, f"scrape() missing 'actions' param; sig={sig}"
|
||||
assert "timeout" in params, f"scrape() missing 'timeout' param; sig={sig}"
|
||||
assert "wait_for" in params, f"scrape() missing 'wait_for' param; sig={sig}"
|
||||
|
||||
@staticmethod
|
||||
def test_search_signature():
|
||||
"""search() must accept query and limit."""
|
||||
import inspect
|
||||
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
client = Firecrawl(api_key="test-contract-key")
|
||||
sig = inspect.signature(client.search)
|
||||
params = sig.parameters
|
||||
|
||||
assert "query" in params, f"search() missing 'query' param; sig={sig}"
|
||||
assert "limit" in params, f"search() missing 'limit' param; sig={sig}"
|
||||
|
||||
@staticmethod
|
||||
def test_extract_signature():
|
||||
"""extract() must accept urls, prompt, and schema."""
|
||||
import inspect
|
||||
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
client = Firecrawl(api_key="test-contract-key")
|
||||
sig = inspect.signature(client.extract)
|
||||
params = sig.parameters
|
||||
|
||||
assert "urls" in params, f"extract() missing 'urls' param; sig={sig}"
|
||||
assert "prompt" in params, f"extract() missing 'prompt' param; sig={sig}"
|
||||
assert "schema" in params, f"extract() missing 'schema' param; sig={sig}"
|
||||
|
||||
Reference in New Issue
Block a user