Shengyuan Wang
2f3744f807
* refactor: replace sync requests with async httpx in Jina AI client Replace synchronous `requests.post()` with `httpx.AsyncClient` in JinaClient.crawl() and make web_fetch_tool async. This is part of the planned async concurrency optimization for the agent hot path (see docs/TODO.md). * fix: address Copilot review feedback on async Jina client - Short-circuit error strings in web_fetch_tool before passing to ReadabilityExtractor, preventing misleading extraction results - Log missing JINA_API_KEY warning only once per process to reduce noise under concurrent async fetching - Use logger.exception instead of logger.error in crawl exception handler to preserve stack traces for debugging - Add async web_fetch_tool tests and warn-once coverage * fix: mock get_app_config in web_fetch_tool tests for CI The web_fetch_tool tests failed in CI because get_app_config requires a config.yaml file that isn't present in the test environment. Mock the config loader to remove the filesystem dependency. --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
44 lines
1.6 KiB
Python
44 lines
1.6 KiB
Python
import logging
|
|
import os
|
|
|
|
import httpx
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_api_key_warned = False
|
|
|
|
|
|
class JinaClient:
|
|
async def crawl(self, url: str, return_format: str = "html", timeout: int = 10) -> str:
|
|
global _api_key_warned
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"X-Return-Format": return_format,
|
|
"X-Timeout": str(timeout),
|
|
}
|
|
if os.getenv("JINA_API_KEY"):
|
|
headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}"
|
|
elif not _api_key_warned:
|
|
_api_key_warned = True
|
|
logger.warning("Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information.")
|
|
data = {"url": url}
|
|
try:
|
|
async with httpx.AsyncClient() as client:
|
|
response = await client.post("https://r.jina.ai/", headers=headers, json=data, timeout=timeout)
|
|
|
|
if response.status_code != 200:
|
|
error_message = f"Jina API returned status {response.status_code}: {response.text}"
|
|
logger.error(error_message)
|
|
return f"Error: {error_message}"
|
|
|
|
if not response.text or not response.text.strip():
|
|
error_message = "Jina API returned empty response"
|
|
logger.error(error_message)
|
|
return f"Error: {error_message}"
|
|
|
|
return response.text
|
|
except Exception as e:
|
|
error_message = f"Request to Jina API failed: {str(e)}"
|
|
logger.exception(error_message)
|
|
return f"Error: {error_message}"
|