fix(web_fetch): support proxy for Jina reader in restricted networks (#3418) (#3430)

* fix(web_fetch): support proxy for Jina reader in restricted networks

The web_fetch tool built a bare httpx.AsyncClient() with no proxy
awareness, so users behind a corporate proxy / in Docker / WSL could
not reach https://r.jina.ai and web_fetch timed out.

- Add optional `proxy` / `trust_env` params to JinaClient.crawl and
  wire them from the `web_fetch` tool config (with type coercion for
  YAML string values).
- Pass internal service hostnames through NO_PROXY in both compose
  files so proxy env inherited via env_file does not break in-cluster
  calls (gateway/provisioner/etc).
- Load proxy vars from .env into the shell in scripts/docker.sh so the
  NO_PROXY interpolation can merge user-provided values on `make` path.
- Document proxy/trust_env options in config.example.yaml.

Closes #3418

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Ryker_Feng
2026-06-08 23:25:29 +08:00
committed by GitHub
parent 3b6dd0a4e3
commit f92a26d56f
7 changed files with 254 additions and 6 deletions
+28
View File
@@ -15,6 +15,32 @@ DOCKER_DIR="$PROJECT_ROOT/docker"
# Docker Compose command with project name
COMPOSE_CMD="docker compose -p deer-flow-dev -f docker-compose-dev.yaml"
load_proxy_env_from_dotenv() {
local env_file="$PROJECT_ROOT/.env"
local var
local line
local value
if [ ! -f "$env_file" ]; then
return
fi
for var in HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy; do
if [ -z "${!var+x}" ]; then
line="$(grep -E "^[[:space:]]*${var}=" "$env_file" | tail -n 1 || true)"
if [ -n "$line" ]; then
value="${line#*=}"
value="${value%\"}"
value="${value#\"}"
value="${value%\'}"
value="${value#\'}"
value="${value%$'\r'}"
export "${var}=${value}"
fi
fi
done
}
detect_sandbox_mode() {
local config_file="$PROJECT_ROOT/config.yaml"
local sandbox_use=""
@@ -220,6 +246,8 @@ start() {
fi
fi
load_proxy_env_from_dotenv
echo "Building and starting containers..."
cd "$DOCKER_DIR" && $COMPOSE_CMD up --build -d --remove-orphans $services
echo ""