furyhawk/deer-flow
1
0
Fork 0
mirror of https://github.com/bytedance/deer-flow.git synced 2026-05-24 17:06:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(docker): Refactor sandbox state management and improve Docker integration (#1068)

Browse Source 
* Refactor sandbox state management and improve Docker integration

- Removed FileSandboxStateStore and SandboxStateStore classes for a cleaner architecture.
- Enhanced LocalContainerBackend to handle port allocation retries and introduced environment variable support for sandbox host configuration.
- Updated Paths class to include host_base_dir for Docker volume mounts and ensured proper permissions for sandbox directories.
- Modified ExtensionsConfig to improve error handling when loading configuration files and adjusted environment variable resolution.
- Updated sandbox configuration to include a replicas option for managing concurrent sandbox containers.
- Improved logging and context management in SandboxMiddleware for better sandbox lifecycle handling.
- Enhanced network port allocation logic to bind to 0.0.0.0 for compatibility with Docker.
- Updated Docker Compose files to ensure proper volume management and environment variable configuration.
- Created scripts to ensure necessary configuration files are present before starting services.
- Cleaned up unused MCP server configurations in extensions_config.example.json.

* Address Copilot review suggestions from PR #1068 (#9)

---------

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
This commit is contained in:
JeffJiang
2026-03-11 10:03:01 +08:00
committed by GitHub
parent 6ae7f0c0ee
commit f836d8e17c
18 changed files with 455 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/community/aio_sandbox/aio_sandbox_provider.py
+222 -106
View File
@@ -1,18 +1,17 @@
"""AIO Sandbox Provider — orchestrates sandbox lifecycle with pluggable backends.
This provider composes two abstractions:
This provider composes:
- SandboxBackend: how sandboxes are provisioned (local container vs remote/K8s)
- SandboxStateStore: how thread→sandbox mappings are persisted (file vs Redis)
The provider itself handles:
- In-process caching for fast repeated access
- Thread-safe locking (in-process + cross-process via state store)
- Idle timeout management
- Graceful shutdown with signal handling
- Mount computation (thread-specific, skills)
"""
import atexit
import fcntl
import hashlib
import logging
import os
@@ -22,17 +21,15 @@ import time
import uuid
from src.config import get_app_config
from src.config.paths import VIRTUAL_PATH_PREFIX, get_paths
from src.config.paths import VIRTUAL_PATH_PREFIX, Paths, get_paths
from src.sandbox.sandbox import Sandbox
from src.sandbox.sandbox_provider import SandboxProvider
from .aio_sandbox import AioSandbox
from .backend import SandboxBackend, wait_for_sandbox_ready
from .file_state_store import FileSandboxStateStore
from .local_backend import LocalContainerBackend
from .remote_backend import RemoteSandboxBackend
from .sandbox_info import SandboxInfo
from .state_store import SandboxStateStore
logger = logging.getLogger(__name__)
@@ -41,6 +38,7 @@ DEFAULT_IMAGE = "enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in
DEFAULT_PORT = 8080
DEFAULT_CONTAINER_PREFIX = "deer-flow-sandbox"
DEFAULT_IDLE_TIMEOUT = 600 # 10 minutes in seconds
DEFAULT_REPLICAS = 3 # Maximum concurrent sandbox containers
IDLE_CHECK_INTERVAL = 60 # Check every 60 seconds
@@ -48,20 +46,17 @@ class AioSandboxProvider(SandboxProvider):
"""Sandbox provider that manages containers running the AIO sandbox.
Architecture:
This provider composes a SandboxBackend (how to provision) and a
SandboxStateStore (how to persist state), enabling:
This provider composes a SandboxBackend (how to provision), enabling:
- Local Docker/Apple Container mode (auto-start containers)
- Remote/K8s mode (connect to pre-existing sandbox URL)
- Cross-process consistency via file-based or Redis state stores
Configuration options in config.yaml under sandbox:
use: src.community.aio_sandbox:AioSandboxProvider
image: <container image>
port: 8080 # Base port for local containers
base_url: http://... # If set, uses remote backend (K8s/external)
auto_start: true # Whether to auto-start local containers
container_prefix: deer-flow-sandbox
idle_timeout: 600 # Idle timeout in seconds (0 to disable)
replicas: 3 # Max concurrent sandbox containers (LRU eviction when exceeded)
mounts: # Volume mounts for local containers
- host_path: /path/on/host
container_path: /path/in/container
@@ -78,13 +73,17 @@ class AioSandboxProvider(SandboxProvider):
self._thread_sandboxes: dict[str, str] = {} # thread_id -> sandbox_id
self._thread_locks: dict[str, threading.Lock] = {} # thread_id -> in-process lock
self._last_activity: dict[str, float] = {} # sandbox_id -> last activity timestamp
# Warm pool: released sandboxes whose containers are still running.
# Maps sandbox_id -> (SandboxInfo, release_timestamp).
# Containers here can be reclaimed quickly (no cold-start) or destroyed
# when replicas capacity is exhausted.
self._warm_pool: dict[str, tuple[SandboxInfo, float]] = {}
self._shutdown_called = False
self._idle_checker_stop = threading.Event()
self._idle_checker_thread: threading.Thread | None = None
self._config = self._load_config()
self._backend: SandboxBackend = self._create_backend()
self._state_store: SandboxStateStore = self._create_state_store()
# Register shutdown handler
atexit.register(self.shutdown)
@@ -102,16 +101,14 @@ class AioSandboxProvider(SandboxProvider):
Selection logic (checked in order):
1. ``provisioner_url`` set → RemoteSandboxBackend (provisioner mode)
Provisioner dynamically creates Pods + Services in k3s.
2. ``auto_start`` → LocalContainerBackend (Docker / Apple Container)
2. Default → LocalContainerBackend (local mode)
Local provider manages container lifecycle directly (start/stop).
"""
provisioner_url = self._config.get("provisioner_url")
if provisioner_url:
logger.info(f"Using remote sandbox backend with provisioner at {provisioner_url}")
return RemoteSandboxBackend(provisioner_url=provisioner_url)
if not self._config.get("auto_start", True):
raise RuntimeError("auto_start is disabled and no base_url is configured")
logger.info("Using local container sandbox backend")
return LocalContainerBackend(
image=self._config["image"],
@@ -121,21 +118,6 @@ class AioSandboxProvider(SandboxProvider):
environment=self._config["environment"],
)
def _create_state_store(self) -> SandboxStateStore:
"""Create the state store for cross-process sandbox mapping persistence.
Currently uses file-based store. For distributed multi-host deployments,
a Redis-based store can be plugged in here.
"""
# TODO: Support RedisSandboxStateStore for distributed deployments.
# Configuration would be:
# sandbox:
# state_store: redis
# redis_url: redis://localhost:6379/0
# This would enable cross-host sandbox discovery (e.g., multiple K8s pods
# without shared PVC, or multi-node Docker Swarm).
return FileSandboxStateStore(base_dir=str(get_paths().base_dir))
# ── Configuration ────────────────────────────────────────────────────
def _load_config(self) -> dict:
@@ -143,13 +125,15 @@ class AioSandboxProvider(SandboxProvider):
config = get_app_config()
sandbox_config = config.sandbox
idle_timeout = getattr(sandbox_config, "idle_timeout", None)
replicas = getattr(sandbox_config, "replicas", None)
return {
"image": sandbox_config.image or DEFAULT_IMAGE,
"port": sandbox_config.port or DEFAULT_PORT,
"base_url": sandbox_config.base_url,
"auto_start": sandbox_config.auto_start if sandbox_config.auto_start is not None else True,
"container_prefix": sandbox_config.container_prefix or DEFAULT_CONTAINER_PREFIX,
"idle_timeout": getattr(sandbox_config, "idle_timeout", None) or DEFAULT_IDLE_TIMEOUT,
"idle_timeout": idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT,
"replicas": replicas if replicas is not None else DEFAULT_REPLICAS,
"mounts": sandbox_config.mounts or [],
"environment": self._resolve_env_vars(sandbox_config.environment or {}),
# provisioner URL for dynamic pod management (e.g. http://provisioner:8002)
@@ -201,28 +185,38 @@ class AioSandboxProvider(SandboxProvider):
"""Get volume mounts for a thread's data directories.
Creates directories if they don't exist (lazy initialization).
Mount sources use host_base_dir so that when running inside Docker with a
mounted Docker socket (DooD), the host Docker daemon can resolve the paths.
"""
paths = get_paths()
paths.ensure_thread_dirs(thread_id)
mounts = [
(str(paths.sandbox_work_dir(thread_id)), f"{VIRTUAL_PATH_PREFIX}/workspace", False),
(str(paths.sandbox_uploads_dir(thread_id)), f"{VIRTUAL_PATH_PREFIX}/uploads", False),
(str(paths.sandbox_outputs_dir(thread_id)), f"{VIRTUAL_PATH_PREFIX}/outputs", False),
]
# host_paths resolves to the host-side base dir when DEER_FLOW_HOST_BASE_DIR
# is set, otherwise falls back to the container's own base dir (native mode).
host_paths = Paths(base_dir=paths.host_base_dir)
return mounts
return [
(str(host_paths.sandbox_work_dir(thread_id)), f"{VIRTUAL_PATH_PREFIX}/workspace", False),
(str(host_paths.sandbox_uploads_dir(thread_id)), f"{VIRTUAL_PATH_PREFIX}/uploads", False),
(str(host_paths.sandbox_outputs_dir(thread_id)), f"{VIRTUAL_PATH_PREFIX}/outputs", False),
]
@staticmethod
def _get_skills_mount() -> tuple[str, str, bool] | None:
"""Get the skills directory mount configuration."""
"""Get the skills directory mount configuration.
Mount source uses DEER_FLOW_HOST_SKILLS_PATH when running inside Docker (DooD)
so the host Docker daemon can resolve the path.
"""
try:
config = get_app_config()
skills_path = config.skills.get_skills_path()
container_path = config.skills.container_path
if skills_path.exists():
return (str(skills_path), container_path, True) # Read-only for security
# When running inside Docker with DooD, use host-side skills path.
host_skills = os.environ.get("DEER_FLOW_HOST_SKILLS_PATH") or str(skills_path)
return (host_skills, container_path, True) # Read-only for security
except Exception as e:
logger.warning(f"Could not setup skills mount: {e}")
return None
@@ -249,21 +243,53 @@ class AioSandboxProvider(SandboxProvider):
def _cleanup_idle_sandboxes(self, idle_timeout: float) -> None:
current_time = time.time()
sandboxes_to_release = []
active_to_destroy = []
warm_to_destroy: list[tuple[str, SandboxInfo]] = []
with self._lock:
# Active sandboxes: tracked via _last_activity
for sandbox_id, last_activity in self._last_activity.items():
idle_duration = current_time - last_activity
if idle_duration > idle_timeout:
sandboxes_to_release.append(sandbox_id)
logger.info(f"Sandbox {sandbox_id} idle for {idle_duration:.1f}s, marking for release")
active_to_destroy.append(sandbox_id)
logger.info(f"Sandbox {sandbox_id} idle for {idle_duration:.1f}s, marking for destroy")
for sandbox_id in sandboxes_to_release:
# Warm pool: tracked via release_timestamp stored in _warm_pool
for sandbox_id, (info, release_ts) in list(self._warm_pool.items()):
warm_duration = current_time - release_ts
if warm_duration > idle_timeout:
warm_to_destroy.append((sandbox_id, info))
del self._warm_pool[sandbox_id]
logger.info(f"Warm-pool sandbox {sandbox_id} idle for {warm_duration:.1f}s, marking for destroy")
# Destroy active sandboxes (re-verify still idle before acting)
for sandbox_id in active_to_destroy:
try:
logger.info(f"Releasing idle sandbox {sandbox_id}")
self.release(sandbox_id)
# Re-verify the sandbox is still idle under the lock before destroying.
# Between the snapshot above and here, the sandbox may have been
# re-acquired (last_activity updated) or already released/destroyed.
with self._lock:
last_activity = self._last_activity.get(sandbox_id)
if last_activity is None:
# Already released or destroyed by another path — skip.
logger.info(f"Sandbox {sandbox_id} already gone before idle destroy, skipping")
continue
if (time.time() - last_activity) < idle_timeout:
# Re-acquired (activity updated) since the snapshot — skip.
logger.info(f"Sandbox {sandbox_id} was re-acquired before idle destroy, skipping")
continue
logger.info(f"Destroying idle sandbox {sandbox_id}")
self.destroy(sandbox_id)
except Exception as e:
logger.error(f"Failed to release idle sandbox {sandbox_id}: {e}")
logger.error(f"Failed to destroy idle sandbox {sandbox_id}: {e}")
# Destroy warm-pool sandboxes (already removed from _warm_pool under lock above)
for sandbox_id, info in warm_to_destroy:
try:
self._backend.destroy(info)
logger.info(f"Destroyed idle warm-pool sandbox {sandbox_id}")
except Exception as e:
logger.error(f"Failed to destroy idle warm-pool sandbox {sandbox_id}: {e}")
# ── Signal handling ──────────────────────────────────────────────────
@@ -321,11 +347,12 @@ class AioSandboxProvider(SandboxProvider):
return self._acquire_internal(thread_id)
def _acquire_internal(self, thread_id: str | None) -> str:
"""Internal sandbox acquisition with three-layer consistency.
"""Internal sandbox acquisition with two-layer consistency.
Layer 1: In-process cache (fastest, covers same-process repeated access)
Layer 2: Cross-process state store + file lock (covers multi-process)
Layer 3: Backend discovery (covers containers started by other processes)
Layer 2: Backend discovery (covers containers started by other processes;
sandbox_id is deterministic from thread_id so no shared state file
is needed — any process can derive the same container name)
"""
# ── Layer 1: In-process cache (fast path) ──
if thread_id:
@@ -342,55 +369,95 @@ class AioSandboxProvider(SandboxProvider):
# Deterministic ID for thread-specific, random for anonymous
sandbox_id = self._deterministic_sandbox_id(thread_id) if thread_id else str(uuid.uuid4())[:8]
# ── Layer 2 & 3: Cross-process recovery + creation ──
# ── Layer 1.5: Warm pool (container still running, no cold-start) ──
if thread_id:
with self._state_store.lock(thread_id):
# Try to recover from persisted state or discover existing container
recovered_id = self._try_recover(thread_id)
if recovered_id is not None:
return recovered_id
# Nothing to recover — create new sandbox (still under cross-process lock)
with self._lock:
if sandbox_id in self._warm_pool:
info, _ = self._warm_pool.pop(sandbox_id)
sandbox = AioSandbox(id=sandbox_id, base_url=info.sandbox_url)
self._sandboxes[sandbox_id] = sandbox
self._sandbox_infos[sandbox_id] = info
self._last_activity[sandbox_id] = time.time()
self._thread_sandboxes[thread_id] = sandbox_id
logger.info(f"Reclaimed warm-pool sandbox {sandbox_id} for thread {thread_id} at {info.sandbox_url}")
return sandbox_id
# ── Layer 2: Backend discovery + create (protected by cross-process lock) ──
# Use a file lock so that two processes racing to create the same sandbox
# for the same thread_id serialize here: the second process will discover
# the container started by the first instead of hitting a name-conflict.
if thread_id:
return self._discover_or_create_with_lock(thread_id, sandbox_id)
return self._create_sandbox(thread_id, sandbox_id)
def _discover_or_create_with_lock(self, thread_id: str, sandbox_id: str) -> str:
"""Discover an existing sandbox or create a new one under a cross-process file lock.
The file lock serializes concurrent sandbox creation for the same thread_id
across multiple processes, preventing container-name conflicts.
"""
paths = get_paths()
paths.ensure_thread_dirs(thread_id)
lock_path = paths.thread_dir(thread_id) / f"{sandbox_id}.lock"
with open(lock_path, "a") as lock_file:
try:
fcntl.flock(lock_file, fcntl.LOCK_EX)
# Re-check in-process caches under the file lock in case another
# thread in this process won the race while we were waiting.
with self._lock:
if thread_id in self._thread_sandboxes:
existing_id = self._thread_sandboxes[thread_id]
if existing_id in self._sandboxes:
logger.info(f"Reusing in-process sandbox {existing_id} for thread {thread_id} (post-lock check)")
self._last_activity[existing_id] = time.time()
return existing_id
if sandbox_id in self._warm_pool:
info, _ = self._warm_pool.pop(sandbox_id)
sandbox = AioSandbox(id=sandbox_id, base_url=info.sandbox_url)
self._sandboxes[sandbox_id] = sandbox
self._sandbox_infos[sandbox_id] = info
self._last_activity[sandbox_id] = time.time()
self._thread_sandboxes[thread_id] = sandbox_id
logger.info(f"Reclaimed warm-pool sandbox {sandbox_id} for thread {thread_id} (post-lock check)")
return sandbox_id
# Backend discovery: another process may have created the container.
discovered = self._backend.discover(sandbox_id)
if discovered is not None:
sandbox = AioSandbox(id=discovered.sandbox_id, base_url=discovered.sandbox_url)
with self._lock:
self._sandboxes[discovered.sandbox_id] = sandbox
self._sandbox_infos[discovered.sandbox_id] = discovered
self._last_activity[discovered.sandbox_id] = time.time()
self._thread_sandboxes[thread_id] = discovered.sandbox_id
logger.info(f"Discovered existing sandbox {discovered.sandbox_id} for thread {thread_id} at {discovered.sandbox_url}")
return discovered.sandbox_id
return self._create_sandbox(thread_id, sandbox_id)
else:
return self._create_sandbox(thread_id, sandbox_id)
finally:
fcntl.flock(lock_file, fcntl.LOCK_UN)
def _try_recover(self, thread_id: str) -> str | None:
"""Try to recover a sandbox from persisted state or backend discovery.
Called under cross-process lock for the given thread_id.
Args:
thread_id: The thread ID.
def _evict_oldest_warm(self) -> str | None:
"""Destroy the oldest container in the warm pool to free capacity.
Returns:
The sandbox_id if recovery succeeded, None otherwise.
The evicted sandbox_id, or None if warm pool is empty.
"""
info = self._state_store.load(thread_id)
if info is None:
return None
# Re-discover: verifies sandbox is alive and gets current connection info
# (handles cases like port changes after container restart)
discovered = self._backend.discover(info.sandbox_id)
if discovered is None:
logger.info(f"Persisted sandbox {info.sandbox_id} for thread {thread_id} could not be recovered")
self._state_store.remove(thread_id)
return None
# Adopt into this process's memory
sandbox = AioSandbox(id=discovered.sandbox_id, base_url=discovered.sandbox_url)
with self._lock:
self._sandboxes[discovered.sandbox_id] = sandbox
self._sandbox_infos[discovered.sandbox_id] = discovered
self._last_activity[discovered.sandbox_id] = time.time()
self._thread_sandboxes[thread_id] = discovered.sandbox_id
if not self._warm_pool:
return None
oldest_id = min(self._warm_pool, key=lambda sid: self._warm_pool[sid][1])
info, _ = self._warm_pool.pop(oldest_id)
# Update state if connection info changed
if discovered.sandbox_url != info.sandbox_url:
self._state_store.save(thread_id, discovered)
logger.info(f"Recovered sandbox {discovered.sandbox_id} for thread {thread_id} at {discovered.sandbox_url}")
return discovered.sandbox_id
try:
self._backend.destroy(info)
logger.info(f"Destroyed warm-pool sandbox {oldest_id}")
except Exception as e:
logger.error(f"Failed to destroy warm-pool sandbox {oldest_id}: {e}")
return None
return oldest_id
def _create_sandbox(self, thread_id: str | None, sandbox_id: str) -> str:
"""Create a new sandbox via the backend.
@@ -407,6 +474,21 @@ class AioSandboxProvider(SandboxProvider):
"""
extra_mounts = self._get_extra_mounts(thread_id)
# Enforce replicas: only warm-pool containers count toward eviction budget.
# Active sandboxes are in use by live threads and must not be forcibly stopped.
replicas = self._config.get("replicas", DEFAULT_REPLICAS)
with self._lock:
total = len(self._sandboxes) + len(self._warm_pool)
if total >= replicas:
evicted = self._evict_oldest_warm()
if evicted:
logger.info(f"Evicted warm-pool sandbox {evicted} to stay within replicas={replicas}")
else:
# All slots are occupied by active sandboxes — proceed anyway and log.
# The replicas limit is a soft cap; we never forcibly stop a container
# that is actively serving a thread.
logger.warning(f"All {replicas} replica slots are in active use; creating sandbox {sandbox_id} beyond the soft limit")
info = self._backend.create(thread_id, sandbox_id, extra_mounts=extra_mounts or None)
# Wait for sandbox to be ready
@@ -422,10 +504,6 @@ class AioSandboxProvider(SandboxProvider):
if thread_id:
self._thread_sandboxes[thread_id] = sandbox_id
# Persist for cross-process discovery
if thread_id:
self._state_store.save(thread_id, info)
logger.info(f"Created sandbox {sandbox_id} for thread {thread_id} at {info.sandbox_url}")
return sandbox_id
@@ -445,7 +523,11 @@ class AioSandboxProvider(SandboxProvider):
return sandbox
def release(self, sandbox_id: str) -> None:
"""Release a sandbox: clean up in-memory state, persisted state, and backend resources.
"""Release a sandbox from active use into the warm pool.
The container is kept running so it can be reclaimed quickly by the same
thread on its next turn without a cold-start. The container will only be
stopped when the replicas limit forces eviction or during shutdown.
Args:
sandbox_id: The ID of the sandbox to release.
@@ -460,15 +542,40 @@ class AioSandboxProvider(SandboxProvider):
for tid in thread_ids_to_remove:
del self._thread_sandboxes[tid]
self._last_activity.pop(sandbox_id, None)
# Park in warm pool — container keeps running
if info and sandbox_id not in self._warm_pool:
self._warm_pool[sandbox_id] = (info, time.time())
# Clean up persisted state (outside lock, involves file I/O)
for tid in thread_ids_to_remove:
self._state_store.remove(tid)
logger.info(f"Released sandbox {sandbox_id} to warm pool (container still running)")
def destroy(self, sandbox_id: str) -> None:
"""Destroy a sandbox: stop the container and free all resources.
Unlike release(), this actually stops the container. Use this for
explicit cleanup, capacity-driven eviction, or shutdown.
Args:
sandbox_id: The ID of the sandbox to destroy.
"""
info = None
thread_ids_to_remove: list[str] = []
with self._lock:
self._sandboxes.pop(sandbox_id, None)
info = self._sandbox_infos.pop(sandbox_id, None)
thread_ids_to_remove = [tid for tid, sid in self._thread_sandboxes.items() if sid == sandbox_id]
for tid in thread_ids_to_remove:
del self._thread_sandboxes[tid]
self._last_activity.pop(sandbox_id, None)
# Also pull from warm pool if it was parked there
if info is None and sandbox_id in self._warm_pool:
info, _ = self._warm_pool.pop(sandbox_id)
else:
self._warm_pool.pop(sandbox_id, None)
# Destroy backend resources (stop container, release port, etc.)
if info:
self._backend.destroy(info)
logger.info(f"Released sandbox {sandbox_id}")
logger.info(f"Destroyed sandbox {sandbox_id}")
def shutdown(self) -> None:
"""Shutdown all sandboxes. Thread-safe and idempotent."""
@@ -477,6 +584,8 @@ class AioSandboxProvider(SandboxProvider):
return
self._shutdown_called = True
sandbox_ids = list(self._sandboxes.keys())
warm_items = list(self._warm_pool.items())
self._warm_pool.clear()
# Stop idle checker
self._idle_checker_stop.set()
@@ -484,10 +593,17 @@ class AioSandboxProvider(SandboxProvider):
self._idle_checker_thread.join(timeout=5)
logger.info("Stopped idle checker thread")
logger.info(f"Shutting down {len(sandbox_ids)} sandbox(es)")
logger.info(f"Shutting down {len(sandbox_ids)} active + {len(warm_items)} warm-pool sandbox(es)")
for sandbox_id in sandbox_ids:
try:
self.release(sandbox_id)
self.destroy(sandbox_id)
except Exception as e:
logger.error(f"Failed to release sandbox {sandbox_id} during shutdown: {e}")
logger.error(f"Failed to destroy sandbox {sandbox_id} during shutdown: {e}")
for sandbox_id, (info, _) in warm_items:
try:
self._backend.destroy(info)
logger.info(f"Destroyed warm-pool sandbox {sandbox_id} during shutdown")
except Exception as e:
logger.error(f"Failed to destroy warm-pool sandbox {sandbox_id} during shutdown: {e}")