fix(sandbox): close AioSandbox HTTP client during provider teardown (#2872) (#3245)

* fix(sandbox): close AioSandbox HTTP client during provider teardown (#2872)

AioSandbox allocates a host-side agent_sandbox client (wrapping an
httpx.Client) in __init__, but AioSandboxProvider.release/destroy/shutdown
only popped provider state and tore down the backend container — the
client/transport owned by each cached AioSandbox was never explicitly
closed, accumulating unreclaimed sockets in long-running services.

- Add AioSandbox.close(): best-effort, idempotent close of the wrapped
  httpx_client (falls back to top-level client.close()); errors are
  logged but never raised so backend cleanup is never blocked.
- AioSandboxProvider.release()/destroy() now close the cached AioSandbox
  before dropping it; shutdown() inherits this via destroy().

* fix(sandbox): close the real httpx.Client owned by AioSandbox (#2872)

The previous close() only walked one level (wrapper.httpx_client), which resolves to the Fern-generated HttpClient wrapper that has no close(). The real socket-owning httpx.Client lives one level deeper at _client_wrapper.httpx_client.httpx_client, so the close path never fired and host-side sockets still leaked.

Resolve the real httpx.Client with graceful degradation; clear self._client under the lock for use-after-close and concurrent double-close safety; mark provider release()/destroy() try/except as defense-in-depth; rewrite TestClose against the real nested structure to lock down the original no-op bug.

backend/tests/test_aio_sandbox.py

@@ -318,3 +318,76 @@ class TestDownloadFile:
         result = sandbox.download_file("/mnt/user-data/outputs/single.bin")
 
         assert result == b"single-chunk"
+
+
+class TestClose:
+    """Verify AioSandbox.close() tears down the host-side HTTP client (#2872)."""
+
+    def test_close_calls_real_nested_httpx_client(self, sandbox):
+        """close() must close the real httpx.Client at the bottom of the chain.
+
+        Mirrors the actual Fern structure:
+            Sandbox._client_wrapper.httpx_client  -> Fern HttpClient (no close())
+                .httpx_client                     -> httpx.Client    (the real owner)
+
+        The intermediate HttpClient deliberately exposes NO close(), so a naive
+        one-level lookup (the original bug) would silently close nothing.
+        """
+        real_httpx = MagicMock(spec=["close"])
+        fern_http = SimpleNamespace(httpx_client=real_httpx)  # no close on this layer
+        sandbox._client._client_wrapper = SimpleNamespace(httpx_client=fern_http)
+
+        sandbox.close()
+
+        real_httpx.close.assert_called_once_with()
+
+    def test_close_clears_client_reference(self, sandbox):
+        """After close(), the client reference must be dropped (use-after-close safety)."""
+        real_httpx = MagicMock(spec=["close"])
+        fern_http = SimpleNamespace(httpx_client=real_httpx)
+        sandbox._client._client_wrapper = SimpleNamespace(httpx_client=fern_http)
+
+        sandbox.close()
+
+        assert sandbox._client is None
+        assert sandbox._closed is True
+
+    def test_close_is_idempotent(self, sandbox):
+        """Calling close() multiple times must close the underlying client at most once."""
+        real_httpx = MagicMock(spec=["close"])
+        fern_http = SimpleNamespace(httpx_client=real_httpx)
+        sandbox._client._client_wrapper = SimpleNamespace(httpx_client=fern_http)
+
+        sandbox.close()
+        sandbox.close()
+        sandbox.close()
+
+        assert real_httpx.close.call_count == 1
+
+    def test_close_swallows_exceptions(self, sandbox, caplog):
+        """close() must be best-effort: client errors are logged but never raised."""
+        real_httpx = MagicMock(spec=["close"])
+        real_httpx.close.side_effect = RuntimeError("teardown boom")
+        fern_http = SimpleNamespace(httpx_client=real_httpx)
+        sandbox._client._client_wrapper = SimpleNamespace(httpx_client=fern_http)
+
+        with caplog.at_level("WARNING"):
+            sandbox.close()
+
+        assert "Error closing AioSandbox client" in caplog.text
+
+    def test_close_falls_back_to_client_close(self, sandbox):
+        """If no nested httpx.Client is reachable, close() degrades to the client's own close()."""
+        # Replace the mocked client with a stub that exposes only top-level close()
+        client = MagicMock(spec=["close"])
+        sandbox._client = client
+
+        sandbox.close()
+
+        client.close.assert_called_once_with()
+
+    def test_close_when_no_close_attr_does_not_raise(self, sandbox):
+        """A client without any close attribute must not crash close()."""
+        sandbox._client = SimpleNamespace()  # no close, no _client_wrapper
+        sandbox.close()  # must not raise
+        assert sandbox._client is None

backend/tests/test_aio_sandbox_provider.py

@@ -348,3 +348,89 @@ def test_remote_backend_create_forwards_effective_user_id(monkeypatch):
         "thread_id": "thread-42",
         "user_id": "user-7",
     }
+
+
+# ── Sandbox client teardown (#2872) ──────────────────────────────────────────
+
+
+def _make_provider_with_active_sandbox(tmp_path, sandbox_id: str):
+    """Build a provider with one active sandbox suitable for release/destroy/shutdown tests."""
+    aio_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider")
+    provider = _make_provider(tmp_path)
+    provider._lock = aio_mod.threading.Lock()
+    provider._warm_pool = {}
+    provider._sandbox_infos = {
+        sandbox_id: aio_mod.SandboxInfo(sandbox_id=sandbox_id, sandbox_url="http://sandbox-host"),
+    }
+    provider._thread_sandboxes = {}
+    provider._last_activity = {sandbox_id: 0.0}
+    provider._shutdown_called = False
+    provider._idle_checker_thread = None
+    provider._backend = SimpleNamespace(destroy=MagicMock())
+
+    sandbox = MagicMock()
+    sandbox.id = sandbox_id
+    sandbox.close = MagicMock()
+    provider._sandboxes = {sandbox_id: sandbox}
+    return provider, sandbox, aio_mod
+
+
+def test_release_closes_cached_sandbox_client(tmp_path):
+    """release() must close the host-side client owned by the cached AioSandbox (#2872)."""
+    provider, sandbox, _ = _make_provider_with_active_sandbox(tmp_path, "sandbox-rel")
+
+    provider.release("sandbox-rel")
+
+    sandbox.close.assert_called_once_with()
+    # And the sandbox is parked in the warm pool (container still running).
+    assert "sandbox-rel" in provider._warm_pool
+    assert "sandbox-rel" not in provider._sandboxes
+
+
+def test_destroy_closes_cached_sandbox_client(tmp_path):
+    """destroy() must close the host-side client before backend container teardown (#2872)."""
+    provider, sandbox, _ = _make_provider_with_active_sandbox(tmp_path, "sandbox-destroy")
+    backend_destroy = provider._backend.destroy
+
+    provider.destroy("sandbox-destroy")
+
+    sandbox.close.assert_called_once_with()
+    backend_destroy.assert_called_once()
+    assert "sandbox-destroy" not in provider._sandboxes
+    assert "sandbox-destroy" not in provider._sandbox_infos
+
+
+def test_shutdown_closes_all_active_sandbox_clients(tmp_path):
+    """shutdown() must close every cached AioSandbox client during teardown (#2872)."""
+    provider, sandbox, _ = _make_provider_with_active_sandbox(tmp_path, "sandbox-shut")
+
+    provider.shutdown()
+
+    sandbox.close.assert_called_once_with()
+    provider._backend.destroy.assert_called_once()
+    assert provider._sandboxes == {}
+
+
+def test_release_swallows_close_errors(tmp_path, caplog):
+    """A failure inside sandbox.close() must not break provider release()."""
+    provider, sandbox, _ = _make_provider_with_active_sandbox(tmp_path, "sandbox-rel-err")
+    sandbox.close.side_effect = RuntimeError("boom")
+
+    with caplog.at_level("WARNING"):
+        provider.release("sandbox-rel-err")
+
+    assert "Error closing sandbox sandbox-rel-err during release" in caplog.text
+    # Still moved to warm pool: client teardown failure must not block lifecycle.
+    assert "sandbox-rel-err" in provider._warm_pool
+
+
+def test_destroy_swallows_close_errors_and_still_destroys_backend(tmp_path, caplog):
+    """A failure in sandbox.close() must not skip backend container destruction."""
+    provider, sandbox, _ = _make_provider_with_active_sandbox(tmp_path, "sandbox-dest-err")
+    sandbox.close.side_effect = RuntimeError("boom")
+
+    with caplog.at_level("WARNING"):
+        provider.destroy("sandbox-dest-err")
+
+    assert "Error closing sandbox sandbox-dest-err during destroy" in caplog.text
+    provider._backend.destroy.assert_called_once()