mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-07-21 10:15:47 +00:00
perf(config): index AppConfig.get_*_config by name (O(n) -> O(1)) (#3688)
get_model_config / get_tool_config / get_tool_group_config did a next(...) linear scan of self.models / self.tools / self.tool_groups on every call. These sit on hot paths: get_tool_config runs 2-3x per community-tool invocation (web_search etc.) and get_model_config several times per agent build, across 40+ call sites. Build name -> config dicts once in a mode="after" validator (stored as PrivateAttr), so each getter is an O(1) dict lookup. A config reload constructs a fresh AppConfig, which rebuilds the indexes; setdefault keeps first-match-wins on duplicate names, matching the prior next(...) semantics. Co-authored-by: ly-wang19 <ly-wang19@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
ly-wang19
Claude Opus 4.8
parent
f956682f31
commit
b66e3253a0
@@ -8,7 +8,7 @@ from typing import Any, Self
|
||||
|
||||
import yaml
|
||||
from dotenv import load_dotenv
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
|
||||
|
||||
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
|
||||
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
|
||||
@@ -163,6 +163,14 @@ class AppConfig(BaseModel):
|
||||
),
|
||||
)
|
||||
|
||||
# Name -> config lookup tables, (re)built after validation by
|
||||
# ``_build_name_indexes``. They make ``get_model_config`` / ``get_tool_config``
|
||||
# / ``get_tool_group_config`` O(1) instead of an O(n) ``next(...)`` scan per
|
||||
# call. Private attrs are excluded from serialization.
|
||||
_models_by_name: dict[str, ModelConfig] = PrivateAttr(default_factory=dict)
|
||||
_tools_by_name: dict[str, ToolConfig] = PrivateAttr(default_factory=dict)
|
||||
_tool_groups_by_name: dict[str, ToolGroupConfig] = PrivateAttr(default_factory=dict)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _drop_null_config_sections(cls, data: Any) -> Any:
|
||||
@@ -375,6 +383,31 @@ class AppConfig(BaseModel):
|
||||
return [cls.resolve_env_variables(item) for item in config]
|
||||
return config
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _build_name_indexes(self) -> "AppConfig":
|
||||
"""Build name -> config lookup tables for O(1) ``get_*_config``.
|
||||
|
||||
``get_tool_config`` runs 2-3x per community-tool invocation (e.g.
|
||||
web_search) and ``get_model_config`` several times per agent build, so
|
||||
the previous O(n) ``next(...)`` scans sat on hot paths. Rebuilt here so a
|
||||
config reload (which constructs a fresh ``AppConfig``) refreshes them.
|
||||
``setdefault`` keeps the first entry on duplicate names, preserving the
|
||||
prior ``next(...)`` first-match semantics.
|
||||
"""
|
||||
models_by_name: dict[str, ModelConfig] = {}
|
||||
for model in self.models:
|
||||
models_by_name.setdefault(model.name, model)
|
||||
tools_by_name: dict[str, ToolConfig] = {}
|
||||
for tool in self.tools:
|
||||
tools_by_name.setdefault(tool.name, tool)
|
||||
tool_groups_by_name: dict[str, ToolGroupConfig] = {}
|
||||
for group in self.tool_groups:
|
||||
tool_groups_by_name.setdefault(group.name, group)
|
||||
self._models_by_name = models_by_name
|
||||
self._tools_by_name = tools_by_name
|
||||
self._tool_groups_by_name = tool_groups_by_name
|
||||
return self
|
||||
|
||||
def get_model_config(self, name: str) -> ModelConfig | None:
|
||||
"""Get the model config by name.
|
||||
|
||||
@@ -384,7 +417,7 @@ class AppConfig(BaseModel):
|
||||
Returns:
|
||||
The model config if found, otherwise None.
|
||||
"""
|
||||
return next((model for model in self.models if model.name == name), None)
|
||||
return self._models_by_name.get(name)
|
||||
|
||||
def get_tool_config(self, name: str) -> ToolConfig | None:
|
||||
"""Get the tool config by name.
|
||||
@@ -395,7 +428,7 @@ class AppConfig(BaseModel):
|
||||
Returns:
|
||||
The tool config if found, otherwise None.
|
||||
"""
|
||||
return next((tool for tool in self.tools if tool.name == name), None)
|
||||
return self._tools_by_name.get(name)
|
||||
|
||||
def get_tool_group_config(self, name: str) -> ToolGroupConfig | None:
|
||||
"""Get the tool group config by name.
|
||||
@@ -406,7 +439,7 @@ class AppConfig(BaseModel):
|
||||
Returns:
|
||||
The tool group config if found, otherwise None.
|
||||
"""
|
||||
return next((group for group in self.tool_groups if group.name == name), None)
|
||||
return self._tool_groups_by_name.get(name)
|
||||
|
||||
|
||||
# Compatibility singleton layer for code paths that have not yet been
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Tests for AppConfig's name-indexed get_*_config lookups.
|
||||
|
||||
``get_model_config`` / ``get_tool_config`` / ``get_tool_group_config`` are
|
||||
served from name -> config dicts built once after validation, instead of an
|
||||
O(n) ``next(...)`` scan per call. These tests lock the indexed lookups to the
|
||||
exact semantics of the linear scan they replaced (match, miss -> None,
|
||||
first-match-wins on duplicate names) and confirm a config reload rebuilds them.
|
||||
"""
|
||||
|
||||
from deerflow.config.app_config import AppConfig
|
||||
|
||||
|
||||
def _build(model_names=(), tool_names=(), group_names=()):
|
||||
return AppConfig.model_validate(
|
||||
{
|
||||
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
"models": [{"name": n, "use": "pkg:Cls", "model": n} for n in model_names],
|
||||
"tools": [{"name": n, "group": "default", "use": "pkg:fn"} for n in tool_names],
|
||||
"tool_groups": [{"name": n} for n in group_names],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_get_config_returns_matching_entry():
|
||||
cfg = _build(model_names=["m1", "m2"], tool_names=["t1", "t2"], group_names=["g1"])
|
||||
assert cfg.get_model_config("m2").name == "m2"
|
||||
assert cfg.get_tool_config("t1").name == "t1"
|
||||
assert cfg.get_tool_group_config("g1").name == "g1"
|
||||
|
||||
|
||||
def test_get_config_returns_none_for_missing():
|
||||
cfg = _build(model_names=["m1"], tool_names=["t1"], group_names=["g1"])
|
||||
assert cfg.get_model_config("nope") is None
|
||||
assert cfg.get_tool_config("nope") is None
|
||||
assert cfg.get_tool_group_config("nope") is None
|
||||
|
||||
|
||||
def test_get_config_first_match_wins_on_duplicate_names():
|
||||
# Two models share a name; the index must return the FIRST, matching the
|
||||
# previous next(...) scan.
|
||||
cfg = AppConfig.model_validate(
|
||||
{
|
||||
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
"models": [
|
||||
{"name": "dup", "use": "pkg:A", "model": "first"},
|
||||
{"name": "dup", "use": "pkg:B", "model": "second"},
|
||||
],
|
||||
}
|
||||
)
|
||||
assert cfg.get_model_config("dup").model == "first"
|
||||
|
||||
|
||||
def test_index_matches_linear_scan_reference():
|
||||
cfg = _build(model_names=["a", "b", "c"], tool_names=["x", "y"], group_names=["g"])
|
||||
for n in ["a", "b", "c", "missing"]:
|
||||
assert cfg.get_model_config(n) == next((m for m in cfg.models if m.name == n), None)
|
||||
for n in ["x", "y", "missing"]:
|
||||
assert cfg.get_tool_config(n) == next((t for t in cfg.tools if t.name == n), None)
|
||||
for n in ["g", "missing"]:
|
||||
assert cfg.get_tool_group_config(n) == next((grp for grp in cfg.tool_groups if grp.name == n), None)
|
||||
|
||||
|
||||
def test_empty_config_lookups_return_none():
|
||||
cfg = _build()
|
||||
assert cfg.get_model_config("anything") is None
|
||||
assert cfg.get_tool_config("anything") is None
|
||||
assert cfg.get_tool_group_config("anything") is None
|
||||
Reference in New Issue
Block a user