style(storage): format storage package

This commit is contained in:
rayhpeng
2026-05-13 12:52:34 +08:00
parent 11a9041b65
commit 34ec205e1d
16 changed files with 47 additions and 94 deletions
@@ -1,5 +1,5 @@
from .enums import DataBaseType from .enums import DataBaseType
__all__ = [ __all__ = [
'DataBaseType', "DataBaseType",
] ]
@@ -3,7 +3,7 @@ from enum import IntEnum as SourceIntEnum
from enum import StrEnum as SourceStrEnum from enum import StrEnum as SourceStrEnum
from typing import Any, TypeVar from typing import Any, TypeVar
T = TypeVar('T', bound=Enum) T = TypeVar("T", bound=Enum)
class _EnumBase: class _EnumBase:
@@ -36,6 +36,6 @@ class StrEnum(_EnumBase, SourceStrEnum):
class DataBaseType(StrEnum): class DataBaseType(StrEnum):
"""Database type.""" """Database type."""
sqlite = 'sqlite' sqlite = "sqlite"
mysql = 'mysql' mysql = "mysql"
postgresql = 'postgresql' postgresql = "postgresql"
@@ -92,8 +92,7 @@ class AppConfig(BaseModel):
elif os.getenv("DEER_FLOW_CONFIG_PATH"): elif os.getenv("DEER_FLOW_CONFIG_PATH"):
path = Path(os.getenv("DEER_FLOW_CONFIG_PATH")) path = Path(os.getenv("DEER_FLOW_CONFIG_PATH"))
if not Path.exists(path): if not Path.exists(path):
raise FileNotFoundError( raise FileNotFoundError(f"Config file specified by environment variable `DEER_FLOW_CONFIG_PATH` not found at {path}")
f"Config file specified by environment variable `DEER_FLOW_CONFIG_PATH` not found at {path}")
return path return path
else: else:
for path in _default_config_candidates(): for path in _default_config_candidates():
@@ -159,8 +158,7 @@ class AppConfig(BaseModel):
if user_version < example_version: if user_version < example_version:
logger.warning( logger.warning(
"Your config.yaml (version %d) is outdated — the latest version is %d. Run `make config-upgrade` to " "Your config.yaml (version %d) is outdated — the latest version is %d. Run `make config-upgrade` to merge new fields into your config.",
"merge new fields into your config.",
user_version, user_version,
example_version, example_version,
) )
@@ -182,14 +180,12 @@ class AppConfig(BaseModel):
return config return config
_app_config: AppConfig | None = None _app_config: AppConfig | None = None
_app_config_path: Path | None = None _app_config_path: Path | None = None
_app_config_mtime: float | None = None _app_config_mtime: float | None = None
_app_config_is_custom = False _app_config_is_custom = False
_current_app_config: ContextVar[AppConfig | None] = ContextVar("deerflow_current_app_config", default=None) _current_app_config: ContextVar[AppConfig | None] = ContextVar("deerflow_current_app_config", default=None)
_current_app_config_stack: ContextVar[tuple[AppConfig | None, ...]] = ContextVar("deerflow_current_app_config_stack", _current_app_config_stack: ContextVar[tuple[AppConfig | None, ...]] = ContextVar("deerflow_current_app_config_stack", default=())
default=())
def _get_config_mtime(config_path: Path) -> float | None: def _get_config_mtime(config_path: Path) -> float | None:
@@ -22,17 +22,14 @@ def _strip_legacy_state_prefix(path: str) -> str:
if path == ".deer-flow": if path == ".deer-flow":
return "." return "."
if path.startswith(prefix): if path.startswith(prefix):
return path[len(prefix):] return path[len(prefix) :]
return path return path
class StorageConfig(BaseModel): class StorageConfig(BaseModel):
driver: Literal["mysql", "sqlite", "postgres", "postgresql"] = Field( driver: Literal["mysql", "sqlite", "postgres", "postgresql"] = Field(
default="sqlite", default="sqlite",
description="Storage driver for both checkpointer and application data. " description="Storage driver for both checkpointer and application data. 'sqlite' for single-node deployment (default),'postgres' for production multi-node deployment, 'mysql' for MySQL databases.",
"'sqlite' for single-node deployment (default),"
"'postgres' for production multi-node deployment, "
"'mysql' for MySQL databases.",
) )
sqlite_dir: str = Field( sqlite_dir: str = Field(
default=".deer-flow/data", default=".deer-flow/data",
@@ -23,7 +23,7 @@ id_key = Annotated[
autoincrement=True, autoincrement=True,
sort_order=-999, sort_order=-999,
comment="Primary key ID", comment="Primary key ID",
) ),
] ]
@@ -16,10 +16,7 @@ def _validate_mysql_driver(db_url: URL) -> str:
driver = url.get_driver_name() driver = url.get_driver_name()
if driver not in {"aiomysql", "asyncmy"}: if driver not in {"aiomysql", "asyncmy"}:
raise ValueError( raise ValueError(f"MySQL persistence requires async SQLAlchemy driver (aiomysql/asyncmy), got: {driver!r}")
"MySQL persistence requires async SQLAlchemy driver "
f"(aiomysql/asyncmy), got: {driver!r}"
)
return driver return driver
@@ -1,4 +1,3 @@
from __future__ import annotations from __future__ import annotations
import json import json
@@ -139,10 +139,7 @@ def _build_clause(compiler: SQLCompiler, typeof: str, extract: str, value: objec
if isinstance(value, int): if isinstance(value, int):
bp = _bind(compiler, value, BigInteger(), **kw) bp = _bind(compiler, value, BigInteger(), **kw)
if dialect.int_guard: if dialect.int_guard:
return ( return f"(CASE WHEN {_type_check(typeof, dialect.int_types)} AND {extract} ~ {dialect.int_guard} THEN CAST({extract} AS {dialect.int_cast}) END = {bp})"
f"(CASE WHEN {_type_check(typeof, dialect.int_types)} AND {extract} ~ {dialect.int_guard} "
f"THEN CAST({extract} AS {dialect.int_cast}) END = {bp})"
)
return f"({_type_check(typeof, dialect.int_types)} AND CAST({extract} AS {dialect.int_cast}) = {bp})" return f"({_type_check(typeof, dialect.int_types)} AND CAST({extract} AS {dialect.int_cast}) = {bp})"
if isinstance(value, float): if isinstance(value, float):
bp = _bind(compiler, value, Float(), **kw) bp = _bind(compiler, value, Float(), **kw)
@@ -15,6 +15,7 @@ class AppPersistence:
""" """
Unified runtime persistence bundle. Unified runtime persistence bundle.
""" """
checkpointer: Checkpointer checkpointer: Checkpointer
engine: AsyncEngine engine: AsyncEngine
session_factory: async_sessionmaker[AsyncSession] session_factory: async_sessionmaker[AsyncSession]
@@ -67,9 +67,7 @@ class DbFeedbackRepository(FeedbackRepositoryProtocol):
return _to_feedback(model) return _to_feedback(model)
async def get_feedback(self, feedback_id: str) -> Feedback | None: async def get_feedback(self, feedback_id: str) -> Feedback | None:
result = await self._session.execute( result = await self._session.execute(select(FeedbackModel).where(FeedbackModel.feedback_id == feedback_id))
select(FeedbackModel).where(FeedbackModel.feedback_id == feedback_id)
)
model = result.scalar_one_or_none() model = result.scalar_one_or_none()
return _to_feedback(model) if model else None return _to_feedback(model) if model else None
@@ -112,9 +110,7 @@ class DbFeedbackRepository(FeedbackRepositoryProtocol):
existing = await self.get_feedback(feedback_id) existing = await self.get_feedback(feedback_id)
if existing is None: if existing is None:
return False return False
await self._session.execute( await self._session.execute(delete(FeedbackModel).where(FeedbackModel.feedback_id == feedback_id))
delete(FeedbackModel).where(FeedbackModel.feedback_id == feedback_id)
)
return True return True
async def delete_feedback_by_run(self, thread_id: str, run_id: str, *, user_id: str | None = None) -> bool: async def delete_feedback_by_run(self, thread_id: str, run_id: str, *, user_id: str | None = None) -> bool:
@@ -64,9 +64,7 @@ class DbRunRepository(RunRepositoryProtocol):
return _to_run(model) return _to_run(model)
async def get_run(self, run_id: str) -> Run | None: async def get_run(self, run_id: str) -> Run | None:
result = await self._session.execute( result = await self._session.execute(select(RunModel).where(RunModel.run_id == run_id))
select(RunModel).where(RunModel.run_id == run_id)
)
model = result.scalar_one_or_none() model = result.scalar_one_or_none()
return _to_run(model) if model else None return _to_run(model) if model else None
@@ -85,15 +83,11 @@ class DbRunRepository(RunRepositoryProtocol):
result = await self._session.execute(stmt) result = await self._session.execute(stmt)
return [_to_run(m) for m in result.scalars().all()] return [_to_run(m) for m in result.scalars().all()]
async def update_run_status( async def update_run_status(self, run_id: str, status: str, *, error: str | None = None) -> None:
self, run_id: str, status: str, *, error: str | None = None
) -> None:
values: dict = {"status": status} values: dict = {"status": status}
if error is not None: if error is not None:
values["error"] = error values["error"] = error
await self._session.execute( await self._session.execute(update(RunModel).where(RunModel.run_id == run_id).values(**values))
update(RunModel).where(RunModel.run_id == run_id).values(**values)
)
async def delete_run(self, run_id: str) -> None: async def delete_run(self, run_id: str) -> None:
await self._session.execute(delete(RunModel).where(RunModel.run_id == run_id)) await self._session.execute(delete(RunModel).where(RunModel.run_id == run_id))
@@ -106,11 +100,7 @@ class DbRunRepository(RunRepositoryProtocol):
else: else:
before_dt = datetime.fromisoformat(before) before_dt = datetime.fromisoformat(before)
result = await self._session.execute( result = await self._session.execute(select(RunModel).where(RunModel.status == "pending", RunModel.created_time <= before_dt).order_by(RunModel.created_time.asc()))
select(RunModel)
.where(RunModel.status == "pending", RunModel.created_time <= before_dt)
.order_by(RunModel.created_time.asc())
)
return [_to_run(m) for m in result.scalars().all()] return [_to_run(m) for m in result.scalars().all()]
async def update_run_completion( async def update_run_completion(
@@ -147,9 +137,7 @@ class DbRunRepository(RunRepositoryProtocol):
values["last_ai_message"] = last_ai_message[:2000] values["last_ai_message"] = last_ai_message[:2000]
if error is not None: if error is not None:
values["error"] = error values["error"] = error
await self._session.execute( await self._session.execute(update(RunModel).where(RunModel.run_id == run_id).values(**values))
update(RunModel).where(RunModel.run_id == run_id).values(**values)
)
async def aggregate_tokens_by_thread(self, thread_id: str) -> dict[str, Any]: async def aggregate_tokens_by_thread(self, thread_id: str) -> dict[str, Any]:
completed = RunModel.status.in_(("success", "error")) completed = RunModel.status.in_(("success", "error"))
@@ -158,13 +158,10 @@ class DbRunEventRepository(RunEventRepositoryProtocol):
after_seq: int | None = None, after_seq: int | None = None,
user_id: str | None = None, user_id: str | None = None,
) -> list[RunEvent]: ) -> list[RunEvent]:
stmt = ( stmt = select(RunEventModel).where(
select(RunEventModel) RunEventModel.thread_id == thread_id,
.where( RunEventModel.run_id == run_id,
RunEventModel.thread_id == thread_id, RunEventModel.category == "message",
RunEventModel.run_id == run_id,
RunEventModel.category == "message",
)
) )
if user_id is not None: if user_id is not None:
stmt = stmt.where(RunEventModel.user_id == user_id) stmt = stmt.where(RunEventModel.user_id == user_id)
@@ -182,11 +179,7 @@ class DbRunEventRepository(RunEventRepositoryProtocol):
return list(reversed([_to_run_event(row) for row in result.scalars().all()])) return list(reversed([_to_run_event(row) for row in result.scalars().all()]))
async def count_messages(self, thread_id: str, *, user_id: str | None = None) -> int: async def count_messages(self, thread_id: str, *, user_id: str | None = None) -> int:
stmt = ( stmt = select(func.count()).select_from(RunEventModel).where(RunEventModel.thread_id == thread_id, RunEventModel.category == "message")
select(func.count())
.select_from(RunEventModel)
.where(RunEventModel.thread_id == thread_id, RunEventModel.category == "message")
)
if user_id is not None: if user_id is not None:
stmt = stmt.where(RunEventModel.user_id == user_id) stmt = stmt.where(RunEventModel.user_id == user_id)
count = await self._session.scalar(stmt) count = await self._session.scalar(stmt)
@@ -55,12 +55,12 @@ class DbThreadMetaRepository(ThreadMetaRepositoryProtocol):
return _to_thread_meta(model) if model else None return _to_thread_meta(model) if model else None
async def update_thread_meta( async def update_thread_meta(
self, self,
thread_id: str, thread_id: str,
*, *,
display_name: str | None = None, display_name: str | None = None,
status: str | None = None, status: str | None = None,
metadata: dict[str, Any] | None = None, metadata: dict[str, Any] | None = None,
) -> None: ) -> None:
values: dict = {} values: dict = {}
if display_name is not None: if display_name is not None:
@@ -71,21 +71,20 @@ class DbThreadMetaRepository(ThreadMetaRepositoryProtocol):
values["meta"] = dict(metadata) values["meta"] = dict(metadata)
if not values: if not values:
return return
await self._session.execute( await self._session.execute(update(ThreadMetaModel).where(ThreadMetaModel.thread_id == thread_id).values(**values))
update(ThreadMetaModel).where(ThreadMetaModel.thread_id == thread_id).values(**values))
async def delete_thread(self, thread_id: str) -> None: async def delete_thread(self, thread_id: str) -> None:
await self._session.execute(delete(ThreadMetaModel).where(ThreadMetaModel.thread_id == thread_id)) await self._session.execute(delete(ThreadMetaModel).where(ThreadMetaModel.thread_id == thread_id))
async def search_threads( async def search_threads(
self, self,
*, *,
metadata: dict[str, Any] | None = None, metadata: dict[str, Any] | None = None,
status: str | None = None, status: str | None = None,
user_id: str | None = None, user_id: str | None = None,
assistant_id: str | None = None, assistant_id: str | None = None,
limit: int = 100, limit: int = 100,
offset: int = 0, offset: int = 0,
) -> list[ThreadMeta]: ) -> list[ThreadMeta]:
stmt = select(ThreadMetaModel) stmt = select(ThreadMetaModel)
@@ -1,3 +1,3 @@
from .timezone import get_timezone from .timezone import get_timezone
__all__ = ["get_timezone"] __all__ = ["get_timezone"]
+4 -12
View File
@@ -24,12 +24,8 @@ def test_storage_json_match_compiles_sqlite() -> None:
table = _table() table = _table()
dialect = create_engine("sqlite://").dialect dialect = create_engine("sqlite://").dialect
assert str(json_match(table.c.data, "k", None).compile(dialect=dialect, compile_kwargs={"literal_binds": True})) == ( assert str(json_match(table.c.data, "k", None).compile(dialect=dialect, compile_kwargs={"literal_binds": True})) == ("json_type(t.data, '$.\"k\"') = 'null'")
"json_type(t.data, '$.\"k\"') = 'null'" assert str(json_match(table.c.data, "k", True).compile(dialect=dialect, compile_kwargs={"literal_binds": True})) == ("json_type(t.data, '$.\"k\"') = 'true'")
)
assert str(json_match(table.c.data, "k", True).compile(dialect=dialect, compile_kwargs={"literal_binds": True})) == (
"json_type(t.data, '$.\"k\"') = 'true'"
)
int_sql = str(json_match(table.c.data, "k", 42).compile(dialect=dialect, compile_kwargs={"literal_binds": True})) int_sql = str(json_match(table.c.data, "k", 42).compile(dialect=dialect, compile_kwargs={"literal_binds": True}))
assert "= 'integer'" in int_sql assert "= 'integer'" in int_sql
@@ -44,12 +40,8 @@ def test_storage_json_match_compiles_postgres() -> None:
table = _table() table = _table()
dialect = postgresql.dialect() dialect = postgresql.dialect()
assert str(json_match(table.c.data, "k", None).compile(dialect=dialect, compile_kwargs={"literal_binds": True})) == ( assert str(json_match(table.c.data, "k", None).compile(dialect=dialect, compile_kwargs={"literal_binds": True})) == ("json_typeof(t.data -> 'k') = 'null'")
"json_typeof(t.data -> 'k') = 'null'" assert str(json_match(table.c.data, "k", False).compile(dialect=dialect, compile_kwargs={"literal_binds": True})) == ("(json_typeof(t.data -> 'k') = 'boolean' AND (t.data ->> 'k') = 'false')")
)
assert str(json_match(table.c.data, "k", False).compile(dialect=dialect, compile_kwargs={"literal_binds": True})) == (
"(json_typeof(t.data -> 'k') = 'boolean' AND (t.data ->> 'k') = 'false')"
)
int_sql = str(json_match(table.c.data, "k", 42).compile(dialect=dialect, compile_kwargs={"literal_binds": True})) int_sql = str(json_match(table.c.data, "k", 42).compile(dialect=dialect, compile_kwargs={"literal_binds": True}))
assert "CASE WHEN" in int_sql assert "CASE WHEN" in int_sql
@@ -110,9 +110,7 @@ def test_storage_models_import_without_config_file(tmp_path):
[ [
sys.executable, sys.executable,
"-c", "-c",
"from store.persistence.base_model import UniversalText, id_key; " "from store.persistence.base_model import UniversalText, id_key; from store.repositories.models import RunEvent; print(UniversalText.__name__, RunEvent.__tablename__, id_key)",
"from store.repositories.models import RunEvent; "
"print(UniversalText.__name__, RunEvent.__tablename__, id_key)",
], ],
check=False, check=False,
capture_output=True, capture_output=True,