feat(persistence): add ORM models, repositories, DB/JSONL event stores, RunJournal, and API endpoints

Phase 2-B: run persistence + event storage + token tracking.

- ORM models: RunRow (with token fields), ThreadMetaRow, RunEventRow
- RunRepository implements RunStore ABC via SQLAlchemy ORM
- ThreadMetaRepository with owner access control
- DbRunEventStore with trace content truncation and cursor pagination
- JsonlRunEventStore with per-run files and seq recovery from disk
- RunJournal (BaseCallbackHandler) captures LLM/tool/lifecycle events,
  accumulates token usage by caller type, buffers and flushes to store
- RunManager now accepts optional RunStore for persistent backing
- Worker creates RunJournal, writes human_message, injects callbacks
- Gateway deps use factory functions (RunRepository when DB available)
- New endpoints: messages, run messages, run events, token-usage
- ThreadCreateRequest gains assistant_id field
- 92 tests pass (33 new), zero regressions

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
rayhpeng
2026-04-02 19:03:38 +08:00
parent 23eacf9533
commit e3179cd54d
21 changed files with 1946 additions and 29 deletions
+32 -10
View File
@@ -31,27 +31,23 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.config import get_app_config
from deerflow.persistence.engine import close_engine, init_engine_from_config
from deerflow.runtime import make_store, make_stream_bridge
from deerflow.runtime.runs.store.memory import MemoryRunStore
async with AsyncExitStack() as stack:
app.state.stream_bridge = await stack.enter_async_context(make_stream_bridge())
app.state.checkpointer = await stack.enter_async_context(make_checkpointer())
app.state.store = await stack.enter_async_context(make_store())
app.state.run_manager = RunManager()
# Initialize persistence layer from unified database config
config = get_app_config()
await init_engine_from_config(config.database)
# Initialize run store (MemoryRunStore for now; switch to ORM-backed
# RunRepository when models are implemented)
app.state.run_store = MemoryRunStore()
# Initialize run store (RunRepository if DB available, else MemoryRunStore)
app.state.run_store = _make_run_store()
# Initialize run event store (MemoryRunEventStore for now)
# TODO(Phase 2-B): switch to db/jsonl backend based on config.run_events.backend
from deerflow.runtime.events.store.memory import MemoryRunEventStore
# Initialize run event store based on config
app.state.run_event_store = _make_run_event_store(config)
app.state.run_event_store = MemoryRunEventStore()
# RunManager with store backing for persistence
app.state.run_manager = RunManager(store=app.state.run_store)
try:
yield
@@ -59,6 +55,32 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
await close_engine()
# ---------------------------------------------------------------------------
# Factories
# ---------------------------------------------------------------------------
def _make_run_store() -> RunStore:
"""Create a RunStore: RunRepository if DB engine is available, else MemoryRunStore."""
from deerflow.persistence.engine import get_session_factory
sf = get_session_factory()
if sf is not None:
from deerflow.persistence.repositories.run_repo import RunRepository
return RunRepository(sf)
from deerflow.runtime.runs.store.memory import MemoryRunStore
return MemoryRunStore()
def _make_run_event_store(config) -> RunEventStore:
from deerflow.runtime.events.store import make_run_event_store
run_events_config = getattr(config, "run_events", None)
return make_run_event_store(run_events_config)
# ---------------------------------------------------------------------------
# Getters -- called by routers per-request
# ---------------------------------------------------------------------------