Add user-owned IM channel connections

This commit is contained in:
taohe
2026-06-10 21:07:44 +08:00
parent 0fb18e368c
commit dbe3a3bb0d
47 changed files with 4009 additions and 47 deletions
@@ -11,6 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
from deerflow.config.channel_connections_config import ChannelConnectionsConfig
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
@@ -116,6 +117,7 @@ class AppConfig(BaseModel):
subagents: SubagentsAppConfig = Field(default_factory=SubagentsAppConfig, description="Subagent runtime configuration")
guardrails: GuardrailsConfig = Field(default_factory=GuardrailsConfig, description="Guardrail middleware configuration")
circuit_breaker: CircuitBreakerConfig = Field(default_factory=CircuitBreakerConfig, description="LLM circuit breaker configuration")
channel_connections: ChannelConnectionsConfig = Field(default_factory=ChannelConnectionsConfig, description="User-facing IM channel connection configuration")
loop_detection: LoopDetectionConfig = Field(default_factory=LoopDetectionConfig, description="Loop detection middleware configuration")
safety_finish_reason: SafetyFinishReasonConfig = Field(default_factory=SafetyFinishReasonConfig, description="Provider safety-filter finish_reason interception middleware configuration")
model_config = ConfigDict(extra="allow")
@@ -0,0 +1,82 @@
"""Configuration for user-owned IM channel connections."""
from __future__ import annotations
from pydantic import BaseModel, Field, model_validator
class SlackChannelConnectionConfig(BaseModel):
enabled: bool = False
client_id: str = ""
client_secret: str = ""
signing_secret: str = ""
scopes: list[str] = Field(
default_factory=lambda: [
"app_mentions:read",
"chat:write",
"channels:history",
"channels:read",
]
)
event_delivery: str = "http"
@property
def configured(self) -> bool:
return bool(self.client_id and self.client_secret and self.signing_secret)
class TelegramChannelConnectionConfig(BaseModel):
enabled: bool = False
bot_token: str = ""
bot_username: str = ""
webhook_secret: str = ""
oidc_client_id: str = ""
oidc_client_secret: str = ""
@property
def configured(self) -> bool:
return bool(self.bot_token and self.bot_username and self.webhook_secret)
class DiscordChannelConnectionConfig(BaseModel):
enabled: bool = False
client_id: str = ""
client_secret: str = ""
bot_token: str = ""
permissions: str = ""
require_message_content_intent: bool = True
@property
def configured(self) -> bool:
return bool(self.client_id and self.client_secret and self.bot_token)
class ChannelConnectionsConfig(BaseModel):
"""Top-level config for browser-connectable IM channels."""
enabled: bool = False
public_base_url: str = ""
encryption_key: str = ""
slack: SlackChannelConnectionConfig = Field(default_factory=SlackChannelConnectionConfig)
telegram: TelegramChannelConnectionConfig = Field(default_factory=TelegramChannelConnectionConfig)
discord: DiscordChannelConnectionConfig = Field(default_factory=DiscordChannelConnectionConfig)
@model_validator(mode="after")
def _require_shared_config_when_enabled(self) -> ChannelConnectionsConfig:
missing: list[str] = []
if self.enabled and not self.public_base_url:
missing.append("public_base_url is required when channel_connections.enabled is true")
if self.enabled and not self.encryption_key:
missing.append("encryption_key is required when channel_connections.enabled is true")
if missing:
raise ValueError("; ".join(missing))
return self
def provider_status(self, provider: str) -> dict[str, bool]:
config = getattr(self, provider, None)
if config is None:
return {"enabled": False, "configured": False}
return {
"enabled": bool(config.enabled),
"configured": bool(config.configured),
}
@@ -0,0 +1,23 @@
"""User-owned IM channel connection persistence."""
from deerflow.persistence.channel_connections.model import (
ChannelConnectionRow,
ChannelConversationRow,
ChannelCredentialRow,
ChannelOAuthStateRow,
ChannelWebhookDeliveryRow,
)
from deerflow.persistence.channel_connections.sql import (
ChannelConnectionRepository,
ChannelCredentialCipher,
)
__all__ = [
"ChannelConnectionRepository",
"ChannelConnectionRow",
"ChannelConversationRow",
"ChannelCredentialCipher",
"ChannelCredentialRow",
"ChannelOAuthStateRow",
"ChannelWebhookDeliveryRow",
]
@@ -0,0 +1,121 @@
"""ORM models for user-owned IM channel connections."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import JSON, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
def _utc_now() -> datetime:
return datetime.now(UTC)
class ChannelConnectionRow(Base):
__tablename__ = "channel_connections"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
provider: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="connected")
external_account_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
external_account_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
workspace_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
workspace_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
bot_user_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
scopes_json: Mapped[list] = mapped_column(JSON, default=list)
capabilities_json: Mapped[dict] = mapped_column(JSON, default=dict)
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now, onupdate=_utc_now)
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
__table_args__ = (
UniqueConstraint(
"owner_user_id",
"provider",
"external_account_id",
"workspace_id",
name="uq_channel_connection_owner_provider_identity",
),
Index("idx_channel_connections_event_lookup", "provider", "workspace_id", "bot_user_id"),
)
class ChannelCredentialRow(Base):
__tablename__ = "channel_credentials"
connection_id: Mapped[str] = mapped_column(
String(64),
ForeignKey("channel_connections.id", ondelete="CASCADE"),
primary_key=True,
)
encrypted_access_token: Mapped[str | None] = mapped_column(Text, nullable=True)
encrypted_refresh_token: Mapped[str | None] = mapped_column(Text, nullable=True)
token_type: Mapped[str | None] = mapped_column(String(32), nullable=True)
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
refresh_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
encrypted_extra_json: Mapped[str | None] = mapped_column(Text, nullable=True)
version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now, onupdate=_utc_now)
class ChannelOAuthStateRow(Base):
__tablename__ = "channel_oauth_states"
state_hash: Mapped[str] = mapped_column(String(128), primary_key=True)
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
provider: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
code_verifier_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
nonce_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
redirect_after: Mapped[str | None] = mapped_column(Text, nullable=True)
requested_scopes_json: Mapped[list] = mapped_column(JSON, default=list)
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
consumed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now)
class ChannelConversationRow(Base):
__tablename__ = "channel_conversations"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
connection_id: Mapped[str] = mapped_column(
String(64),
ForeignKey("channel_connections.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
provider: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
external_conversation_id: Mapped[str] = mapped_column(String(128), nullable=False)
external_topic_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now, onupdate=_utc_now)
__table_args__ = (
UniqueConstraint(
"connection_id",
"external_conversation_id",
"external_topic_id",
name="uq_channel_conversation_connection_external",
),
)
class ChannelWebhookDeliveryRow(Base):
__tablename__ = "channel_webhook_deliveries"
provider: Mapped[str] = mapped_column(String(32), primary_key=True)
delivery_id: Mapped[str] = mapped_column(String(128), primary_key=True)
payload_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
event_type: Mapped[str | None] = mapped_column(String(64), nullable=True)
processed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now)
@@ -0,0 +1,363 @@
"""SQL repository for user-owned IM channel connections."""
from __future__ import annotations
import base64
import hashlib
import json
import uuid
from datetime import UTC, datetime
from typing import Any
from cryptography.fernet import Fernet
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.channel_connections.model import (
ChannelConnectionRow,
ChannelConversationRow,
ChannelCredentialRow,
ChannelOAuthStateRow,
ChannelWebhookDeliveryRow,
)
from deerflow.utils.time import coerce_iso
class ChannelCredentialCipher:
"""Encrypts provider credentials before they are persisted."""
def __init__(self, fernet: Fernet) -> None:
self._fernet = fernet
@classmethod
def from_key(cls, key: str) -> ChannelCredentialCipher:
digest = hashlib.sha256(key.encode("utf-8")).digest()
return cls(Fernet(base64.urlsafe_b64encode(digest)))
def encrypt_text(self, value: str | None) -> str | None:
if value is None:
return None
return "fernet:v1:" + self._fernet.encrypt(value.encode("utf-8")).decode("ascii")
def decrypt_text(self, value: str | None) -> str | None:
if value is None:
return None
token = value.removeprefix("fernet:v1:")
return self._fernet.decrypt(token.encode("ascii")).decode("utf-8")
class ChannelConnectionRepository:
"""Persistence facade for channel connections, credentials, and conversations."""
def __init__(
self,
session_factory: async_sessionmaker[AsyncSession],
*,
cipher: ChannelCredentialCipher,
) -> None:
self.session_factory = session_factory
self._cipher = cipher
async def close(self) -> None:
from deerflow.persistence.engine import close_engine
await close_engine()
@staticmethod
def _new_id() -> str:
return uuid.uuid4().hex
@staticmethod
def _normalize_optional_identity(value: str | None) -> str:
return value or ""
@staticmethod
def _coerce_datetime(value: datetime | None) -> datetime | None:
if value is None or value.tzinfo is not None:
return value
return value.replace(tzinfo=UTC)
@staticmethod
def _connection_to_dict(row: ChannelConnectionRow) -> dict[str, Any]:
data = row.to_dict()
data["external_account_id"] = data["external_account_id"] or None
data["workspace_id"] = data["workspace_id"] or None
data["scopes"] = data.pop("scopes_json") or []
data["capabilities"] = data.pop("capabilities_json") or {}
data["metadata"] = data.pop("metadata_json") or {}
for key in ("created_at", "updated_at", "last_seen_at", "last_error_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = coerce_iso(value)
return data
async def upsert_connection(
self,
*,
owner_user_id: str,
provider: str,
external_account_id: str | None = None,
external_account_name: str | None = None,
workspace_id: str | None = None,
workspace_name: str | None = None,
bot_user_id: str | None = None,
scopes: list[str] | None = None,
capabilities: dict[str, Any] | None = None,
metadata: dict[str, Any] | None = None,
status: str = "connected",
) -> dict[str, Any]:
external_account_id_value = self._normalize_optional_identity(external_account_id)
workspace_id_value = self._normalize_optional_identity(workspace_id)
async with self.session_factory() as session:
stmt = select(ChannelConnectionRow).where(
ChannelConnectionRow.owner_user_id == owner_user_id,
ChannelConnectionRow.provider == provider,
ChannelConnectionRow.external_account_id == external_account_id_value,
ChannelConnectionRow.workspace_id == workspace_id_value,
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = ChannelConnectionRow(
id=self._new_id(),
owner_user_id=owner_user_id,
provider=provider,
external_account_id=external_account_id_value,
workspace_id=workspace_id_value,
)
session.add(row)
row.status = status
row.external_account_name = external_account_name
row.workspace_name = workspace_name
row.bot_user_id = bot_user_id
row.scopes_json = list(scopes or [])
row.capabilities_json = dict(capabilities or {})
row.metadata_json = dict(metadata or {})
await session.commit()
await session.refresh(row)
return self._connection_to_dict(row)
async def list_connections(self, owner_user_id: str) -> list[dict[str, Any]]:
async with self.session_factory() as session:
result = await session.execute(select(ChannelConnectionRow).where(ChannelConnectionRow.owner_user_id == owner_user_id).order_by(ChannelConnectionRow.updated_at.desc(), ChannelConnectionRow.id.desc()))
return [self._connection_to_dict(row) for row in result.scalars()]
async def disconnect_connection(self, *, connection_id: str, owner_user_id: str) -> bool:
async with self.session_factory() as session:
row = await session.get(ChannelConnectionRow, connection_id)
if row is None or row.owner_user_id != owner_user_id:
return False
row.status = "revoked"
credential = await session.get(ChannelCredentialRow, connection_id)
if credential is not None:
await session.delete(credential)
await session.commit()
return True
async def store_credentials(
self,
connection_id: str,
*,
access_token: str | None,
refresh_token: str | None = None,
token_type: str | None = None,
expires_at: datetime | None = None,
refresh_expires_at: datetime | None = None,
extra: dict[str, Any] | None = None,
) -> None:
async with self.session_factory() as session:
row = await session.get(ChannelCredentialRow, connection_id)
if row is None:
row = ChannelCredentialRow(connection_id=connection_id)
session.add(row)
row.encrypted_access_token = self._cipher.encrypt_text(access_token)
row.encrypted_refresh_token = self._cipher.encrypt_text(refresh_token)
row.token_type = token_type
row.expires_at = expires_at
row.refresh_expires_at = refresh_expires_at
row.encrypted_extra_json = self._cipher.encrypt_text(json.dumps(extra or {}, ensure_ascii=False))
row.version = (row.version or 0) + 1
await session.commit()
async def get_credentials(self, connection_id: str) -> dict[str, Any] | None:
async with self.session_factory() as session:
row = await session.get(ChannelCredentialRow, connection_id)
if row is None:
return None
extra_raw = self._cipher.decrypt_text(row.encrypted_extra_json)
return {
"connection_id": row.connection_id,
"access_token": self._cipher.decrypt_text(row.encrypted_access_token),
"refresh_token": self._cipher.decrypt_text(row.encrypted_refresh_token),
"token_type": row.token_type,
"expires_at": self._coerce_datetime(row.expires_at),
"refresh_expires_at": self._coerce_datetime(row.refresh_expires_at),
"extra": json.loads(extra_raw) if extra_raw else {},
}
@staticmethod
def hash_state(state: str) -> str:
return hashlib.sha256(state.encode("utf-8")).hexdigest()
async def create_oauth_state(
self,
*,
owner_user_id: str,
provider: str,
state: str,
expires_at: datetime,
code_verifier: str | None = None,
nonce_hash: str | None = None,
redirect_after: str | None = None,
requested_scopes: list[str] | None = None,
metadata: dict[str, Any] | None = None,
) -> None:
row = ChannelOAuthStateRow(
state_hash=self.hash_state(state),
owner_user_id=owner_user_id,
provider=provider,
code_verifier_encrypted=self._cipher.encrypt_text(code_verifier),
nonce_hash=nonce_hash,
redirect_after=redirect_after,
requested_scopes_json=list(requested_scopes or []),
metadata_json=dict(metadata or {}),
expires_at=expires_at,
)
async with self.session_factory() as session:
session.add(row)
await session.commit()
async def count_oauth_states(self, *, owner_user_id: str, provider: str) -> int:
async with self.session_factory() as session:
result = await session.execute(
select(ChannelOAuthStateRow).where(
ChannelOAuthStateRow.owner_user_id == owner_user_id,
ChannelOAuthStateRow.provider == provider,
)
)
return len(list(result.scalars()))
async def consume_oauth_state(
self,
*,
provider: str,
state: str,
now: datetime | None = None,
) -> dict[str, Any] | None:
current_time = now or datetime.now(UTC)
async with self.session_factory() as session:
row = await session.get(ChannelOAuthStateRow, self.hash_state(state))
if row is None or row.provider != provider or row.consumed_at is not None:
return None
expires_at = self._coerce_datetime(row.expires_at)
if expires_at is not None and expires_at < current_time:
return None
row.consumed_at = current_time
await session.commit()
return {
"owner_user_id": row.owner_user_id,
"provider": row.provider,
"requested_scopes": row.requested_scopes_json or [],
"metadata": row.metadata_json or {},
"redirect_after": row.redirect_after,
}
async def find_connection_by_external_identity(
self,
*,
provider: str,
external_account_id: str,
workspace_id: str | None = None,
) -> dict[str, Any] | None:
async with self.session_factory() as session:
result = await session.execute(
select(ChannelConnectionRow)
.where(
ChannelConnectionRow.provider == provider,
ChannelConnectionRow.external_account_id == self._normalize_optional_identity(external_account_id),
ChannelConnectionRow.workspace_id == self._normalize_optional_identity(workspace_id),
ChannelConnectionRow.status == "connected",
)
.order_by(ChannelConnectionRow.updated_at.desc(), ChannelConnectionRow.id.desc())
.limit(1)
)
row = result.scalar_one_or_none()
return self._connection_to_dict(row) if row is not None else None
async def set_thread_id(
self,
*,
connection_id: str,
owner_user_id: str,
provider: str,
external_conversation_id: str,
thread_id: str,
external_topic_id: str | None = None,
) -> None:
topic_id = external_topic_id or ""
async with self.session_factory() as session:
stmt = select(ChannelConversationRow).where(
ChannelConversationRow.connection_id == connection_id,
ChannelConversationRow.external_conversation_id == external_conversation_id,
ChannelConversationRow.external_topic_id == topic_id,
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = ChannelConversationRow(
id=self._new_id(),
connection_id=connection_id,
owner_user_id=owner_user_id,
provider=provider,
external_conversation_id=external_conversation_id,
external_topic_id=topic_id,
thread_id=thread_id,
)
session.add(row)
else:
row.thread_id = thread_id
row.owner_user_id = owner_user_id
row.provider = provider
await session.commit()
async def get_thread_id(
self,
connection_id: str,
external_conversation_id: str,
external_topic_id: str | None = None,
) -> str | None:
async with self.session_factory() as session:
stmt = select(ChannelConversationRow.thread_id).where(
ChannelConversationRow.connection_id == connection_id,
ChannelConversationRow.external_conversation_id == external_conversation_id,
ChannelConversationRow.external_topic_id == (external_topic_id or ""),
)
return (await session.execute(stmt)).scalar_one_or_none()
async def record_webhook_delivery(
self,
*,
provider: str,
delivery_id: str,
payload_sha256: str,
event_type: str | None = None,
) -> bool:
async with self.session_factory() as session:
existing = await session.get(
ChannelWebhookDeliveryRow,
{"provider": provider, "delivery_id": delivery_id},
)
if existing is not None:
return False
session.add(
ChannelWebhookDeliveryRow(
provider=provider,
delivery_id=delivery_id,
payload_sha256=payload_sha256,
event_type=event_type,
)
)
await session.commit()
return True
@@ -14,10 +14,28 @@ its storage implementation lives in ``deerflow.runtime.events.store.db`` and
there is no matching entity directory.
"""
from deerflow.persistence.channel_connections.model import (
ChannelConnectionRow,
ChannelConversationRow,
ChannelCredentialRow,
ChannelOAuthStateRow,
ChannelWebhookDeliveryRow,
)
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.user.model import UserRow
__all__ = ["FeedbackRow", "RunEventRow", "RunRow", "ThreadMetaRow", "UserRow"]
__all__ = [
"ChannelConnectionRow",
"ChannelConversationRow",
"ChannelCredentialRow",
"ChannelOAuthStateRow",
"ChannelWebhookDeliveryRow",
"FeedbackRow",
"RunEventRow",
"RunRow",
"ThreadMetaRow",
"UserRow",
]
+1
View File
@@ -36,6 +36,7 @@ dependencies = [
"sqlalchemy[asyncio]>=2.0,<3.0",
"aiosqlite>=0.19",
"alembic>=1.13",
"cryptography>=43.0.0",
]
[project.optional-dependencies]