Support local IM channel connections

This commit is contained in:
taohe
2026-06-10 21:59:33 +08:00
parent 9effa7be6d
commit 92c185b90d
16 changed files with 381 additions and 53 deletions
+3 -1
View File
@@ -402,7 +402,9 @@ Bridges external messaging platforms (Feishu, Slack, Telegram, Discord, DingTalk
- Per-channel configs: `feishu` (app_id, app_secret), `slack` (bot_token, app_token), `telegram` (bot_token), `dingtalk` (client_id, client_secret, optional `card_template_id` for AI Card streaming)
**User-owned channel connections** (`config.yaml` -> `channel_connections`):
- Disabled by default. When enabled, `public_base_url` and `encryption_key` are required.
- Disabled by default. `mode: local` is the default for local/private deployments; `mode: public` requires `public_base_url`.
- `public_base_url` is only required for provider-to-server public callbacks/webhooks such as Slack HTTP Events and Telegram webhooks. If it is omitted, OAuth redirect URLs fall back to the current request origin for localhost/private testing.
- `encryption_key` is required before storing provider OAuth credentials for Slack and Discord. Telegram deep-link binding can run without it because it stores only connection identity/state, not per-user provider tokens.
- Frontend APIs: `GET /api/channels/providers`, `GET /api/channels/connections`, `POST /api/channels/{provider}/connect`, and `DELETE /api/channels/connections/{connection_id}`.
- Public provider routes: Slack/Discord OAuth callbacks and Slack/Telegram webhooks are explicitly allowed through AuthMiddleware; webhooks are exempt from CSRF because they are provider-to-server calls and validate Slack signatures or Telegram secret tokens.
- Slack HTTP Events mode uses per-connection encrypted bot tokens for replies. Legacy Slack Socket Mode remains available through the `channels.slack` config.
+2 -3
View File
@@ -85,8 +85,6 @@ def _make_connection_repo(app_config: AppConfig):
if connection_config is None or not getattr(connection_config, "enabled", False):
return None
encryption_key = getattr(connection_config, "encryption_key", "")
if not encryption_key:
return None
try:
from deerflow.persistence.channel_connections import ChannelConnectionRepository, ChannelCredentialCipher
@@ -99,7 +97,8 @@ def _make_connection_repo(app_config: AppConfig):
if session_factory is None:
logger.warning("Channel connections are enabled but database persistence is not available")
return None
return ChannelConnectionRepository(session_factory, cipher=ChannelCredentialCipher.from_key(encryption_key))
cipher = ChannelCredentialCipher.from_key(encryption_key) if encryption_key else None
return ChannelConnectionRepository(session_factory, cipher=cipher)
class ChannelService:
@@ -29,6 +29,8 @@ class ChannelProviderResponse(BaseModel):
display_name: str
enabled: bool
configured: bool
connectable: bool
unavailable_reason: str | None = None
auth_mode: str
connection_status: str
@@ -93,10 +95,9 @@ def _get_repository(request: Request, config: ChannelConnectionsConfig) -> Chann
sf = get_session_factory()
if sf is None:
raise HTTPException(status_code=503, detail="Channel connection persistence is not available")
if not config.encryption_key:
raise HTTPException(status_code=503, detail="Channel connection encryption key is not configured")
repo = ChannelConnectionRepository(sf, cipher=ChannelCredentialCipher.from_key(config.encryption_key))
cipher = ChannelCredentialCipher.from_key(config.encryption_key) if config.encryption_key else None
repo = ChannelConnectionRepository(sf, cipher=cipher)
request.app.state.channel_connection_repo = repo
return repo
@@ -108,6 +109,43 @@ def _provider_config(config: ChannelConnectionsConfig, provider: str):
return provider_config
def _provider_unavailable_reason(config: ChannelConnectionsConfig, provider: str) -> str | None:
provider_config = _provider_config(config, provider)
if not provider_config.enabled or not provider_config.configured:
return None
if provider == "telegram" and getattr(provider_config, "delivery", "polling") == "webhook":
if not provider_config.webhook_secret:
return "Telegram webhook delivery requires channel_connections.telegram.webhook_secret"
if not config.public_base_url:
return "Telegram webhook delivery requires channel_connections.public_base_url; use polling for local/private deployments"
if provider == "slack" and getattr(provider_config, "event_delivery", "http") == "http" and not config.public_base_url:
return "Slack HTTP Events require channel_connections.public_base_url; use a public URL/tunnel or Slack Socket Mode for private deployments"
if provider in {"slack", "discord"} and not config.encryption_key:
display_name = _PROVIDER_META[provider]["display_name"]
return f"{display_name} connections require channel_connections.encryption_key to store OAuth credentials"
return None
def _require_provider_connectable(config: ChannelConnectionsConfig, provider: str) -> None:
reason = _provider_unavailable_reason(config, provider)
if reason:
raise HTTPException(status_code=400, detail=reason)
def _callback_base_url(config: ChannelConnectionsConfig, request: Request) -> str:
if config.public_base_url:
return config.public_base_url.rstrip("/")
return str(request.base_url).rstrip("/")
def _callback_redirect_uri(config: ChannelConnectionsConfig, request: Request, provider: str) -> str:
return f"{_callback_base_url(config, request)}/api/channels/{provider}/callback"
async def _create_state(
repo: ChannelConnectionRepository,
*,
@@ -128,12 +166,12 @@ async def _create_state(
return state
def _build_connect_url(config: ChannelConnectionsConfig, provider: str, state: str) -> str:
def _build_connect_url(config: ChannelConnectionsConfig, request: Request, provider: str, state: str) -> str:
provider_config = _provider_config(config, provider)
if provider == "telegram":
return f"https://t.me/{provider_config.bot_username}?start={state}"
redirect_uri = f"{config.public_base_url.rstrip('/')}/api/channels/{provider}/callback"
redirect_uri = _callback_redirect_uri(config, request, provider)
if provider == "slack":
query = urlencode(
{
@@ -242,7 +280,13 @@ async def _publish_slack_event(
@router.get("/providers", response_model=ChannelProvidersResponse)
async def get_channel_providers(request: Request) -> ChannelProvidersResponse:
config = _get_channel_connections_config(request)
repo = _get_repository(request, config) if config.enabled and config.encryption_key else None
repo = None
if config.enabled:
try:
repo = _get_repository(request, config)
except HTTPException as exc:
if exc.status_code != 503:
raise
owner_user_id = _get_user_id(request)
connections = await repo.list_connections(owner_user_id) if repo is not None else []
by_provider = {item["provider"]: item for item in connections}
@@ -251,12 +295,15 @@ async def get_channel_providers(request: Request) -> ChannelProvidersResponse:
for provider, meta in _PROVIDER_META.items():
status = config.provider_status(provider)
connection = by_provider.get(provider)
unavailable_reason = _provider_unavailable_reason(config, provider)
providers.append(
ChannelProviderResponse(
provider=provider,
display_name=meta["display_name"],
enabled=status["enabled"],
configured=status["configured"],
connectable=status["enabled"] and status["configured"] and unavailable_reason is None,
unavailable_reason=unavailable_reason,
auth_mode=meta["auth_mode"],
connection_status=connection["status"] if connection else "not_connected",
)
@@ -307,7 +354,7 @@ async def slack_oauth_callback(request: Request, code: str | None = None, state:
if state_data is None:
raise HTTPException(status_code=400, detail="Invalid or expired OAuth state")
redirect_uri = f"{config.public_base_url.rstrip('/')}/api/channels/slack/callback"
redirect_uri = _callback_redirect_uri(config, request, "slack")
install = await slack_connect.exchange_slack_oauth_code(
client_id=provider_config.client_id,
client_secret=provider_config.client_secret,
@@ -351,7 +398,7 @@ async def discord_oauth_callback(request: Request, code: str | None = None, stat
if state_data is None:
raise HTTPException(status_code=400, detail="Invalid or expired OAuth state")
redirect_uri = f"{config.public_base_url.rstrip('/')}/api/channels/discord/callback"
redirect_uri = _callback_redirect_uri(config, request, "discord")
identity = await discord_connect.complete_discord_oauth(
client_id=provider_config.client_id,
client_secret=provider_config.client_secret,
@@ -471,6 +518,7 @@ async def connect_channel_provider(provider: str, request: Request) -> ChannelCo
provider_config = _provider_config(config, provider)
if not provider_config.enabled or not provider_config.configured:
raise HTTPException(status_code=400, detail="Channel provider is not configured")
_require_provider_connectable(config, provider)
repo = _get_repository(request, config)
state = await _create_state(
@@ -482,6 +530,6 @@ async def connect_channel_provider(provider: str, request: Request) -> ChannelCo
return ChannelConnectResponse(
provider=provider,
mode=_PROVIDER_META[provider]["auth_mode"],
url=_build_connect_url(config, provider, state),
url=_build_connect_url(config, request, provider, state),
expires_in=_STATE_TTL_SECONDS,
)
+26 -5
View File
@@ -6,9 +6,27 @@ DeerFlow supports user-owned IM channel connections for Telegram, Slack, and Dis
Enable the top-level `channel_connections` block in `config.yaml`:
Local/private deployment:
```yaml
channel_connections:
enabled: true
mode: local
telegram:
enabled: true
bot_token: $TELEGRAM_BOT_TOKEN
bot_username: $TELEGRAM_BOT_USERNAME
```
This mode is intended for a DeerFlow instance running on a developer machine or a private network. Telegram uses the existing long-polling worker, so it does not need a public URL. The frontend `Connect` button returns a Telegram deep link and stores a one-time state locally so the `/start` message can bind the Telegram chat to the current DeerFlow user.
Public deployment:
```yaml
channel_connections:
enabled: true
mode: public
public_base_url: https://deerflow.example.com
encryption_key: $DEER_FLOW_CHANNEL_CONNECTIONS_KEY
@@ -33,7 +51,9 @@ channel_connections:
permissions: "274877975552"
```
`public_base_url` must be the externally reachable HTTPS origin used by provider callbacks and webhooks. `encryption_key` encrypts provider tokens at rest with Fernet. Keep it stable; v1 does not support transparent key rotation, so changing it requires users to reconnect.
`public_base_url` is only required for public callback/webhook deployments. If it is omitted, OAuth redirect URLs are built from the current request origin, which is suitable for localhost development when the provider allows an exact localhost redirect URI. Provider-to-server webhooks such as Slack HTTP Events and Telegram webhooks still need a reachable public URL or a tunnel.
`encryption_key` encrypts provider tokens at rest with Fernet. Telegram deep-link binding does not store user provider tokens, so it can run locally without this key. Slack and Discord connections store OAuth credentials and require a stable key; v1 does not support transparent key rotation, so changing it requires users to reconnect.
## Frontend Flow
@@ -44,10 +64,10 @@ The workspace sidebar shows a Channels group with Telegram, Slack, and Discord.
Telegram:
- Register a bot with BotFather.
- Configure the bot username, bot token, and a random webhook secret.
- Configure the bot username and bot token.
- Users connect with a deep link: `https://t.me/<bot_username>?start=<state>`.
- Production webhook path: `POST /api/channels/webhooks/telegram`, protected by `X-Telegram-Bot-Api-Secret-Token`.
- Local/self-hosted long polling still works through the existing Telegram channel worker.
- Local/private delivery uses the existing long-polling channel worker and does not require `public_base_url`.
- Production webhook path: `POST /api/channels/webhooks/telegram`, protected by `X-Telegram-Bot-Api-Secret-Token`; webhook delivery requires `webhook_secret` and a public `public_base_url`.
Slack:
@@ -57,11 +77,12 @@ Slack:
- Required signing secret: Slack's request signing secret, not the deprecated verification token.
- Suggested MVP bot scopes: `app_mentions:read`, `chat:write`, `channels:history`, `channels:read`.
- Slack events are signature-verified, deduplicated by `event_id`, and then routed to a matching user connection.
- In local/private mode, Slack HTTP Events are reported as unavailable unless `public_base_url` is set to a tunnel or public HTTPS URL.
Discord:
- Create a Discord application and bot.
- Redirect URL: `https://<public_base_url>/api/channels/discord/callback`.
- Redirect URL: `https://<public_base_url>/api/channels/discord/callback` in public mode, or the matching localhost callback URL in local development if the Discord application is configured to allow it.
- DeerFlow starts OAuth with `identify guilds bot applications.commands` and the configured bot permissions.
- The Discord Gateway is still handled by `discord.py`; message content may require the privileged Message Content Intent depending on your bot setup.
@@ -2,8 +2,13 @@
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel, Field, model_validator
ChannelConnectionMode = Literal["local", "private", "public"]
TelegramDeliveryMode = Literal["polling", "webhook"]
class SlackChannelConnectionConfig(BaseModel):
enabled: bool = False
@@ -29,13 +34,16 @@ class TelegramChannelConnectionConfig(BaseModel):
enabled: bool = False
bot_token: str = ""
bot_username: str = ""
delivery: TelegramDeliveryMode = "polling"
webhook_secret: str = ""
oidc_client_id: str = ""
oidc_client_secret: str = ""
@property
def configured(self) -> bool:
return bool(self.bot_token and self.bot_username and self.webhook_secret)
if self.delivery == "webhook":
return bool(self.bot_token and self.bot_username and self.webhook_secret)
return bool(self.bot_token and self.bot_username)
class DiscordChannelConnectionConfig(BaseModel):
@@ -55,6 +63,7 @@ class ChannelConnectionsConfig(BaseModel):
"""Top-level config for browser-connectable IM channels."""
enabled: bool = False
mode: ChannelConnectionMode = "local"
public_base_url: str = ""
encryption_key: str = ""
slack: SlackChannelConnectionConfig = Field(default_factory=SlackChannelConnectionConfig)
@@ -64,10 +73,8 @@ class ChannelConnectionsConfig(BaseModel):
@model_validator(mode="after")
def _require_shared_config_when_enabled(self) -> ChannelConnectionsConfig:
missing: list[str] = []
if self.enabled and not self.public_base_url:
missing.append("public_base_url is required when channel_connections.enabled is true")
if self.enabled and not self.encryption_key:
missing.append("encryption_key is required when channel_connections.enabled is true")
if self.enabled and self.mode == "public" and not self.public_base_url:
missing.append("public_base_url is required when channel_connections.mode is public")
if missing:
raise ValueError("; ".join(missing))
return self
@@ -53,7 +53,7 @@ class ChannelConnectionRepository:
self,
session_factory: async_sessionmaker[AsyncSession],
*,
cipher: ChannelCredentialCipher,
cipher: ChannelCredentialCipher | None = None,
) -> None:
self.session_factory = session_factory
self._cipher = cipher
@@ -77,6 +77,13 @@ class ChannelConnectionRepository:
return value
return value.replace(tzinfo=UTC)
def _encrypt_optional_secret(self, value: str | None) -> str | None:
if value is None:
return None
if self._cipher is None:
raise RuntimeError("channel connection encryption key is required")
return self._cipher.encrypt_text(value)
@staticmethod
def _connection_to_dict(row: ChannelConnectionRow) -> dict[str, Any]:
data = row.to_dict()
@@ -166,6 +173,8 @@ class ChannelConnectionRepository:
refresh_expires_at: datetime | None = None,
extra: dict[str, Any] | None = None,
) -> None:
if self._cipher is None:
raise RuntimeError("channel connection encryption key is required")
async with self.session_factory() as session:
row = await session.get(ChannelCredentialRow, connection_id)
if row is None:
@@ -181,6 +190,8 @@ class ChannelConnectionRepository:
await session.commit()
async def get_credentials(self, connection_id: str) -> dict[str, Any] | None:
if self._cipher is None:
return None
async with self.session_factory() as session:
row = await session.get(ChannelCredentialRow, connection_id)
if row is None:
@@ -217,7 +228,7 @@ class ChannelConnectionRepository:
state_hash=self.hash_state(state),
owner_user_id=owner_user_id,
provider=provider,
code_verifier_encrypted=self._cipher.encrypt_text(code_verifier),
code_verifier_encrypted=self._encrypt_optional_secret(code_verifier),
nonce_hash=nonce_hash,
redirect_after=redirect_after,
requested_scopes_json=list(requested_scopes or []),
@@ -17,13 +17,35 @@ def test_channel_connections_disabled_by_default():
assert config.discord.enabled is False
def test_enabled_channel_connections_require_public_url_and_encryption_key():
with pytest.raises(ValidationError) as excinfo:
ChannelConnectionsConfig(enabled=True)
def test_enabled_channel_connections_can_run_in_local_mode_without_public_url_or_encryption_key():
config = ChannelConnectionsConfig.model_validate(
{
"enabled": True,
"mode": "local",
"telegram": {
"enabled": True,
"bot_token": "telegram-token",
"bot_username": "deerflow_bot",
},
}
)
message = str(excinfo.value)
assert "public_base_url is required" in message
assert "encryption_key is required" in message
assert config.public_base_url == ""
assert config.encryption_key == ""
assert config.provider_status("telegram") == {"enabled": True, "configured": True}
def test_public_mode_requires_public_url():
with pytest.raises(ValidationError) as excinfo:
ChannelConnectionsConfig.model_validate(
{
"enabled": True,
"mode": "public",
"encryption_key": "test-secret",
}
)
assert "public_base_url is required when channel_connections.mode is public" in str(excinfo.value)
def test_provider_config_completeness_is_reported_without_crashing():
@@ -25,15 +25,13 @@ def _user() -> User:
)
async def _make_repo(tmp_path):
async def _make_repo(tmp_path, encryption_key: str | None = "router-secret"):
from deerflow.persistence.channel_connections import ChannelConnectionRepository, ChannelCredentialCipher
from deerflow.persistence.engine import get_session_factory, init_engine
await init_engine("sqlite", url=f"sqlite+aiosqlite:///{tmp_path / 'router.db'}", sqlite_dir=str(tmp_path))
return ChannelConnectionRepository(
get_session_factory(),
cipher=ChannelCredentialCipher.from_key("router-secret"),
)
cipher = ChannelCredentialCipher.from_key(encryption_key) if encryption_key else None
return ChannelConnectionRepository(get_session_factory(), cipher=cipher)
def _make_app(config: ChannelConnectionsConfig, repo):
@@ -164,6 +162,74 @@ def test_connect_telegram_returns_deep_link_and_persists_state(tmp_path):
anyio.run(repo.close)
def test_connect_telegram_local_mode_without_public_url_or_encryption_key(tmp_path):
import anyio
repo = anyio.run(_make_repo, tmp_path, None)
app = _make_app(
ChannelConnectionsConfig.model_validate(
{
"enabled": True,
"mode": "local",
"telegram": {
"enabled": True,
"bot_token": "telegram-token",
"bot_username": "deerflow_bot",
},
}
),
repo,
)
with TestClient(app) as client:
response = client.post("/api/channels/telegram/connect")
assert response.status_code == 200
body = response.json()
assert body["provider"] == "telegram"
assert body["url"].startswith("https://t.me/deerflow_bot?start=")
async def count_states():
return await repo.count_oauth_states(owner_user_id=str(_user().id), provider="telegram")
assert anyio.run(count_states) == 1
anyio.run(repo.close)
def test_get_providers_reports_slack_http_unavailable_without_public_url(tmp_path):
import anyio
repo = anyio.run(_make_repo, tmp_path)
config = ChannelConnectionsConfig.model_validate(
{
"enabled": True,
"mode": "local",
"encryption_key": "router-secret",
"slack": {
"enabled": True,
"client_id": "slack-client",
"client_secret": "slack-secret",
"signing_secret": "slack-signing-secret",
"event_delivery": "http",
},
}
)
app = _make_app(config, repo)
with TestClient(app) as client:
response = client.get("/api/channels/providers")
assert response.status_code == 200
slack = next(item for item in response.json()["providers"] if item["provider"] == "slack")
assert slack["enabled"] is True
assert slack["configured"] is True
assert slack["connectable"] is False
assert "public_base_url" in slack["unavailable_reason"]
anyio.run(repo.close)
def test_connect_unconfigured_provider_returns_400(tmp_path):
import anyio
@@ -189,6 +255,99 @@ def test_connect_unconfigured_provider_returns_400(tmp_path):
anyio.run(repo.close)
def test_connect_slack_http_without_public_url_returns_400(tmp_path):
import anyio
repo = anyio.run(_make_repo, tmp_path)
app = _make_app(
ChannelConnectionsConfig.model_validate(
{
"enabled": True,
"mode": "local",
"encryption_key": "router-secret",
"slack": {
"enabled": True,
"client_id": "slack-client",
"client_secret": "slack-secret",
"signing_secret": "slack-signing-secret",
"event_delivery": "http",
},
}
),
repo,
)
with TestClient(app) as client:
response = client.post("/api/channels/slack/connect")
assert response.status_code == 400
assert "public_base_url" in response.json()["detail"]
anyio.run(repo.close)
def test_connect_discord_uses_request_base_url_without_public_base_url(tmp_path):
import anyio
repo = anyio.run(_make_repo, tmp_path)
app = _make_app(
ChannelConnectionsConfig.model_validate(
{
"enabled": True,
"mode": "local",
"encryption_key": "router-secret",
"discord": {
"enabled": True,
"client_id": "discord-client",
"client_secret": "discord-secret",
"bot_token": "discord-bot",
"permissions": "274877975552",
},
}
),
repo,
)
with TestClient(app, base_url="http://localhost:2026") as client:
response = client.post("/api/channels/discord/connect")
assert response.status_code == 200
parsed = urlparse(response.json()["url"])
query = parse_qs(parsed.query)
assert query["redirect_uri"] == ["http://localhost:2026/api/channels/discord/callback"]
anyio.run(repo.close)
def test_connect_discord_without_encryption_key_returns_400(tmp_path):
import anyio
repo = anyio.run(_make_repo, tmp_path, None)
app = _make_app(
ChannelConnectionsConfig.model_validate(
{
"enabled": True,
"mode": "local",
"discord": {
"enabled": True,
"client_id": "discord-client",
"client_secret": "discord-secret",
"bot_token": "discord-bot",
},
}
),
repo,
)
with TestClient(app) as client:
response = client.post("/api/channels/discord/connect")
assert response.status_code == 400
assert "encryption_key" in response.json()["detail"]
anyio.run(repo.close)
def test_connect_discord_includes_bot_install_scope_and_permissions(tmp_path):
import anyio