Merge branch 'main' into rayhpeng/persistence-scaffold

# Conflicts:
#	config.example.yaml
This commit is contained in:
rayhpeng
2026-04-06 22:16:42 +08:00
27 changed files with 1495 additions and 64 deletions
@@ -276,6 +276,31 @@ class _DummyChannel(Channel):
class TestBaseChannelOnOutbound:
def test_default_receive_file_returns_original_message(self):
"""The base Channel.receive_file returns the original message unchanged."""
class MinimalChannel(Channel):
async def start(self):
pass
async def stop(self):
pass
async def send(self, msg):
pass
from app.channels.message_bus import InboundMessage
bus = MessageBus()
ch = MinimalChannel(name="minimal", bus=bus, config={})
msg = InboundMessage(channel_name="minimal", chat_id="c1", user_id="u1", text="hello", files=[{"file_key": "k1"}])
result = _run(ch.receive_file(msg, "thread-1"))
assert result is msg
assert result.text == "hello"
assert result.files == [{"file_key": "k1"}]
def test_send_file_called_for_each_attachment(self, tmp_path):
"""_on_outbound sends text first, then uploads each attachment."""
bus = MessageBus()
+56
View File
@@ -414,6 +414,62 @@ def _make_async_iterator(items):
class TestChannelManager:
def test_handle_chat_calls_channel_receive_file_for_inbound_files(self, monkeypatch):
from app.channels.manager import ChannelManager
async def go():
bus = MessageBus()
store = ChannelStore(path=Path(tempfile.mkdtemp()) / "store.json")
manager = ChannelManager(bus=bus, store=store)
outbound_received = []
async def capture_outbound(msg):
outbound_received.append(msg)
bus.subscribe_outbound(capture_outbound)
mock_client = _make_mock_langgraph_client()
manager._client = mock_client
modified_msg = InboundMessage(
channel_name="test",
chat_id="chat1",
user_id="user1",
text="with /mnt/user-data/uploads/demo.png",
files=[{"image_key": "img_1"}],
)
mock_channel = MagicMock()
mock_channel.receive_file = AsyncMock(return_value=modified_msg)
mock_service = MagicMock()
mock_service.get_channel.return_value = mock_channel
monkeypatch.setattr("app.channels.service.get_channel_service", lambda: mock_service)
await manager.start()
inbound = InboundMessage(
channel_name="test",
chat_id="chat1",
user_id="user1",
text="hi [image]",
files=[{"image_key": "img_1"}],
)
await bus.publish_inbound(inbound)
await _wait_for(lambda: len(outbound_received) >= 1)
await manager.stop()
mock_channel.receive_file.assert_awaited_once()
called_msg, called_thread_id = mock_channel.receive_file.await_args.args
assert called_msg.text == "hi [image]"
assert isinstance(called_thread_id, str)
assert called_thread_id
mock_client.runs.wait.assert_called_once()
run_call_args = mock_client.runs.wait.call_args
assert run_call_args[1]["input"]["messages"][0]["content"] == "with /mnt/user-data/uploads/demo.png"
_run(go())
def test_handle_chat_creates_thread(self):
from app.channels.manager import ChannelManager
+69 -2
View File
@@ -1,11 +1,20 @@
import asyncio
import json
from unittest.mock import MagicMock
from unittest.mock import AsyncMock, MagicMock
import pytest
from app.channels.commands import KNOWN_CHANNEL_COMMANDS
from app.channels.feishu import FeishuChannel
from app.channels.message_bus import MessageBus
from app.channels.message_bus import InboundMessage, MessageBus
def _run(coro):
loop = asyncio.new_event_loop()
try:
return loop.run_until_complete(coro)
finally:
loop.close()
def test_feishu_on_message_plain_text():
@@ -71,6 +80,64 @@ def test_feishu_on_message_rich_text():
assert "\n\n" in parsed_text
def test_feishu_receive_file_replaces_placeholders_in_order():
async def go():
bus = MessageBus()
channel = FeishuChannel(bus, {"app_id": "test", "app_secret": "test"})
msg = InboundMessage(
channel_name="feishu",
chat_id="chat_1",
user_id="user_1",
text="before [image] middle [file] after",
thread_ts="msg_1",
files=[{"image_key": "img_key"}, {"file_key": "file_key"}],
)
channel._receive_single_file = AsyncMock(side_effect=["/mnt/user-data/uploads/a.png", "/mnt/user-data/uploads/b.pdf"])
result = await channel.receive_file(msg, "thread_1")
assert result.text == "before /mnt/user-data/uploads/a.png middle /mnt/user-data/uploads/b.pdf after"
_run(go())
def test_feishu_on_message_extracts_image_and_file_keys():
bus = MessageBus()
channel = FeishuChannel(bus, {"app_id": "test", "app_secret": "test"})
event = MagicMock()
event.event.message.chat_id = "chat_1"
event.event.message.message_id = "msg_1"
event.event.message.root_id = None
event.event.sender.sender_id.open_id = "user_1"
# Rich text with one image and one file element.
event.event.message.content = json.dumps(
{
"content": [
[
{"tag": "text", "text": "See"},
{"tag": "img", "image_key": "img_123"},
{"tag": "file", "file_key": "file_456"},
]
]
}
)
with pytest.MonkeyPatch.context() as m:
mock_make_inbound = MagicMock()
m.setattr(channel, "_make_inbound", mock_make_inbound)
channel._on_message(event)
mock_make_inbound.assert_called_once()
files = mock_make_inbound.call_args[1]["files"]
assert files == [{"image_key": "img_123"}, {"file_key": "file_456"}]
assert "[image]" in mock_make_inbound.call_args[1]["text"]
assert "[file]" in mock_make_inbound.call_args[1]["text"]
@pytest.mark.parametrize("command", sorted(KNOWN_CHANNEL_COMMANDS))
def test_feishu_recognizes_all_known_slash_commands(command):
"""Every entry in KNOWN_CHANNEL_COMMANDS must be classified as a command."""
+48
View File
@@ -1,4 +1,5 @@
from pathlib import Path
from types import SimpleNamespace
from deerflow.agents.lead_agent.prompt import get_skills_prompt_section
from deerflow.config.agents_config import AgentConfig
@@ -41,6 +42,7 @@ def test_get_skills_prompt_section_returns_skills(monkeypatch):
result = get_skills_prompt_section(available_skills={"skill1"})
assert "skill1" in result
assert "skill2" not in result
assert "[built-in]" in result
def test_get_skills_prompt_section_returns_all_when_available_skills_is_none(monkeypatch):
@@ -52,6 +54,52 @@ def test_get_skills_prompt_section_returns_all_when_available_skills_is_none(mon
assert "skill2" in result
def test_get_skills_prompt_section_includes_self_evolution_rules(monkeypatch):
skills = [_make_skill("skill1")]
monkeypatch.setattr("deerflow.agents.lead_agent.prompt.load_skills", lambda enabled_only: skills)
monkeypatch.setattr(
"deerflow.config.get_app_config",
lambda: SimpleNamespace(
skills=SimpleNamespace(container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True),
),
)
result = get_skills_prompt_section(available_skills=None)
assert "Skill Self-Evolution" in result
def test_get_skills_prompt_section_includes_self_evolution_rules_without_skills(monkeypatch):
monkeypatch.setattr("deerflow.agents.lead_agent.prompt.load_skills", lambda enabled_only: [])
monkeypatch.setattr(
"deerflow.config.get_app_config",
lambda: SimpleNamespace(
skills=SimpleNamespace(container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True),
),
)
result = get_skills_prompt_section(available_skills=None)
assert "Skill Self-Evolution" in result
def test_get_skills_prompt_section_cache_respects_skill_evolution_toggle(monkeypatch):
skills = [_make_skill("skill1")]
monkeypatch.setattr("deerflow.agents.lead_agent.prompt.load_skills", lambda enabled_only: skills)
config = SimpleNamespace(
skills=SimpleNamespace(container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True),
)
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
enabled_result = get_skills_prompt_section(available_skills=None)
assert "Skill Self-Evolution" in enabled_result
config.skill_evolution.enabled = False
disabled_result = get_skills_prompt_section(available_skills=None)
assert "Skill Self-Evolution" not in disabled_result
def test_make_lead_agent_empty_skills_passed_correctly(monkeypatch):
from unittest.mock import MagicMock
+17
View File
@@ -0,0 +1,17 @@
from types import SimpleNamespace
import pytest
from deerflow.skills.security_scanner import scan_skill_content
@pytest.mark.anyio
async def test_scan_skill_content_blocks_when_model_unavailable(monkeypatch):
config = SimpleNamespace(skill_evolution=SimpleNamespace(moderation_model_name=None))
monkeypatch.setattr("deerflow.skills.security_scanner.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.security_scanner.create_chat_model", lambda **kwargs: (_ for _ in ()).throw(RuntimeError("boom")))
result = await scan_skill_content("---\nname: demo-skill\ndescription: demo\n---\n", executable=False)
assert result.decision == "block"
assert "manual review required" in result.reason
+163
View File
@@ -0,0 +1,163 @@
import importlib
from types import SimpleNamespace
import anyio
import pytest
skill_manage_module = importlib.import_module("deerflow.tools.skill_manage_tool")
def _skill_content(name: str, description: str = "Demo skill") -> str:
return f"---\nname: {name}\ndescription: {description}\n---\n\n# {name}\n"
async def _async_result(decision: str, reason: str):
from deerflow.skills.security_scanner import ScanResult
return ScanResult(decision=decision, reason=reason)
def test_skill_manage_create_and_patch(monkeypatch, tmp_path):
skills_root = tmp_path / "skills"
config = SimpleNamespace(
skills=SimpleNamespace(get_skills_path=lambda: skills_root, container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True, moderation_model_name=None),
)
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.manager.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.security_scanner.get_app_config", lambda: config)
monkeypatch.setattr(skill_manage_module, "clear_skills_system_prompt_cache", lambda: None)
monkeypatch.setattr(skill_manage_module, "scan_skill_content", lambda *args, **kwargs: _async_result("allow", "ok"))
runtime = SimpleNamespace(context={"thread_id": "thread-1"}, config={"configurable": {"thread_id": "thread-1"}})
result = anyio.run(
skill_manage_module.skill_manage_tool.coroutine,
runtime,
"create",
"demo-skill",
_skill_content("demo-skill"),
)
assert "Created custom skill" in result
patch_result = anyio.run(
skill_manage_module.skill_manage_tool.coroutine,
runtime,
"patch",
"demo-skill",
None,
None,
"Demo skill",
"Patched skill",
1,
)
assert "Patched custom skill" in patch_result
assert "Patched skill" in (skills_root / "custom" / "demo-skill" / "SKILL.md").read_text(encoding="utf-8")
def test_skill_manage_patch_replaces_single_occurrence_by_default(monkeypatch, tmp_path):
skills_root = tmp_path / "skills"
config = SimpleNamespace(
skills=SimpleNamespace(get_skills_path=lambda: skills_root, container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True, moderation_model_name=None),
)
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.manager.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.security_scanner.get_app_config", lambda: config)
monkeypatch.setattr(skill_manage_module, "clear_skills_system_prompt_cache", lambda: None)
monkeypatch.setattr(skill_manage_module, "scan_skill_content", lambda *args, **kwargs: _async_result("allow", "ok"))
runtime = SimpleNamespace(context={"thread_id": "thread-1"}, config={"configurable": {"thread_id": "thread-1"}})
content = _skill_content("demo-skill", "Demo skill") + "\nRepeated: Demo skill\n"
anyio.run(skill_manage_module.skill_manage_tool.coroutine, runtime, "create", "demo-skill", content)
patch_result = anyio.run(
skill_manage_module.skill_manage_tool.coroutine,
runtime,
"patch",
"demo-skill",
None,
None,
"Demo skill",
"Patched skill",
)
skill_text = (skills_root / "custom" / "demo-skill" / "SKILL.md").read_text(encoding="utf-8")
assert "1 replacement(s) applied, 2 match(es) found" in patch_result
assert skill_text.count("Patched skill") == 1
assert skill_text.count("Demo skill") == 1
def test_skill_manage_rejects_public_skill_patch(monkeypatch, tmp_path):
skills_root = tmp_path / "skills"
public_dir = skills_root / "public" / "deep-research"
public_dir.mkdir(parents=True, exist_ok=True)
(public_dir / "SKILL.md").write_text(_skill_content("deep-research"), encoding="utf-8")
config = SimpleNamespace(
skills=SimpleNamespace(get_skills_path=lambda: skills_root, container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True, moderation_model_name=None),
)
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.manager.get_app_config", lambda: config)
runtime = SimpleNamespace(context={}, config={"configurable": {}})
with pytest.raises(ValueError, match="built-in skill"):
anyio.run(
skill_manage_module.skill_manage_tool.coroutine,
runtime,
"patch",
"deep-research",
None,
None,
"Demo skill",
"Patched",
)
def test_skill_manage_sync_wrapper_supported(monkeypatch, tmp_path):
skills_root = tmp_path / "skills"
config = SimpleNamespace(
skills=SimpleNamespace(get_skills_path=lambda: skills_root, container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True, moderation_model_name=None),
)
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.manager.get_app_config", lambda: config)
monkeypatch.setattr(skill_manage_module, "clear_skills_system_prompt_cache", lambda: None)
monkeypatch.setattr(skill_manage_module, "scan_skill_content", lambda *args, **kwargs: _async_result("allow", "ok"))
runtime = SimpleNamespace(context={"thread_id": "thread-sync"}, config={"configurable": {"thread_id": "thread-sync"}})
result = skill_manage_module.skill_manage_tool.func(
runtime=runtime,
action="create",
name="sync-skill",
content=_skill_content("sync-skill"),
)
assert "Created custom skill" in result
def test_skill_manage_rejects_support_path_traversal(monkeypatch, tmp_path):
skills_root = tmp_path / "skills"
config = SimpleNamespace(
skills=SimpleNamespace(get_skills_path=lambda: skills_root, container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True, moderation_model_name=None),
)
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.manager.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.security_scanner.get_app_config", lambda: config)
monkeypatch.setattr(skill_manage_module, "clear_skills_system_prompt_cache", lambda: None)
monkeypatch.setattr(skill_manage_module, "scan_skill_content", lambda *args, **kwargs: _async_result("allow", "ok"))
runtime = SimpleNamespace(context={"thread_id": "thread-1"}, config={"configurable": {"thread_id": "thread-1"}})
anyio.run(skill_manage_module.skill_manage_tool.coroutine, runtime, "create", "demo-skill", _skill_content("demo-skill"))
with pytest.raises(ValueError, match="parent-directory traversal|selected support directory"):
anyio.run(
skill_manage_module.skill_manage_tool.coroutine,
runtime,
"write_file",
"demo-skill",
"malicious overwrite",
"references/../SKILL.md",
)
+132
View File
@@ -0,0 +1,132 @@
import json
from types import SimpleNamespace
from fastapi import FastAPI
from fastapi.testclient import TestClient
from app.gateway.routers import skills as skills_router
from deerflow.skills.manager import get_skill_history_file
def _skill_content(name: str, description: str = "Demo skill") -> str:
return f"---\nname: {name}\ndescription: {description}\n---\n\n# {name}\n"
async def _async_scan(decision: str, reason: str):
from deerflow.skills.security_scanner import ScanResult
return ScanResult(decision=decision, reason=reason)
def test_custom_skills_router_lifecycle(monkeypatch, tmp_path):
skills_root = tmp_path / "skills"
custom_dir = skills_root / "custom" / "demo-skill"
custom_dir.mkdir(parents=True, exist_ok=True)
(custom_dir / "SKILL.md").write_text(_skill_content("demo-skill"), encoding="utf-8")
config = SimpleNamespace(
skills=SimpleNamespace(get_skills_path=lambda: skills_root, container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True, moderation_model_name=None),
)
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.manager.get_app_config", lambda: config)
monkeypatch.setattr("app.gateway.routers.skills.scan_skill_content", lambda *args, **kwargs: _async_scan("allow", "ok"))
monkeypatch.setattr("app.gateway.routers.skills.clear_skills_system_prompt_cache", lambda: None)
app = FastAPI()
app.include_router(skills_router.router)
with TestClient(app) as client:
response = client.get("/api/skills/custom")
assert response.status_code == 200
assert response.json()["skills"][0]["name"] == "demo-skill"
get_response = client.get("/api/skills/custom/demo-skill")
assert get_response.status_code == 200
assert "# demo-skill" in get_response.json()["content"]
update_response = client.put(
"/api/skills/custom/demo-skill",
json={"content": _skill_content("demo-skill", "Edited skill")},
)
assert update_response.status_code == 200
assert update_response.json()["description"] == "Edited skill"
history_response = client.get("/api/skills/custom/demo-skill/history")
assert history_response.status_code == 200
assert history_response.json()["history"][-1]["action"] == "human_edit"
rollback_response = client.post("/api/skills/custom/demo-skill/rollback", json={"history_index": -1})
assert rollback_response.status_code == 200
assert rollback_response.json()["description"] == "Demo skill"
def test_custom_skill_rollback_blocked_by_scanner(monkeypatch, tmp_path):
skills_root = tmp_path / "skills"
custom_dir = skills_root / "custom" / "demo-skill"
custom_dir.mkdir(parents=True, exist_ok=True)
original_content = _skill_content("demo-skill")
edited_content = _skill_content("demo-skill", "Edited skill")
(custom_dir / "SKILL.md").write_text(edited_content, encoding="utf-8")
config = SimpleNamespace(
skills=SimpleNamespace(get_skills_path=lambda: skills_root, container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True, moderation_model_name=None),
)
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.manager.get_app_config", lambda: config)
get_skill_history_file("demo-skill").write_text(
'{"action":"human_edit","prev_content":' + json.dumps(original_content) + ',"new_content":' + json.dumps(edited_content) + "}\n",
encoding="utf-8",
)
monkeypatch.setattr("app.gateway.routers.skills.clear_skills_system_prompt_cache", lambda: None)
async def _scan(*args, **kwargs):
from deerflow.skills.security_scanner import ScanResult
return ScanResult(decision="block", reason="unsafe rollback")
monkeypatch.setattr("app.gateway.routers.skills.scan_skill_content", _scan)
app = FastAPI()
app.include_router(skills_router.router)
with TestClient(app) as client:
rollback_response = client.post("/api/skills/custom/demo-skill/rollback", json={"history_index": -1})
assert rollback_response.status_code == 400
assert "unsafe rollback" in rollback_response.json()["detail"]
history_response = client.get("/api/skills/custom/demo-skill/history")
assert history_response.status_code == 200
assert history_response.json()["history"][-1]["scanner"]["decision"] == "block"
def test_custom_skill_delete_preserves_history_and_allows_restore(monkeypatch, tmp_path):
skills_root = tmp_path / "skills"
custom_dir = skills_root / "custom" / "demo-skill"
custom_dir.mkdir(parents=True, exist_ok=True)
original_content = _skill_content("demo-skill")
(custom_dir / "SKILL.md").write_text(original_content, encoding="utf-8")
config = SimpleNamespace(
skills=SimpleNamespace(get_skills_path=lambda: skills_root, container_path="/mnt/skills"),
skill_evolution=SimpleNamespace(enabled=True, moderation_model_name=None),
)
monkeypatch.setattr("deerflow.config.get_app_config", lambda: config)
monkeypatch.setattr("deerflow.skills.manager.get_app_config", lambda: config)
monkeypatch.setattr("app.gateway.routers.skills.scan_skill_content", lambda *args, **kwargs: _async_scan("allow", "ok"))
monkeypatch.setattr("app.gateway.routers.skills.clear_skills_system_prompt_cache", lambda: None)
app = FastAPI()
app.include_router(skills_router.router)
with TestClient(app) as client:
delete_response = client.delete("/api/skills/custom/demo-skill")
assert delete_response.status_code == 200
assert not (custom_dir / "SKILL.md").exists()
history_response = client.get("/api/skills/custom/demo-skill/history")
assert history_response.status_code == 200
assert history_response.json()["history"][-1]["action"] == "human_delete"
rollback_response = client.post("/api/skills/custom/demo-skill/rollback", json={"history_index": -1})
assert rollback_response.status_code == 200
assert rollback_response.json()["description"] == "Demo skill"
assert (custom_dir / "SKILL.md").read_text(encoding="utf-8") == original_content
+12
View File
@@ -62,3 +62,15 @@ def test_load_skills_skips_hidden_directories(tmp_path: Path):
assert "ok-skill" in names
assert "secret-skill" not in names
def test_load_skills_prefers_custom_over_public_with_same_name(tmp_path: Path):
skills_root = tmp_path / "skills"
_write_skill(skills_root / "public" / "shared-skill", "shared-skill", "Public version")
_write_skill(skills_root / "custom" / "shared-skill", "shared-skill", "Custom version")
skills = load_skills(skills_path=skills_root, use_config=False, enabled_only=False)
shared = next(skill for skill in skills if skill.name == "shared-skill")
assert shared.category == "custom"
assert shared.description == "Custom version"