mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-06-13 19:06:01 +00:00
fix(skills): keep skill archive installation off the event loop (#3505)
* fix(skills): keep skill archive installation off the event loop ainstall_skill_from_archive — the async entry point awaited by the gateway POST /skills/install route — ran its entire filesystem pipeline inline on the event loop: zip extraction, frontmatter validation, rglob enumeration, per-file read_text, shutil.copytree staging, and tempdir cleanup. Restructure into offloaded phases: prepare (extract + validate) and commit (stage + move) run via asyncio.to_thread, the tempdir lifecycle is offloaded, and the security scanner's file enumeration and reads move off the loop — only the per-file LLM scan (genuinely async) stays awaited. Security decision logic and exception contract are unchanged. Anchor: tests/blocking_io/test_skills_install.py drives the real install pipeline (real .skill archive, real FS; only scan_skill_content stubbed) under the strict Blockbuster gate. Verified red on pre-fix code (BlockingError: os.stat), green with the fix. * fix(skills): log temp-dir cleanup failures instead of swallowing them Review follow-up on the install offload: rmtree(ignore_errors=True) kept the primary install exception but silently leaked the extraction dir on cleanup failure. Keep the never-mask behaviour, add a warning log. * fix(skills): bound install tmp cleanup and pass skill_dir explicitly (review) - Wrap the best-effort temp-dir cleanup in asyncio.wait_for (5s) so a hung filesystem in the finally block cannot stall or mask the install outcome; timeout is logged like the existing OSError path. - Hoist _collect_scannable_files to module level with skill_dir as an explicit argument instead of a closure capture.
This commit is contained in:
@@ -153,7 +153,7 @@ async def _scan_skill_file_or_raise(skill_dir: Path, path: Path, skill_name: str
|
||||
rel_path = path.relative_to(skill_dir).as_posix()
|
||||
location = f"{skill_name}/{rel_path}"
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8")
|
||||
content = await asyncio.to_thread(path.read_text, encoding="utf-8")
|
||||
except UnicodeDecodeError as e:
|
||||
raise SkillSecurityScanError(f"Security scan failed for skill '{skill_name}': {location} must be valid UTF-8") from e
|
||||
|
||||
@@ -174,15 +174,17 @@ async def _scan_skill_file_or_raise(skill_dir: Path, path: Path, skill_name: str
|
||||
raise SkillSecurityScanError(f"Security scan failed for {location}: invalid scanner decision {decision!r}")
|
||||
|
||||
|
||||
def _collect_scannable_files(skill_dir: Path) -> list[Path]:
|
||||
"""Enumerate archive files for scanning (blocking; run off the event loop)."""
|
||||
return [candidate for candidate in sorted(skill_dir.rglob("*")) if candidate.is_file()]
|
||||
|
||||
|
||||
async def _scan_skill_archive_contents_or_raise(skill_dir: Path, skill_name: str) -> None:
|
||||
"""Run the skill security scanner against all installable text and script files."""
|
||||
skill_md = skill_dir / "SKILL.md"
|
||||
await _scan_skill_file_or_raise(skill_dir, skill_md, skill_name, executable=False)
|
||||
|
||||
for path in sorted(skill_dir.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
|
||||
for path in await asyncio.to_thread(_collect_scannable_files, skill_dir):
|
||||
rel_path = path.relative_to(skill_dir)
|
||||
if rel_path == Path("SKILL.md"):
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user