mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-07-21 02:05:45 +00:00
Ryker_FengandGitHub
41658c5ff4
* feat(skills): add skill review quality gate * fix(skills): skip review eval fixtures in CI * fix(skills): ignore review eval fixtures in bundled scans * fix(skill-review): harden review gate boundaries * fix(skills): address skill review gate feedback
106 lines
3.7 KiB
Python
106 lines
3.7 KiB
Python
"""Eval-manifest adapters for deterministic skill review facts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
from deerflow.skills.review.models import make_finding
|
|
|
|
|
|
def analyze_eval_manifests(snapshot: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
files = {str(entry["path"]): entry for entry in snapshot.get("files", [])}
|
|
eval_files = [path for path in sorted(files) if path.startswith("evals/") and path.endswith(".json")]
|
|
findings: list[dict[str, Any]] = []
|
|
aggregate = {
|
|
"schema": None,
|
|
"valid": None,
|
|
"case_count": 0,
|
|
"positive_trigger_cases": 0,
|
|
"negative_trigger_cases": 0,
|
|
"manifests": [],
|
|
}
|
|
if not eval_files:
|
|
return aggregate, findings
|
|
|
|
schemas: set[str] = set()
|
|
valid = True
|
|
for path in eval_files:
|
|
entry = files[path]
|
|
if entry.get("kind") != "text":
|
|
findings.append(
|
|
make_finding(
|
|
"eval.binary-manifest",
|
|
severity="warning",
|
|
path=path,
|
|
message="Eval manifest is not UTF-8 JSON text.",
|
|
remediation="Store eval manifests as UTF-8 JSON.",
|
|
)
|
|
)
|
|
valid = False
|
|
continue
|
|
try:
|
|
payload = json.loads(str(entry.get("content") or ""))
|
|
except json.JSONDecodeError as exc:
|
|
findings.append(
|
|
make_finding(
|
|
"eval.invalid-json",
|
|
severity="warning",
|
|
path=path,
|
|
line=exc.lineno,
|
|
message="Eval manifest is not valid JSON.",
|
|
remediation="Fix the JSON syntax or remove the manifest.",
|
|
evidence=exc.msg,
|
|
)
|
|
)
|
|
valid = False
|
|
continue
|
|
manifest = _classify_manifest(payload)
|
|
manifest["path"] = path
|
|
aggregate["manifests"].append(manifest)
|
|
schemas.add(manifest["schema"])
|
|
aggregate["case_count"] += manifest["case_count"]
|
|
aggregate["positive_trigger_cases"] += manifest["positive_trigger_cases"]
|
|
aggregate["negative_trigger_cases"] += manifest["negative_trigger_cases"]
|
|
|
|
if schemas:
|
|
aggregate["schema"] = next(iter(schemas)) if len(schemas) == 1 else "mixed"
|
|
aggregate["valid"] = valid
|
|
return aggregate, findings
|
|
|
|
|
|
def _classify_manifest(payload: Any) -> dict[str, Any]:
|
|
if isinstance(payload, dict) and isinstance(payload.get("schema_version"), str):
|
|
cases = payload.get("cases")
|
|
if isinstance(cases, list):
|
|
return _case_stats("versioned", cases)
|
|
return {"schema": "versioned", "valid": True, "case_count": 0, "positive_trigger_cases": 0, "negative_trigger_cases": 0}
|
|
|
|
if isinstance(payload, dict) and isinstance(payload.get("evals"), list):
|
|
return _case_stats("skill-creator-evals", payload["evals"])
|
|
|
|
if isinstance(payload, list):
|
|
return _case_stats("trigger-eval-list", payload)
|
|
|
|
return {"schema": "unknown", "valid": True, "case_count": 0, "positive_trigger_cases": 0, "negative_trigger_cases": 0}
|
|
|
|
|
|
def _case_stats(schema: str, cases: list[Any]) -> dict[str, Any]:
|
|
positive = 0
|
|
negative = 0
|
|
for case in cases:
|
|
if not isinstance(case, dict):
|
|
continue
|
|
should_trigger = case.get("should_trigger")
|
|
if should_trigger is True:
|
|
positive += 1
|
|
elif should_trigger is False:
|
|
negative += 1
|
|
return {
|
|
"schema": schema,
|
|
"valid": True,
|
|
"case_count": len(cases),
|
|
"positive_trigger_cases": positive,
|
|
"negative_trigger_cases": negative,
|
|
}
|