mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-06-11 09:55:59 +00:00
Compare commits
52 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b8323024c9 | |||
| d1768606c0 | |||
| ddd1c5e42f | |||
| a270e8b310 | |||
| f330ddce01 | |||
| b26b30ac3d | |||
| dae7c7870e | |||
| 4f56437030 | |||
| 42fd0cc22f | |||
| a4202028d9 | |||
| 4a0278420f | |||
| ade4a55cfe | |||
| 09872af36c | |||
| 92f562920d | |||
| 9d51e38641 | |||
| c966eb71a7 | |||
| c4368c9018 | |||
| f83767bb17 | |||
| 0e939bfe23 | |||
| 89da9b70db | |||
| a52deada8b | |||
| b7097baaec | |||
| 87200ff920 | |||
| 2d5f0787de | |||
| 5819bd8a59 | |||
| b3c2cc42cf | |||
| 167ef4512f | |||
| ba9cc5e972 | |||
| 6a94b58ad1 | |||
| d06643d8a2 | |||
| 92c185b90d | |||
| 9effa7be6d | |||
| 582bfda6f8 | |||
| 05ae4467ae | |||
| b66152c514 | |||
| 78fbc0abdb | |||
| ec5ed185cd | |||
| dbe3a3bb0d | |||
| 2b795265e7 | |||
| a57d05fe0a | |||
| ae9e8bc0bf | |||
| 16391e35ab | |||
| 18bbb82f07 | |||
| b62c5a7b5b | |||
| 5b81588b87 | |||
| 63ce88f874 | |||
| 37337b77f9 | |||
| 8db16bb3d8 | |||
| 93e3281cbf | |||
| 0fb18e368c | |||
| 90e23bfd09 | |||
| f92a26d56f |
@@ -21,6 +21,7 @@ INFOQUEST_API_KEY=your-infoquest-api-key
|
||||
# DEEPSEEK_API_KEY=your-deepseek-api-key
|
||||
# NOVITA_API_KEY=your-novita-api-key # OpenAI-compatible, see https://novita.ai
|
||||
# MINIMAX_API_KEY=your-minimax-api-key # OpenAI-compatible, see https://platform.minimax.io
|
||||
# STEPFUN_API_KEY=your-stepfun-api-key # OpenAI-compatible, see https://platform.stepfun.com
|
||||
# VLLM_API_KEY=your-vllm-api-key # OpenAI-compatible
|
||||
# FEISHU_APP_ID=your-feishu-app-id
|
||||
# FEISHU_APP_SECRET=your-feishu-app-secret
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
# Path-based PR auto-labeling config for actions/labeler@v5.
|
||||
# Each key is a label (must exist — see .github/labels.yml); the globs decide
|
||||
# when it is applied. A PR can match several areas, which is expected.
|
||||
|
||||
"area:frontend":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- "frontend/**"
|
||||
|
||||
"area:backend":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- "backend/app/**"
|
||||
- "backend/packages/harness/deerflow/runtime/**"
|
||||
- "backend/packages/harness/deerflow/persistence/**"
|
||||
- "backend/packages/harness/deerflow/config/**"
|
||||
- "backend/packages/harness/deerflow/tools/**"
|
||||
- "backend/packages/harness/deerflow/guardrails/**"
|
||||
- "backend/packages/harness/deerflow/tracing/**"
|
||||
- "backend/packages/harness/deerflow/models/**"
|
||||
- "backend/packages/harness/deerflow/utils/**"
|
||||
- "backend/packages/harness/deerflow/uploads/**"
|
||||
|
||||
"area:agents":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- "backend/packages/harness/deerflow/agents/**"
|
||||
- "backend/packages/harness/deerflow/subagents/**"
|
||||
- "backend/packages/harness/deerflow/reflection/**"
|
||||
- "backend/langgraph.json"
|
||||
- "backend/**/prompts/**"
|
||||
|
||||
"area:sandbox":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- "docker/**"
|
||||
- "backend/packages/harness/deerflow/sandbox/**"
|
||||
- "backend/Dockerfile"
|
||||
- "frontend/Dockerfile"
|
||||
|
||||
"area:skills":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- "skills/**"
|
||||
- "backend/packages/harness/deerflow/skills/**"
|
||||
- "frontend/src/core/skills/**"
|
||||
|
||||
"area:mcp":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- "backend/packages/harness/deerflow/mcp/**"
|
||||
- "frontend/src/core/mcp/**"
|
||||
|
||||
"area:ci":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- ".github/**"
|
||||
- "scripts/**"
|
||||
|
||||
"area:docs":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- "docs/**"
|
||||
- "**/*.md"
|
||||
|
||||
"area:deps":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- "backend/pyproject.toml"
|
||||
- "backend/uv.lock"
|
||||
- "frontend/package.json"
|
||||
- "frontend/pnpm-lock.yaml"
|
||||
@@ -1,44 +0,0 @@
|
||||
name: Issue Triage
|
||||
|
||||
# Ensures every newly opened issue carries `needs-triage`, even blank or
|
||||
# API-created ones that bypass the issue templates. Creates the label if it is
|
||||
# somehow missing, so the workflow is self-healing.
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [opened]
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
needs-triage:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Add needs-triage label
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const { owner, repo } = context.repo;
|
||||
const issue_number = context.payload.issue.number;
|
||||
|
||||
const current = (context.payload.issue.labels || []).map(l => l.name);
|
||||
if (current.includes('needs-triage')) {
|
||||
core.info('Issue already has needs-triage; nothing to do.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Self-heal: create the label if it does not exist yet.
|
||||
try {
|
||||
await github.rest.issues.createLabel({
|
||||
owner, repo, name: 'needs-triage', color: 'fef2c0',
|
||||
description: 'Awaiting maintainer triage',
|
||||
});
|
||||
} catch (e) {
|
||||
if (e.status !== 422) throw e; // 422 = already exists
|
||||
}
|
||||
|
||||
await github.rest.issues.addLabels({
|
||||
owner, repo, issue_number, labels: ['needs-triage'],
|
||||
});
|
||||
core.info(`Added needs-triage to #${issue_number}.`);
|
||||
@@ -10,7 +10,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
lint-backend:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
name: PR Labeler
|
||||
|
||||
# Applies area:* labels based on which files a PR changes (see .github/labeler.yml).
|
||||
# Uses pull_request_target so it also works on fork PRs. SAFE: actions/labeler
|
||||
# only reads the changed-file list via the API — it never checks out or runs PR code.
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: pr-labeler-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
label:
|
||||
if: github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Apply area labels
|
||||
uses: actions/labeler@v5
|
||||
with:
|
||||
configuration-path: .github/labeler.yml
|
||||
sync-labels: true
|
||||
@@ -1,164 +0,0 @@
|
||||
name: PR Triage
|
||||
|
||||
# Two responsibilities, both pure-metadata (no PR code is checked out or run):
|
||||
# 1. On open/sync: apply size/* + risk:* labels, and needs-validation when the
|
||||
# PR touches the front/back contract surface (backend API, SSE, agents, or
|
||||
# the frontend streaming client). A `skip-validation` label opts out.
|
||||
# 2. On maintainer review: apply the `reviewing` label.
|
||||
#
|
||||
# All labels are managed within their own namespace — labels outside size/*,
|
||||
# risk:*, needs-validation and reviewing are never touched here.
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened, ready_for_review]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: pr-triage-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
size-and-risk:
|
||||
if: github.event_name == 'pull_request_target' && github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Label size, risk and validation need
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const pr = context.payload.pull_request;
|
||||
const { owner, repo } = context.repo;
|
||||
const prNumber = pr.number;
|
||||
|
||||
// ---- size, from additions + deletions ----
|
||||
const churn = (pr.additions || 0) + (pr.deletions || 0);
|
||||
const sizeLabel =
|
||||
churn < 20 ? 'size/XS' :
|
||||
churn < 100 ? 'size/S' :
|
||||
churn < 300 ? 'size/M' :
|
||||
churn < 700 ? 'size/L' : 'size/XL';
|
||||
|
||||
// ---- changed paths ----
|
||||
const files = await github.paginate(github.rest.pulls.listFiles, {
|
||||
owner, repo, pull_number: prNumber, per_page: 100,
|
||||
});
|
||||
const paths = files.map(f => f.filename);
|
||||
|
||||
const matches = (re) => paths.some(p => re.test(p));
|
||||
|
||||
const docsOnly = paths.length > 0 && paths.every(p =>
|
||||
/\.(md|mdx|txt)$/i.test(p) || p.startsWith('docs/') ||
|
||||
/\.(png|jpe?g|gif|svg|webp|ico)$/i.test(p));
|
||||
|
||||
const highRisk = matches(
|
||||
/^backend\/app\/gateway\//) || matches(
|
||||
/^backend\/packages\/harness\/deerflow\/(agents|subagents|sandbox)\//) || matches(
|
||||
/(^|\/)langgraph\.json$/) || matches(
|
||||
/(^|\/)(auth|authz|security)/i) || matches(
|
||||
/(pyproject\.toml|uv\.lock|package\.json|pnpm-lock\.yaml)$/) || matches(
|
||||
/^docker\//) || matches(
|
||||
/^\.github\/workflows\//);
|
||||
|
||||
const riskLabel = docsOnly ? 'risk:low' : (highRisk ? 'risk:high' : 'risk:medium');
|
||||
|
||||
// needs-validation: front/back contract surface
|
||||
const contractSurface =
|
||||
matches(/^backend\/app\/gateway\//) ||
|
||||
matches(/^backend\/packages\/harness\/deerflow\/(agents|subagents)\//) ||
|
||||
matches(/(^|\/)langgraph\.json$/) ||
|
||||
matches(/^frontend\/src\/core\/(api|threads|messages)\//);
|
||||
|
||||
const current = (pr.labels || []).map(l => l.name);
|
||||
const hasSkip = current.includes('skip-validation');
|
||||
|
||||
const desired = [sizeLabel, riskLabel];
|
||||
if (contractSurface && !hasSkip) desired.push('needs-validation');
|
||||
|
||||
const managed = (name) =>
|
||||
name.startsWith('size/') || name.startsWith('risk:') || name === 'needs-validation';
|
||||
|
||||
const toRemove = current.filter(l => managed(l) && !desired.includes(l));
|
||||
const toAdd = desired.filter(l => !current.includes(l));
|
||||
|
||||
for (const name of toRemove) {
|
||||
try {
|
||||
await github.rest.issues.removeLabel({ owner, repo, issue_number: prNumber, name });
|
||||
} catch (e) {
|
||||
if (e.status !== 404) throw e;
|
||||
}
|
||||
}
|
||||
if (toAdd.length) {
|
||||
await github.rest.issues.addLabels({ owner, repo, issue_number: prNumber, labels: toAdd });
|
||||
}
|
||||
core.info(`size=${sizeLabel} risk=${riskLabel} churn=${churn} ` +
|
||||
`validation=${desired.includes('needs-validation')} ` +
|
||||
`(+${toAdd.join(',') || '-'} / -${toRemove.join(',') || '-'})`);
|
||||
|
||||
first-time:
|
||||
if: github.event_name == 'pull_request_target' && github.event.action == 'opened'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Label first-time contributors
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const pr = context.payload.pull_request;
|
||||
const { owner, repo } = context.repo;
|
||||
const assoc = pr.author_association;
|
||||
const isBot = pr.user.type === 'Bot';
|
||||
core.info(`author=${pr.user.login} association=${assoc} bot=${isBot}`);
|
||||
|
||||
// FIRST_TIME_CONTRIBUTOR = no prior merged commit to this repo;
|
||||
// FIRST_TIMER = no prior commit anywhere on GitHub. Either counts.
|
||||
if (isBot || !['FIRST_TIME_CONTRIBUTOR', 'FIRST_TIMER'].includes(assoc)) {
|
||||
core.info('Not a first-time contributor; skipping.');
|
||||
return;
|
||||
}
|
||||
await github.rest.issues.addLabels({
|
||||
owner, repo, issue_number: pr.number, labels: ['first-time-contributor'],
|
||||
});
|
||||
core.info(`Added first-time-contributor to #${pr.number}.`);
|
||||
|
||||
reviewing:
|
||||
if: github.event_name == 'pull_request_review'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Add reviewing label for maintainer reviews
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const { owner, repo } = context.repo;
|
||||
const prNumber = context.payload.pull_request.number;
|
||||
const reviewer = context.payload.review.user.login;
|
||||
|
||||
const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({
|
||||
owner, repo, username: reviewer,
|
||||
});
|
||||
if (!['admin', 'write', 'maintain'].includes(perm.permission)) {
|
||||
core.info(`Reviewer ${reviewer} (${perm.permission}) is not a maintainer; skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const { data: labels } = await github.rest.issues.listLabelsOnIssue({
|
||||
owner, repo, issue_number: prNumber,
|
||||
});
|
||||
if (labels.some(l => l.name === 'reviewing')) {
|
||||
core.info('Already labeled reviewing; skipping.');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await github.rest.issues.addLabels({
|
||||
owner, repo, issue_number: prNumber, labels: ['reviewing'],
|
||||
});
|
||||
core.info(`Added "reviewing" (reviewer ${reviewer}).`);
|
||||
} catch (e) {
|
||||
// 403 is expected for review events on some fork PR contexts.
|
||||
if (e.status === 403) core.info('No permission to label (expected on some fork PRs).');
|
||||
else throw e;
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
name: Triage
|
||||
|
||||
# One workflow for all event-driven PR/issue labeling. Replaces the former
|
||||
# pr-labeler / pr-triage / issue-triage workflows (and drops actions/labeler).
|
||||
#
|
||||
# Design notes:
|
||||
# * All jobs are pure-metadata: they read changed-file lists / PR fields / the
|
||||
# review payload via the API and write labels. PR code is NEVER checked out
|
||||
# or executed, so pull_request_target is safe here.
|
||||
# * Each job only reconciles labels in namespaces IT owns
|
||||
# (area:* / size/* / risk:* / needs-validation). It never touches labels
|
||||
# applied by maintainers or other tools (bug, priority, etc.). first-time-
|
||||
# contributor and reviewing are add-only.
|
||||
# * State is read LIVE (listFiles + listLabelsOnIssue) at run time, not from
|
||||
# the (stale) event payload, so rapid synchronize events converge instead
|
||||
# of thrashing.
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened, ready_for_review]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
issues:
|
||||
types: [opened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
# ── PR: area / size / risk / needs-validation / first-time ─────────────────
|
||||
pr-labels:
|
||||
if: github.event_name == 'pull_request_target' && github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-latest
|
||||
concurrency:
|
||||
group: triage-pr-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
steps:
|
||||
- name: Apply PR labels from live state
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const pr = context.payload.pull_request;
|
||||
const { owner, repo } = context.repo;
|
||||
const num = pr.number;
|
||||
|
||||
// ---- live changed files ----
|
||||
const files = await github.paginate(github.rest.pulls.listFiles, {
|
||||
owner, repo, pull_number: num, per_page: 100,
|
||||
});
|
||||
const paths = files.map(f => f.filename);
|
||||
const m = (re) => paths.some(p => re.test(p));
|
||||
|
||||
// ---- area: replaces .github/labeler.yml (path -> area) ----
|
||||
const AREA_RULES = [
|
||||
['area:frontend', [/^frontend\//]],
|
||||
['area:backend', [/^backend\/app\//, /^backend\/packages\/harness\/deerflow\/(runtime|persistence|config|tools|guardrails|tracing|models|utils|uploads)\//]],
|
||||
['area:agents', [/^backend\/packages\/harness\/deerflow\/(agents|subagents|reflection)\//, /(^|\/)langgraph\.json$/, /^backend\/.*\/prompts\//]],
|
||||
['area:sandbox', [/^docker\//, /^backend\/packages\/harness\/deerflow\/sandbox\//, /(^|\/)Dockerfile$/]],
|
||||
['area:skills', [/^skills\//, /^backend\/packages\/harness\/deerflow\/skills\//, /^frontend\/src\/core\/skills\//]],
|
||||
['area:mcp', [/^backend\/packages\/harness\/deerflow\/mcp\//, /^frontend\/src\/core\/mcp\//]],
|
||||
['area:ci', [/^\.github\//, /^scripts\//]],
|
||||
['area:docs', [/^docs\//, /\.mdx?$/]],
|
||||
['area:deps', [/(^|\/)(pyproject\.toml|uv\.lock|package\.json|pnpm-lock\.yaml)$/]],
|
||||
];
|
||||
const areaLabels = AREA_RULES
|
||||
.filter(([, res]) => res.some(re => m(re)))
|
||||
.map(([label]) => label);
|
||||
|
||||
// ---- size: additions+deletions, excluding lockfiles/snapshots ----
|
||||
const EXCLUDE_SIZE = /(^|\/)(uv\.lock|pnpm-lock\.yaml|package-lock\.json)$|\.snap$/;
|
||||
const churn = files
|
||||
.filter(f => !EXCLUDE_SIZE.test(f.filename))
|
||||
.reduce((s, f) => s + (f.additions || 0) + (f.deletions || 0), 0);
|
||||
const sizeLabel =
|
||||
churn < 20 ? 'size/XS' :
|
||||
churn < 100 ? 'size/S' :
|
||||
churn < 300 ? 'size/M' :
|
||||
churn < 700 ? 'size/L' : 'size/XL';
|
||||
|
||||
// ---- risk ----
|
||||
const docsOnly = paths.length > 0 && paths.every(p =>
|
||||
/\.(md|mdx|txt)$/i.test(p) || p.startsWith('docs/') ||
|
||||
/\.(png|jpe?g|gif|svg|webp|ico)$/i.test(p));
|
||||
const highRisk =
|
||||
m(/^backend\/app\/gateway\//) ||
|
||||
m(/^backend\/packages\/harness\/deerflow\/(agents|subagents|sandbox)\//) ||
|
||||
m(/(^|\/)langgraph\.json$/) ||
|
||||
m(/(^|\/)(auth|authz|security)/i) ||
|
||||
m(/(pyproject\.toml|uv\.lock|package\.json|pnpm-lock\.yaml)$/) ||
|
||||
m(/^docker\//) ||
|
||||
m(/^\.github\/workflows\//);
|
||||
const riskLabel = docsOnly ? 'risk:low' : (highRisk ? 'risk:high' : 'risk:medium');
|
||||
|
||||
// ---- needs-validation: front/back contract surface ----
|
||||
const contract =
|
||||
m(/^backend\/app\/gateway\//) ||
|
||||
m(/^backend\/packages\/harness\/deerflow\/(agents|subagents)\//) ||
|
||||
m(/(^|\/)langgraph\.json$/) ||
|
||||
m(/^frontend\/src\/core\/(api|threads|messages)\//);
|
||||
|
||||
// ---- live current labels (NOT the stale event payload) ----
|
||||
const current = (await github.paginate(github.rest.issues.listLabelsOnIssue, {
|
||||
owner, repo, issue_number: num, per_page: 100,
|
||||
})).map(l => l.name);
|
||||
const hasSkip = current.includes('skip-validation');
|
||||
|
||||
// Reconcile ONLY namespaces we own; never touch others.
|
||||
const owned = (n) =>
|
||||
n.startsWith('area:') || n.startsWith('size/') ||
|
||||
n.startsWith('risk:') || n === 'needs-validation';
|
||||
const desired = new Set([...areaLabels, sizeLabel, riskLabel]);
|
||||
if (contract && !hasSkip) desired.add('needs-validation');
|
||||
|
||||
const toRemove = current.filter(n => owned(n) && !desired.has(n));
|
||||
const toAdd = [...desired].filter(n => !current.includes(n));
|
||||
|
||||
// first-time-contributor: add-only, on opened, real users only.
|
||||
if (context.payload.action === 'opened' &&
|
||||
pr.user.type === 'User' &&
|
||||
['FIRST_TIME_CONTRIBUTOR', 'FIRST_TIMER'].includes(pr.author_association) &&
|
||||
!current.includes('first-time-contributor')) {
|
||||
toAdd.push('first-time-contributor');
|
||||
}
|
||||
|
||||
for (const name of toRemove) {
|
||||
try {
|
||||
await github.rest.issues.removeLabel({ owner, repo, issue_number: num, name });
|
||||
} catch (e) {
|
||||
if (e.status !== 404) throw e;
|
||||
}
|
||||
}
|
||||
if (toAdd.length) {
|
||||
await github.rest.issues.addLabels({ owner, repo, issue_number: num, labels: toAdd });
|
||||
}
|
||||
core.info(`area=[${areaLabels.join(',')}] ${sizeLabel} ${riskLabel} churn=${churn} ` +
|
||||
`validation=${desired.has('needs-validation')} ` +
|
||||
`(+${toAdd.join(',') || '-'} / -${toRemove.join(',') || '-'})`);
|
||||
|
||||
# ── PR: reviewing label on a maintainer's human review ─────────────────────
|
||||
reviewing:
|
||||
if: github.event_name == 'pull_request_review'
|
||||
runs-on: ubuntu-latest
|
||||
concurrency:
|
||||
group: triage-review-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Add reviewing label for maintainer reviews
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const { owner, repo } = context.repo;
|
||||
const num = context.payload.pull_request.number;
|
||||
const review = context.payload.review;
|
||||
const assoc = review.author_association; // payload field; no API call
|
||||
const type = review.user && review.user.type;
|
||||
|
||||
// author_association is NONE for every automated reviewer
|
||||
// (Copilot, CodeRabbit, Codex, Sourcery, ...), so this allowlist
|
||||
// drops them all without a denylist — and never calls the
|
||||
// collaborators API that 404s on "Copilot is not a user".
|
||||
// user.type === 'User' guards the rare bot-added-as-collaborator case.
|
||||
if (!['OWNER', 'MEMBER', 'COLLABORATOR'].includes(assoc) || type !== 'User') {
|
||||
core.info(`reviewer ${review.user && review.user.login} assoc=${assoc} type=${type}; skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const labels = (await github.paginate(github.rest.issues.listLabelsOnIssue, {
|
||||
owner, repo, issue_number: num, per_page: 100,
|
||||
})).map(l => l.name);
|
||||
if (labels.includes('reviewing')) {
|
||||
core.info('Already labeled reviewing; skipping.');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await github.rest.issues.addLabels({
|
||||
owner, repo, issue_number: num, labels: ['reviewing'],
|
||||
});
|
||||
core.info('Added "reviewing".');
|
||||
} catch (e) {
|
||||
if (e.status === 403) core.info('No permission to label (expected on some fork PRs).');
|
||||
else throw e;
|
||||
}
|
||||
|
||||
# ── Issue: needs-triage on every new issue ────────────────────────────────
|
||||
issue-triage:
|
||||
if: github.event_name == 'issues'
|
||||
runs-on: ubuntu-latest
|
||||
concurrency:
|
||||
group: triage-issue-${{ github.event.issue.number }}
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Add needs-triage label
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const { owner, repo } = context.repo;
|
||||
const issue_number = context.payload.issue.number;
|
||||
|
||||
// Read live labels (not the event payload) so labels added at creation
|
||||
// time via the API or by another automation are seen — consistent with
|
||||
// the live-state reads in the PR jobs above.
|
||||
const current = (await github.paginate(github.rest.issues.listLabelsOnIssue, {
|
||||
owner, repo, issue_number, per_page: 100,
|
||||
})).map(l => l.name);
|
||||
if (current.includes('needs-triage')) {
|
||||
core.info('Issue already has needs-triage; nothing to do.');
|
||||
return;
|
||||
}
|
||||
// Self-heal: create the label if it does not exist yet.
|
||||
try {
|
||||
await github.rest.issues.createLabel({
|
||||
owner, repo, name: 'needs-triage', color: 'fef2c0',
|
||||
description: 'Awaiting maintainer triage',
|
||||
});
|
||||
} catch (e) {
|
||||
if (e.status !== 422) throw e; // 422 = already exists
|
||||
}
|
||||
await github.rest.issues.addLabels({
|
||||
owner, repo, issue_number, labels: ['needs-triage'],
|
||||
});
|
||||
core.info(`Added needs-triage to #${issue_number}.`);
|
||||
@@ -247,6 +247,9 @@ Access: http://localhost:2026
|
||||
|
||||
The unified nginx endpoint is same-origin by default and does not emit browser CORS headers. If you run a split-origin or port-forwarded browser client, set `GATEWAY_CORS_ORIGINS` to comma-separated exact origins such as `http://localhost:3000`; the Gateway then applies the CORS allowlist and matching CSRF origin checks.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The Gateway holds run state (RunManager and the stream bridge) in process, so production defaults to a single Gateway worker (`GATEWAY_WORKERS=1`). Raising the worker count without a shared cross-worker stream bridge — which is not yet available — breaks run cancellation, SSE reconnects, request de-duplication, and IM channels, because nginx uses no sticky sessions and each worker keeps its own run state. Scale a single worker up with more CPU/RAM (or move the database and sandbox onto dedicated tiers) instead of raising `GATEWAY_WORKERS`.
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed Docker development guide.
|
||||
|
||||
#### Option 2: Local Development
|
||||
@@ -340,6 +343,8 @@ See the [MCP Server Guide](backend/docs/MCP_SERVER.md) for detailed instructions
|
||||
|
||||
DeerFlow supports receiving tasks from messaging apps. Channels auto-start when configured — no public IP required for any of them.
|
||||
|
||||
DeerFlow can also expose user-owned IM channel connections in the workspace UI. When `channel_connections` is enabled, logged-in users can bind Telegram, Slack, Discord, Feishu/Lark, DingTalk, WeChat, or WeCom from the sidebar / Settings > Channels. It reuses the existing outbound `channels.*` transports, so no public IP or provider callback URL is required. Incoming IM messages then run under the connected DeerFlow user account. See [IM Channel Connections](backend/docs/IM_CHANNEL_CONNECTIONS.md) for setup and security notes.
|
||||
|
||||
| Channel | Transport | Difficulty |
|
||||
|---------|-----------|------------|
|
||||
| Telegram | Bot API (long-polling) | Easy |
|
||||
@@ -585,6 +590,8 @@ A standard Agent Skill is a structured capability module — a Markdown file tha
|
||||
|
||||
Skills are loaded progressively — only when the task needs them, not all at once. This keeps the context window lean and makes DeerFlow work well even with token-sensitive models.
|
||||
|
||||
Users can explicitly activate an enabled skill for a single turn by starting the request with `/skill-name`, for example `/data-analysis analyze uploads/foo.csv`. DeerFlow loads that skill's `SKILL.md` as hidden current-turn context while leaving the base prompt limited to skill metadata. Slash activation respects disabled skills, custom-agent skill whitelists, and existing channel commands such as `/new` and `/help`.
|
||||
|
||||
When you install `.skill` archives through the Gateway, DeerFlow accepts standard optional frontmatter metadata such as `version`, `author`, and `compatibility` instead of rejecting otherwise valid external skills.
|
||||
|
||||
Tools follow the same philosophy. DeerFlow comes with a core toolset — web search, web fetch, file operations, bash execution — and supports custom tools via MCP servers and Python functions. Swap anything. Add anything.
|
||||
|
||||
@@ -24,5 +24,10 @@ config.yaml
|
||||
# Langgraph
|
||||
.langgraph_api
|
||||
|
||||
# Sandbox runtime working dir — pre-created and excluded from uvicorn reload
|
||||
# (scripts/serve.sh, docker/dev-entrypoint.sh). Anchored so it does not match
|
||||
# the source package backend/packages/harness/deerflow/sandbox/.
|
||||
/sandbox/
|
||||
|
||||
# Claude Code settings
|
||||
.claude/settings.local.json
|
||||
|
||||
+43
-22
@@ -192,7 +192,7 @@ from deerflow.config import get_app_config
|
||||
|
||||
### Middleware Chain
|
||||
|
||||
Lead-agent middlewares are assembled in strict append order across `packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py` (`build_lead_runtime_middlewares`) and `packages/harness/deerflow/agents/lead_agent/agent.py` (`_build_middlewares`):
|
||||
Lead-agent middlewares are assembled in strict append order across `packages/harness/deerflow/agents/middlewares/tool_error_handling_middleware.py` (`build_lead_runtime_middlewares`) and `packages/harness/deerflow/agents/lead_agent/agent.py` (`build_middlewares`):
|
||||
|
||||
1. **ThreadDataMiddleware** - Creates per-thread directories under the user's isolation scope (`backend/.deer-flow/users/{user_id}/threads/{thread_id}/user-data/{workspace,uploads,outputs}`); resolves `user_id` via `get_effective_user_id()` (falls back to `"default"` in no-auth mode); Web UI thread deletion now follows LangGraph thread removal with Gateway cleanup of the local thread directory
|
||||
2. **UploadsMiddleware** - Tracks and injects newly uploaded files into conversation
|
||||
@@ -202,16 +202,17 @@ Lead-agent middlewares are assembled in strict append order across `packages/har
|
||||
6. **GuardrailMiddleware** - Pre-tool-call authorization via pluggable `GuardrailProvider` protocol (optional, if `guardrails.enabled` in config). Evaluates each tool call and returns error ToolMessage on deny. Three provider options: built-in `AllowlistProvider` (zero deps), OAP policy providers (e.g. `aport-agent-guardrails`), or custom providers. See [docs/GUARDRAILS.md](docs/GUARDRAILS.md) for setup, usage, and how to implement a provider.
|
||||
7. **SandboxAuditMiddleware** - Audits sandboxed shell/file operations for security logging before tool execution continues
|
||||
8. **ToolErrorHandlingMiddleware** - Converts tool exceptions into error `ToolMessage`s so the run can continue instead of aborting
|
||||
9. **SummarizationMiddleware** - Context reduction when approaching token limits (optional, if enabled)
|
||||
10. **TodoListMiddleware** - Task tracking with `write_todos` tool (optional, if plan_mode)
|
||||
11. **TokenUsageMiddleware** - Records token usage metrics when token tracking is enabled (optional); subagent usage is cached by `tool_call_id` only while token usage is enabled and merged back into the dispatching AIMessage by message position rather than message id
|
||||
12. **TitleMiddleware** - Auto-generates thread title after first complete exchange and normalizes structured message content before prompting the title model
|
||||
13. **MemoryMiddleware** - Queues conversations for async memory update (filters to user + final AI responses)
|
||||
14. **ViewImageMiddleware** - Injects base64 image data before LLM call (conditional on vision support)
|
||||
15. **DeferredToolFilterMiddleware** - Hides deferred (MCP) tool schemas from the bound model using a build-time deferred-name set + catalog hash, reading per-thread promotions from `ThreadState.promoted` (hash-scoped, no ContextVar); a tool becomes bound on subsequent turns after `tool_search` returns its schema (optional, if `tool_search.enabled`)
|
||||
16. **SubagentLimitMiddleware** - Truncates excess `task` tool calls from model response to enforce `MAX_CONCURRENT_SUBAGENTS` limit (optional, if `subagent_enabled`)
|
||||
17. **LoopDetectionMiddleware** - Detects repeated tool-call loops; hard-stop responses clear both structured `tool_calls` and raw provider tool-call metadata before forcing a final text answer
|
||||
18. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, interrupts via `Command(goto=END)` (must be last)
|
||||
9. **SkillActivationMiddleware** - Detects strict `/skill-name task` syntax on the latest real user message, resolves only enabled and runtime-allowed skills, reads `SKILL.md` from trusted skill storage, injects the skill body as hidden current-turn model context, and records a `middleware:skill_activation` audit event with skill name, category, path, and content hash
|
||||
10. **SummarizationMiddleware** - Context reduction when approaching token limits (optional, if enabled)
|
||||
11. **TodoListMiddleware** - Task tracking with `write_todos` tool (optional, if plan_mode)
|
||||
12. **TokenUsageMiddleware** - Records token usage metrics when token tracking is enabled (optional); subagent usage is cached by `tool_call_id` only while token usage is enabled and merged back into the dispatching AIMessage by message position rather than message id
|
||||
13. **TitleMiddleware** - Auto-generates thread title after first complete exchange and normalizes structured message content before prompting the title model
|
||||
14. **MemoryMiddleware** - Queues conversations for async memory update (filters to user + final AI responses)
|
||||
15. **ViewImageMiddleware** - Injects base64 image data before LLM call (conditional on vision support)
|
||||
16. **DeferredToolFilterMiddleware** - Hides deferred (MCP) tool schemas from the bound model using a build-time deferred-name set + catalog hash, reading per-thread promotions from `ThreadState.promoted` (hash-scoped, no ContextVar); a tool becomes bound on subsequent turns after `tool_search` returns its schema (optional, if `tool_search.enabled`)
|
||||
17. **SubagentLimitMiddleware** - Truncates excess `task` tool calls from model response to enforce `MAX_CONCURRENT_SUBAGENTS` limit (optional, if `subagent_enabled`)
|
||||
18. **LoopDetectionMiddleware** - Detects repeated tool-call loops; hard-stop responses clear both structured `tool_calls` and raw provider tool-call metadata before forcing a final text answer
|
||||
19. **ClarificationMiddleware** - Intercepts `ask_clarification` tool calls, interrupts via `Command(goto=END)` (must be last)
|
||||
|
||||
### Configuration System
|
||||
|
||||
@@ -348,6 +349,7 @@ Proxied through nginx: `/api/langgraph/*` → Gateway LangGraph-compatible runti
|
||||
- **Format**: Directory with `SKILL.md` (YAML frontmatter: name, description, license, allowed-tools)
|
||||
- **Loading**: `load_skills()` recursively scans `skills/{public,custom}` for `SKILL.md`, parses metadata, and reads enabled state from extensions_config.json
|
||||
- **Injection**: Enabled skills listed in agent system prompt with container paths
|
||||
- **Slash activation**: `/skill-name task` loads that enabled skill's `SKILL.md` for the current model call only. The resolver rejects leading whitespace, missing separators, reserved channel commands (`/new`, `/help`, `/bootstrap`, `/status`, `/models`, `/memory`), disabled skills, and skills outside a custom agent's whitelist.
|
||||
- **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory
|
||||
|
||||
### Model Factory (`packages/harness/deerflow/models/factory.py`)
|
||||
@@ -367,8 +369,7 @@ Proxied through nginx: `/api/langgraph/*` → Gateway LangGraph-compatible runti
|
||||
|
||||
### IM Channels System (`app/channels/`)
|
||||
|
||||
Bridges external messaging platforms (Feishu, Slack, Telegram, DingTalk) to the DeerFlow agent via Gateway's LangGraph-compatible API.
|
||||
|
||||
Bridges external messaging platforms (Feishu, Slack, Telegram, Discord, DingTalk) to the DeerFlow agent via Gateway's LangGraph-compatible API.
|
||||
|
||||
**Architecture**: Channels communicate with Gateway through the `langgraph-sdk` HTTP client (same as the frontend), ensuring threads are created and managed server-side. The internal SDK client injects process-local internal auth plus a matching CSRF cookie/header pair so Gateway accepts state-changing thread/run requests from channel workers without relying on browser session cookies.
|
||||
|
||||
@@ -378,18 +379,21 @@ Bridges external messaging platforms (Feishu, Slack, Telegram, DingTalk) to the
|
||||
- `manager.py` - Core dispatcher: creates threads via `client.threads.create()`, routes commands, keeps Slack/Telegram on `client.runs.wait()`, and uses `client.runs.stream(["messages-tuple", "values"])` for Feishu incremental outbound updates
|
||||
- `base.py` - Abstract `Channel` base class (start/stop/send lifecycle)
|
||||
- `service.py` - Manages lifecycle of all configured channels from `config.yaml`
|
||||
- `slack.py` / `feishu.py` / `telegram.py` / `dingtalk.py` - Platform-specific implementations (`feishu.py` tracks the running card `message_id` in memory and patches the same card in place; `dingtalk.py` optionally uses AI Card streaming for in-place updates when `card_template_id` is configured)
|
||||
- `slack.py` / `feishu.py` / `telegram.py` / `discord.py` / `dingtalk.py` - Platform-specific implementations (`feishu.py` tracks the running card `message_id` in memory and patches the same card in place; `dingtalk.py` optionally uses AI Card streaming for in-place updates when `card_template_id` is configured)
|
||||
- `app/gateway/routers/channel_connections.py` - Browser-facing user connection and disconnect APIs
|
||||
- `deerflow.persistence.channel_connections` - SQL-backed user-owned connection, optional credential, connect state, and conversation store
|
||||
|
||||
**Message Flow**:
|
||||
1. External platform -> Channel impl -> `MessageBus.publish_inbound()`
|
||||
2. `ChannelManager._dispatch_loop()` consumes from queue
|
||||
3. For chat: look up/create thread through Gateway's LangGraph-compatible API
|
||||
4. Feishu chat: `runs.stream()` → accumulate AI text → publish multiple outbound updates (`is_final=False`) → publish final outbound (`is_final=True`)
|
||||
5. Slack/Telegram chat: `runs.wait()` → extract final response → publish outbound
|
||||
6. Feishu channel sends one running reply card up front, then patches the same card for each outbound update (card JSON sets `config.update_multi=true` for Feishu's patch API requirement)
|
||||
7. DingTalk AI Card mode (when `card_template_id` configured): `runs.stream()` → create card with initial text → stream updates via `PUT /v1.0/card/streaming` → finalize on `is_final=True`. Falls back to `sampleMarkdown` if card creation or streaming fails
|
||||
8. For commands (`/new`, `/status`, `/models`, `/memory`, `/help`): handle locally or query Gateway API
|
||||
9. Outbound → channel callbacks → platform reply
|
||||
3. For user-owned channel connections, incoming messages carry `connection_id`, `owner_user_id`, and `workspace_id`; `owner_user_id` becomes the DeerFlow run `user_id`, while the raw platform user id remains `channel_user_id`
|
||||
4. For chat: look up/create thread through Gateway's LangGraph-compatible API
|
||||
5. Feishu chat: `runs.stream()` → accumulate AI text → publish multiple outbound updates (`is_final=False`) → publish final outbound (`is_final=True`)
|
||||
6. Slack/Telegram chat: `runs.wait()` → extract final response → publish outbound
|
||||
7. Feishu channel sends one running reply card up front, then patches the same card for each outbound update (card JSON sets `config.update_multi=true` for Feishu's patch API requirement)
|
||||
8. DingTalk AI Card mode (when `card_template_id` configured): `runs.stream()` → create card with initial text → stream updates via `PUT /v1.0/card/streaming` → finalize on `is_final=True`. Falls back to `sampleMarkdown` if card creation or streaming fails
|
||||
9. For commands (`/new`, `/status`, `/models`, `/memory`, `/help`): handle locally or query Gateway API
|
||||
10. Outbound → channel callbacks → platform reply
|
||||
|
||||
**Configuration** (`config.yaml` -> `channels`):
|
||||
- `langgraph_url` - LangGraph-compatible Gateway API base URL (default: `http://localhost:8001/api`)
|
||||
@@ -397,6 +401,16 @@ Bridges external messaging platforms (Feishu, Slack, Telegram, DingTalk) to the
|
||||
- In Docker Compose, IM channels run inside the `gateway` container, so `localhost` points back to that container. Use `http://gateway:8001/api` for `langgraph_url` and `http://gateway:8001` for `gateway_url`, or set `DEER_FLOW_CHANNELS_LANGGRAPH_URL` / `DEER_FLOW_CHANNELS_GATEWAY_URL`.
|
||||
- Per-channel configs: `feishu` (app_id, app_secret), `slack` (bot_token, app_token), `telegram` (bot_token), `dingtalk` (client_id, client_secret, optional `card_template_id` for AI Card streaming)
|
||||
|
||||
**User-owned channel connections** (`config.yaml` -> `channel_connections`):
|
||||
- Disabled by default. It is a user-binding layer on top of the existing `channels.*` runtime config, not a replacement for provider bot credentials.
|
||||
- No public IP, OAuth callback URL, or provider webhook route is required by the current implementation.
|
||||
- Telegram uses a deep-link `/start <code>` flow over the existing long-polling worker. Slack, Discord, Feishu/Lark, DingTalk, WeChat, and WeCom use `/connect <code>` over their existing outbound channel workers.
|
||||
- Frontend APIs: `GET /api/channels/providers`, `GET /api/channels/connections`, `POST /api/channels/{provider}/connect`, and `DELETE /api/channels/connections/{connection_id}`.
|
||||
- Browser APIs remain protected by normal Gateway auth/CSRF. Provider messages arrive through the already-configured channel workers.
|
||||
- Slack replies use the configured operator bot token from `channels.slack` unless a future provider-token flow stores per-connection credentials.
|
||||
- Telegram, Slack, Discord, Feishu/Lark, DingTalk, WeChat, and WeCom workers resolve incoming platform identities to connection records before reaching `ChannelManager`.
|
||||
- See `backend/docs/IM_CHANNEL_CONNECTIONS.md` for provider setup and operational notes.
|
||||
|
||||
|
||||
### Memory System (`packages/harness/deerflow/agents/memory/`)
|
||||
|
||||
@@ -427,6 +441,12 @@ Bridges external messaging platforms (Feishu, Slack, Telegram, DingTalk) to the
|
||||
4. Applies updates atomically (temp file + rename) with cache invalidation, skipping duplicate fact content before append
|
||||
5. Next interaction injects top 15 facts + context into `<memory>` tags in system prompt
|
||||
|
||||
**Token counting** (`packages/harness/deerflow/agents/memory/prompt.py`):
|
||||
- `_count_tokens` budgets the injection. In default `tiktoken` mode, the encoding is loaded lazily and cached.
|
||||
- Failed tiktoken loads are cached with a timestamp. During the fixed cooldown (`_TIKTOKEN_RETRY_COOLDOWN_S`, 600s), callers fall back to char estimation immediately instead of re-triggering the blocking BPE download; after the cooldown, transient outages can self-heal without a restart.
|
||||
- In-flight loads are cached as a LOADING sentinel so concurrent callers fall back instead of spawning more blocking threads.
|
||||
- Set `memory.token_counting: char` to skip tiktoken entirely and use the network-free CJK-aware char estimate.
|
||||
|
||||
Focused regression coverage for the updater lives in `backend/tests/test_memory_updater.py`.
|
||||
|
||||
**Configuration** (`config.yaml` → `memory`):
|
||||
@@ -436,6 +456,7 @@ Focused regression coverage for the updater lives in `backend/tests/test_memory_
|
||||
- `model_name` - LLM for updates (null = default model)
|
||||
- `max_facts` / `fact_confidence_threshold` - Fact storage limits (100 / 0.7)
|
||||
- `max_injection_tokens` - Token limit for prompt injection (2000)
|
||||
- `token_counting` - Token counting strategy for the injection budget: `tiktoken` (default, accurate but may download BPE data from a public endpoint on first use — can block for a long time in network-restricted environments, see issues #3402/#3429) or `char` (network-free CJK-aware char estimate, never touches tiktoken)
|
||||
|
||||
### Reflection System (`packages/harness/deerflow/reflection/`)
|
||||
|
||||
@@ -493,7 +514,7 @@ Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` me
|
||||
- `"messages-tuple"` — per-chunk update: for AI text this is a **delta** (concat per `id` to rebuild the full message); tool calls and tool results are emitted once each
|
||||
- `"custom"` — forwarded from `StreamWriter`
|
||||
- `"end"` — stream finished (carries cumulative `usage` counted once per message id)
|
||||
- Agent created lazily via `create_agent()` + `_build_middlewares()`, same as `make_lead_agent`
|
||||
- Agent created lazily via `create_agent()` + `build_middlewares()`, same as `make_lead_agent`
|
||||
- Supports `checkpointer` parameter for state persistence across turns
|
||||
- `reset_agent()` forces agent recreation (e.g. after memory or skill changes)
|
||||
- See [docs/STREAMING.md](docs/STREAMING.md) for the full design: why Gateway and DeerFlowClient are parallel paths, LangGraph's `stream_mode` semantics, the per-id dedup invariants, and regression testing strategy
|
||||
|
||||
@@ -18,3 +18,21 @@ KNOWN_CHANNEL_COMMANDS: frozenset[str] = frozenset(
|
||||
"/help",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def extract_connect_code(text: str) -> str | None:
|
||||
"""Extract the one-time channel binding code from a connect command."""
|
||||
parts = text.strip().split()
|
||||
if len(parts) < 2:
|
||||
return None
|
||||
command = parts[0].lower()
|
||||
if command in {"/connect", "connect"}:
|
||||
return parts[1]
|
||||
return None
|
||||
|
||||
|
||||
def is_known_channel_command(text: str) -> bool:
|
||||
"""Return whether text starts with a registered channel control command."""
|
||||
if not text.startswith("/"):
|
||||
return False
|
||||
return text.split(maxsplit=1)[0].lower() in KNOWN_CHANNEL_COMMANDS
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Helpers for attaching persisted channel connection ownership to inbound messages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.channels.message_bus import InboundMessage
|
||||
|
||||
|
||||
async def attach_connection_identity(
|
||||
inbound: InboundMessage,
|
||||
*,
|
||||
repo: Any,
|
||||
provider: str,
|
||||
workspace_id: str | None,
|
||||
fallback_without_workspace: bool = False,
|
||||
) -> InboundMessage:
|
||||
"""Attach connection metadata to an inbound message when a persisted binding exists."""
|
||||
if repo is None:
|
||||
return inbound
|
||||
|
||||
workspace_candidates: list[str | None] = []
|
||||
if workspace_id:
|
||||
workspace_candidates.append(workspace_id)
|
||||
if fallback_without_workspace:
|
||||
workspace_candidates.append(None)
|
||||
if not workspace_candidates:
|
||||
return inbound
|
||||
|
||||
for candidate in workspace_candidates:
|
||||
connection = await repo.find_connection_by_external_identity(
|
||||
provider=provider,
|
||||
external_account_id=inbound.user_id,
|
||||
workspace_id=candidate,
|
||||
)
|
||||
if connection is None:
|
||||
continue
|
||||
|
||||
inbound.connection_id = connection["id"]
|
||||
inbound.owner_user_id = connection["owner_user_id"]
|
||||
inbound.workspace_id = connection.get("workspace_id")
|
||||
return inbound
|
||||
|
||||
return inbound
|
||||
@@ -14,7 +14,8 @@ from typing import Any
|
||||
import httpx
|
||||
|
||||
from app.channels.base import Channel
|
||||
from app.channels.commands import KNOWN_CHANNEL_COMMANDS
|
||||
from app.channels.commands import extract_connect_code, is_known_channel_command
|
||||
from app.channels.connection_identity import attach_connection_identity
|
||||
from app.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -59,9 +60,7 @@ def _normalize_allowed_users(allowed_users: Any) -> set[str]:
|
||||
|
||||
|
||||
def _is_dingtalk_command(text: str) -> bool:
|
||||
if not text.startswith("/"):
|
||||
return False
|
||||
return text.split(maxsplit=1)[0].lower() in KNOWN_CHANNEL_COMMANDS
|
||||
return is_known_channel_command(text)
|
||||
|
||||
|
||||
def _extract_text_from_rich_text(rich_text_list: list) -> str:
|
||||
@@ -138,6 +137,7 @@ class DingTalkChannel(Channel):
|
||||
self._incoming_messages: dict[str, Any] = {}
|
||||
self._incoming_messages_lock = threading.Lock()
|
||||
self._card_repliers: dict[str, Any] = {}
|
||||
self._connection_repo = config.get("connection_repo")
|
||||
|
||||
@property
|
||||
def supports_streaming(self) -> bool:
|
||||
@@ -397,6 +397,24 @@ class DingTalkChannel(Channel):
|
||||
text[:100],
|
||||
)
|
||||
|
||||
connect_code = extract_connect_code(text)
|
||||
if connect_code and self._connection_repo is not None:
|
||||
if self._main_loop and self._main_loop.is_running():
|
||||
fut = asyncio.run_coroutine_threadsafe(
|
||||
self._bind_connection_from_connect_code(
|
||||
conversation_type=conversation_type,
|
||||
sender_staff_id=sender_staff_id,
|
||||
sender_nick=sender_nick,
|
||||
conversation_id=conversation_id,
|
||||
code=connect_code,
|
||||
),
|
||||
self._main_loop,
|
||||
)
|
||||
fut.add_done_callback(lambda f, mid=msg_id: self._log_future_error(f, "bind_connection", mid))
|
||||
else:
|
||||
logger.warning("[DingTalk] main loop not running, cannot bind channel connection")
|
||||
return
|
||||
|
||||
if _is_dingtalk_command(text):
|
||||
msg_type = InboundMessageType.COMMAND
|
||||
else:
|
||||
@@ -452,11 +470,95 @@ class DingTalkChannel(Channel):
|
||||
return ""
|
||||
|
||||
async def _prepare_inbound(self, chat_id: str, inbound: InboundMessage) -> None:
|
||||
inbound = await self._attach_connection_identity(inbound)
|
||||
# Running reply must finish before publish_inbound so AI card tracks are
|
||||
# registered before the manager emits streaming outbounds.
|
||||
await self._send_running_reply(chat_id, inbound)
|
||||
await self.bus.publish_inbound(inbound)
|
||||
|
||||
@staticmethod
|
||||
def _connection_workspace_id(conversation_type: str, conversation_id: str) -> str | None:
|
||||
if conversation_type == _CONVERSATION_TYPE_GROUP and conversation_id:
|
||||
return conversation_id
|
||||
return None
|
||||
|
||||
async def _attach_connection_identity(self, inbound: InboundMessage) -> InboundMessage:
|
||||
conversation_type = str(inbound.metadata.get("conversation_type") or _CONVERSATION_TYPE_P2P)
|
||||
conversation_id = str(inbound.metadata.get("conversation_id") or "")
|
||||
return await attach_connection_identity(
|
||||
inbound,
|
||||
repo=self._connection_repo,
|
||||
provider="dingtalk",
|
||||
workspace_id=self._connection_workspace_id(conversation_type, conversation_id),
|
||||
fallback_without_workspace=True,
|
||||
)
|
||||
|
||||
async def _bind_connection_from_connect_code(
|
||||
self,
|
||||
*,
|
||||
conversation_type: str,
|
||||
sender_staff_id: str,
|
||||
sender_nick: str,
|
||||
conversation_id: str,
|
||||
code: str,
|
||||
) -> bool:
|
||||
if self._connection_repo is None or not code:
|
||||
return False
|
||||
|
||||
state = await self._connection_repo.consume_oauth_state(provider="dingtalk", state=code)
|
||||
if state is None:
|
||||
await self._send_connection_reply(
|
||||
conversation_type,
|
||||
sender_staff_id,
|
||||
conversation_id,
|
||||
"DingTalk connection code is invalid or expired.",
|
||||
)
|
||||
return True
|
||||
|
||||
if not sender_staff_id:
|
||||
await self._send_connection_reply(
|
||||
conversation_type,
|
||||
sender_staff_id,
|
||||
conversation_id,
|
||||
"DingTalk connection could not be completed from this message.",
|
||||
)
|
||||
return True
|
||||
|
||||
await self._connection_repo.upsert_connection(
|
||||
owner_user_id=state["owner_user_id"],
|
||||
provider="dingtalk",
|
||||
external_account_id=sender_staff_id,
|
||||
external_account_name=sender_nick or None,
|
||||
workspace_id=self._connection_workspace_id(conversation_type, conversation_id),
|
||||
metadata={
|
||||
"conversation_type": conversation_type,
|
||||
"conversation_id": conversation_id,
|
||||
},
|
||||
status="connected",
|
||||
)
|
||||
await self._send_connection_reply(
|
||||
conversation_type,
|
||||
sender_staff_id,
|
||||
conversation_id,
|
||||
"DingTalk connected to DeerFlow.",
|
||||
)
|
||||
return True
|
||||
|
||||
async def _send_connection_reply(
|
||||
self,
|
||||
conversation_type: str,
|
||||
sender_staff_id: str,
|
||||
conversation_id: str,
|
||||
text: str,
|
||||
) -> None:
|
||||
robot_code = self._client_id
|
||||
if conversation_type == _CONVERSATION_TYPE_GROUP:
|
||||
if conversation_id:
|
||||
await self._send_text_message_to_group(robot_code, conversation_id, text)
|
||||
return
|
||||
if sender_staff_id:
|
||||
await self._send_text_message_to_user(robot_code, sender_staff_id, text)
|
||||
|
||||
async def _send_running_reply(self, chat_id: str, inbound: InboundMessage) -> None:
|
||||
conversation_type = inbound.metadata.get("conversation_type", _CONVERSATION_TYPE_P2P)
|
||||
sender_staff_id = inbound.metadata.get("sender_staff_id", "")
|
||||
|
||||
@@ -10,7 +10,9 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.channels.base import Channel
|
||||
from app.channels.message_bus import InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment
|
||||
from app.channels.commands import extract_connect_code, is_known_channel_command
|
||||
from app.channels.connection_identity import attach_connection_identity
|
||||
from app.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -69,6 +71,7 @@ class DiscordChannel(Channel):
|
||||
self._discord_loop: asyncio.AbstractEventLoop | None = None
|
||||
self._main_loop: asyncio.AbstractEventLoop | None = None
|
||||
self._discord_module = None
|
||||
self._connection_repo = config.get("connection_repo")
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._running:
|
||||
@@ -286,6 +289,10 @@ class DiscordChannel(Channel):
|
||||
text = text.replace(bot_mention or "", "").replace(alt_mention or "", "").replace(standard_mention or "", "").strip()
|
||||
# Don't return early if text is empty — still process the mention (e.g., create thread)
|
||||
|
||||
connect_code = extract_connect_code(text)
|
||||
if connect_code and await self._bind_connection_from_connect_code(message, connect_code):
|
||||
return
|
||||
|
||||
# --- Determine thread/channel routing and typing target ---
|
||||
thread_id = None
|
||||
chat_id = None
|
||||
@@ -300,7 +307,7 @@ class DiscordChannel(Channel):
|
||||
|
||||
# If this is a known active thread, process normally
|
||||
if thread_id in self._active_thread_ids:
|
||||
msg_type = InboundMessageType.COMMAND if text.startswith("/") else InboundMessageType.CHAT
|
||||
msg_type = InboundMessageType.COMMAND if is_known_channel_command(text) else InboundMessageType.CHAT
|
||||
inbound = self._make_inbound(
|
||||
chat_id=chat_id,
|
||||
user_id=str(message.author.id),
|
||||
@@ -314,6 +321,7 @@ class DiscordChannel(Channel):
|
||||
},
|
||||
)
|
||||
inbound.topic_id = thread_id
|
||||
inbound = await self._attach_connection_identity(inbound, guild_id=str(guild.id) if guild else None)
|
||||
self._publish(inbound)
|
||||
# Start typing indicator in the thread
|
||||
if typing_target:
|
||||
@@ -407,7 +415,7 @@ class DiscordChannel(Channel):
|
||||
chat_id = channel_id
|
||||
typing_target = message.channel # Type into the channel
|
||||
|
||||
msg_type = InboundMessageType.COMMAND if text.startswith("/") else InboundMessageType.CHAT
|
||||
msg_type = InboundMessageType.COMMAND if is_known_channel_command(text) else InboundMessageType.CHAT
|
||||
inbound = self._make_inbound(
|
||||
chat_id=chat_id,
|
||||
user_id=str(message.author.id),
|
||||
@@ -421,6 +429,7 @@ class DiscordChannel(Channel):
|
||||
},
|
||||
)
|
||||
inbound.topic_id = thread_id
|
||||
inbound = await self._attach_connection_identity(inbound, guild_id=str(guild.id) if guild else None)
|
||||
|
||||
# Start typing indicator in the correct target (thread or channel)
|
||||
if typing_target:
|
||||
@@ -435,6 +444,60 @@ class DiscordChannel(Channel):
|
||||
future = asyncio.run_coroutine_threadsafe(self.bus.publish_inbound(inbound), self._main_loop)
|
||||
future.add_done_callback(lambda f: logger.exception("[Discord] publish_inbound failed", exc_info=f.exception()) if f.exception() else None)
|
||||
|
||||
async def _attach_connection_identity(self, inbound: InboundMessage, guild_id: str | None = None) -> InboundMessage:
|
||||
return await attach_connection_identity(
|
||||
inbound,
|
||||
repo=self._connection_repo,
|
||||
provider="discord",
|
||||
workspace_id=guild_id,
|
||||
fallback_without_workspace=True,
|
||||
)
|
||||
|
||||
async def _bind_connection_from_connect_code(self, message, code: str) -> bool:
|
||||
if self._connection_repo is None or not code:
|
||||
return False
|
||||
|
||||
state = await self._connection_repo.consume_oauth_state(provider="discord", state=code)
|
||||
if state is None:
|
||||
await self._send_connection_reply(message, "Discord connection code is invalid or expired.")
|
||||
return True
|
||||
|
||||
guild = getattr(message, "guild", None)
|
||||
channel = getattr(message, "channel", None)
|
||||
author = getattr(message, "author", None)
|
||||
user_id = str(getattr(author, "id", "") or "")
|
||||
if not user_id:
|
||||
await self._send_connection_reply(message, "Discord connection could not be completed from this message.")
|
||||
return True
|
||||
|
||||
guild_id = str(getattr(guild, "id", "") or "") or None
|
||||
await self._connection_repo.upsert_connection(
|
||||
owner_user_id=state["owner_user_id"],
|
||||
provider="discord",
|
||||
external_account_id=user_id,
|
||||
external_account_name=getattr(author, "display_name", None) or getattr(author, "name", None),
|
||||
workspace_id=guild_id,
|
||||
workspace_name=getattr(guild, "name", None) if guild is not None else None,
|
||||
metadata={
|
||||
"guild_id": guild_id,
|
||||
"channel_id": str(getattr(channel, "id", "") or ""),
|
||||
},
|
||||
status="connected",
|
||||
)
|
||||
await self._send_connection_reply(message, "Discord connected to DeerFlow.")
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
async def _send_connection_reply(message, text: str) -> None:
|
||||
channel = getattr(message, "channel", None)
|
||||
send = getattr(channel, "send", None)
|
||||
if send is None:
|
||||
return
|
||||
try:
|
||||
await send(text)
|
||||
except Exception:
|
||||
logger.exception("[Discord] failed to send connection reply")
|
||||
|
||||
def _run_client(self) -> None:
|
||||
self._discord_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._discord_loop)
|
||||
|
||||
@@ -11,7 +11,8 @@ import time
|
||||
from typing import Any, Literal
|
||||
|
||||
from app.channels.base import Channel
|
||||
from app.channels.commands import KNOWN_CHANNEL_COMMANDS
|
||||
from app.channels.commands import extract_connect_code, is_known_channel_command
|
||||
from app.channels.connection_identity import attach_connection_identity
|
||||
from app.channels.message_bus import (
|
||||
PENDING_CLARIFICATION_METADATA_KEY,
|
||||
RESOLVED_FROM_PENDING_CLARIFICATION_METADATA_KEY,
|
||||
@@ -30,9 +31,7 @@ PENDING_CLARIFICATION_TTL_SECONDS = 30 * 60
|
||||
|
||||
|
||||
def _is_feishu_command(text: str) -> bool:
|
||||
if not text.startswith("/"):
|
||||
return False
|
||||
return text.split(maxsplit=1)[0].lower() in KNOWN_CHANNEL_COMMANDS
|
||||
return is_known_channel_command(text)
|
||||
|
||||
|
||||
class FeishuChannel(Channel):
|
||||
@@ -73,6 +72,7 @@ class FeishuChannel(Channel):
|
||||
self._CreateImageRequestBody = None
|
||||
self._GetMessageResourceRequest = None
|
||||
self._thread_lock = threading.Lock()
|
||||
self._connection_repo = config.get("connection_repo")
|
||||
|
||||
@staticmethod
|
||||
def _non_empty_str(value: Any) -> str | None:
|
||||
@@ -88,6 +88,23 @@ class FeishuChannel(Channel):
|
||||
def supports_streaming(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
if not self._running:
|
||||
return False
|
||||
return self._thread is not None and self._thread.is_alive()
|
||||
|
||||
def _build_event_handler(self, lark):
|
||||
return (
|
||||
lark.EventDispatcherHandler.builder("", "")
|
||||
.register_p2_im_message_receive_v1(self._on_message)
|
||||
.register_p2_im_message_message_read_v1(self._on_ignored_message_event)
|
||||
.register_p2_im_message_reaction_created_v1(self._on_ignored_message_event)
|
||||
.register_p2_im_message_reaction_deleted_v1(self._on_ignored_message_event)
|
||||
.register_p2_im_message_recalled_v1(self._on_ignored_message_event)
|
||||
.build()
|
||||
)
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._running:
|
||||
return
|
||||
@@ -181,7 +198,7 @@ class FeishuChannel(Channel):
|
||||
# thread's uvloop.
|
||||
_ws_client_mod.loop = loop
|
||||
|
||||
event_handler = lark.EventDispatcherHandler.builder("", "").register_p2_im_message_receive_v1(self._on_message).build()
|
||||
event_handler = self._build_event_handler(lark)
|
||||
ws_client = lark.ws.Client(
|
||||
app_id=app_id,
|
||||
app_secret=app_secret,
|
||||
@@ -193,6 +210,10 @@ class FeishuChannel(Channel):
|
||||
except Exception:
|
||||
if self._running:
|
||||
logger.exception("Feishu WebSocket error")
|
||||
self._running = False
|
||||
|
||||
def _on_ignored_message_event(self, event) -> None:
|
||||
logger.debug("[Feishu] ignoring non-content message event: %s", type(event).__name__)
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._running = False
|
||||
@@ -728,11 +749,47 @@ class FeishuChannel(Channel):
|
||||
|
||||
async def _prepare_inbound(self, msg_id: str, inbound) -> None:
|
||||
"""Kick off Feishu side effects without delaying inbound dispatch."""
|
||||
inbound = await self._attach_connection_identity(inbound)
|
||||
reaction_task = asyncio.create_task(self._add_reaction(msg_id, "OK"))
|
||||
self._track_background_task(reaction_task, name="add_reaction", msg_id=msg_id)
|
||||
self._ensure_running_card_started(msg_id)
|
||||
await self.bus.publish_inbound(inbound)
|
||||
|
||||
async def _attach_connection_identity(self, inbound: InboundMessage) -> InboundMessage:
|
||||
return await attach_connection_identity(
|
||||
inbound,
|
||||
repo=self._connection_repo,
|
||||
provider="feishu",
|
||||
workspace_id=inbound.chat_id,
|
||||
)
|
||||
|
||||
async def _bind_connection_from_connect_code(self, *, message_id: str, chat_id: str, user_id: str, code: str) -> bool:
|
||||
if self._connection_repo is None or not code:
|
||||
return False
|
||||
|
||||
state = await self._connection_repo.consume_oauth_state(provider="feishu", state=code)
|
||||
if state is None:
|
||||
await self._reply_card(message_id, "Feishu connection code is invalid or expired.")
|
||||
return True
|
||||
|
||||
if not user_id or not chat_id:
|
||||
await self._reply_card(message_id, "Feishu connection could not be completed from this message.")
|
||||
return True
|
||||
|
||||
await self._connection_repo.upsert_connection(
|
||||
owner_user_id=state["owner_user_id"],
|
||||
provider="feishu",
|
||||
external_account_id=user_id,
|
||||
workspace_id=chat_id,
|
||||
metadata={
|
||||
"chat_id": chat_id,
|
||||
"message_id": message_id,
|
||||
},
|
||||
status="connected",
|
||||
)
|
||||
await self._reply_card(message_id, "Feishu connected to DeerFlow.")
|
||||
return True
|
||||
|
||||
def _on_message(self, event) -> None:
|
||||
"""Called by lark-oapi when a message is received (runs in lark thread)."""
|
||||
try:
|
||||
@@ -821,6 +878,23 @@ class FeishuChannel(Channel):
|
||||
logger.info("[Feishu] empty text, ignoring message")
|
||||
return
|
||||
|
||||
connect_code = extract_connect_code(text)
|
||||
if connect_code and self._connection_repo is not None:
|
||||
if self._main_loop and self._main_loop.is_running():
|
||||
fut = asyncio.run_coroutine_threadsafe(
|
||||
self._bind_connection_from_connect_code(
|
||||
message_id=msg_id,
|
||||
chat_id=chat_id,
|
||||
user_id=sender_id,
|
||||
code=connect_code,
|
||||
),
|
||||
self._main_loop,
|
||||
)
|
||||
fut.add_done_callback(lambda f, mid=msg_id: self._log_future_error(f, "bind_connection", mid))
|
||||
else:
|
||||
logger.warning("[Feishu] main loop not running, cannot bind channel connection")
|
||||
return
|
||||
|
||||
# Only treat known slash commands as commands; absolute paths and
|
||||
# other slash-prefixed text should be handled as normal chat.
|
||||
if _is_feishu_command(text):
|
||||
|
||||
+256
-40
@@ -8,6 +8,7 @@ import mimetypes
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -26,8 +27,13 @@ from app.channels.message_bus import (
|
||||
from app.channels.store import ChannelStore
|
||||
from app.gateway.csrf_middleware import CSRF_COOKIE_NAME, CSRF_HEADER_NAME, generate_csrf_token
|
||||
from app.gateway.internal_auth import create_internal_auth_headers
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.paths import make_safe_user_id
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.skills.slash import parse_slash_skill_reference
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.storage.skill_storage import SkillStorage
|
||||
from deerflow.utils.messages import ORIGINAL_USER_CONTENT_KEY
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -124,6 +130,16 @@ class InvalidChannelSessionConfigError(ValueError):
|
||||
"""Raised when IM channel session overrides contain invalid agent config."""
|
||||
|
||||
|
||||
class SlashSkillCommandResolutionError(RuntimeError):
|
||||
"""Raised when IM slash-skill command resolution cannot complete safely."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SlashSkillCommandResolution:
|
||||
route_to_chat: bool = False
|
||||
failure_message: str | None = None
|
||||
|
||||
|
||||
def _is_thread_busy_error(exc: BaseException | None) -> bool:
|
||||
if exc is None:
|
||||
return False
|
||||
@@ -258,6 +274,22 @@ def _response_metadata(base_metadata: dict[str, Any], *, pending_clarification:
|
||||
return metadata
|
||||
|
||||
|
||||
def _thread_channel_metadata(msg: InboundMessage) -> dict[str, Any]:
|
||||
channel_source: dict[str, Any] = {
|
||||
"type": "im_channel",
|
||||
"provider": msg.channel_name,
|
||||
"chat_id": msg.chat_id,
|
||||
}
|
||||
if msg.topic_id:
|
||||
channel_source["topic_id"] = msg.topic_id
|
||||
if msg.thread_ts:
|
||||
channel_source["thread_ts"] = msg.thread_ts
|
||||
if msg.connection_id:
|
||||
channel_source["connection_id"] = msg.connection_id
|
||||
|
||||
return {"channel_source": channel_source}
|
||||
|
||||
|
||||
def _extract_text_content(content: Any) -> str:
|
||||
"""Extract text from a streaming payload content field."""
|
||||
if isinstance(content, str):
|
||||
@@ -410,6 +442,73 @@ def _format_artifact_text(artifacts: list[str]) -> str:
|
||||
_OUTPUTS_VIRTUAL_PREFIX = "/mnt/user-data/outputs/"
|
||||
|
||||
|
||||
def _unknown_command_reply(command: str | None = None) -> str:
|
||||
available = " | ".join(sorted(KNOWN_CHANNEL_COMMANDS))
|
||||
if command:
|
||||
return f"Unknown command: /{command}. Available commands: {available}"
|
||||
return f"Unknown command. Available commands: {available}"
|
||||
|
||||
|
||||
def _human_input_message(content: str, *, original_content: str | None = None) -> dict[str, Any]:
|
||||
message: dict[str, Any] = {"role": "human", "content": content}
|
||||
if original_content is not None and original_content != content:
|
||||
message["additional_kwargs"] = {ORIGINAL_USER_CONTENT_KEY: original_content}
|
||||
return message
|
||||
|
||||
|
||||
def _auth_disabled_owner_user_id() -> str | None:
|
||||
try:
|
||||
from app.gateway.auth_disabled import AUTH_DISABLED_USER_ID, is_auth_disabled
|
||||
except Exception:
|
||||
logger.debug("Unable to inspect auth-disabled mode for channel owner fallback", exc_info=True)
|
||||
return None
|
||||
return AUTH_DISABLED_USER_ID if is_auth_disabled() else None
|
||||
|
||||
|
||||
def _effective_owner_user_id(msg: InboundMessage) -> str | None:
|
||||
return _auth_disabled_owner_user_id() or msg.owner_user_id
|
||||
|
||||
|
||||
def _apply_effective_owner(msg: InboundMessage) -> InboundMessage:
|
||||
owner_user_id = _effective_owner_user_id(msg)
|
||||
if owner_user_id:
|
||||
msg.owner_user_id = owner_user_id
|
||||
return msg
|
||||
|
||||
|
||||
def _owner_headers(msg: InboundMessage) -> dict[str, str] | None:
|
||||
owner_user_id = _effective_owner_user_id(msg)
|
||||
if not owner_user_id:
|
||||
return None
|
||||
return create_internal_auth_headers(owner_user_id=owner_user_id)
|
||||
|
||||
|
||||
def _resolve_slash_skill_command(
|
||||
text: str,
|
||||
available_skills: set[str] | None = None,
|
||||
storage: SkillStorage | Callable[[], SkillStorage] | None = None,
|
||||
) -> _SlashSkillCommandResolution | None:
|
||||
reference = parse_slash_skill_reference(text)
|
||||
if reference is None:
|
||||
return None
|
||||
try:
|
||||
resolved_storage = storage() if callable(storage) else storage or get_or_new_skill_storage()
|
||||
skills = resolved_storage.load_skills(enabled_only=False)
|
||||
|
||||
skill = next((candidate for candidate in skills if candidate.name == reference.name), None)
|
||||
if skill is None:
|
||||
return None
|
||||
if not skill.enabled:
|
||||
return _SlashSkillCommandResolution(failure_message=f"Skill `/{reference.name}` is installed but disabled. Enable it before using slash activation.")
|
||||
if available_skills is not None and reference.name not in available_skills:
|
||||
return _SlashSkillCommandResolution(failure_message=f"Skill `/{reference.name}` is not available for this agent.")
|
||||
|
||||
return _SlashSkillCommandResolution(route_to_chat=True)
|
||||
except Exception as exc:
|
||||
logger.exception("[Manager] failed to resolve slash skill command")
|
||||
raise SlashSkillCommandResolutionError("Failed to resolve slash skill command. Please check the skill configuration.") from exc
|
||||
|
||||
|
||||
def _resolve_attachments(thread_id: str, artifacts: list[str]) -> list[ResolvedAttachment]:
|
||||
"""Resolve virtual artifact paths to host filesystem paths with metadata.
|
||||
|
||||
@@ -614,6 +713,7 @@ class ChannelManager:
|
||||
assistant_id: str = DEFAULT_ASSISTANT_ID,
|
||||
default_session: dict[str, Any] | None = None,
|
||||
channel_sessions: dict[str, Any] | None = None,
|
||||
connection_repo: Any | None = None,
|
||||
) -> None:
|
||||
self.bus = bus
|
||||
self.store = store
|
||||
@@ -623,7 +723,9 @@ class ChannelManager:
|
||||
self._assistant_id = assistant_id
|
||||
self._default_session = _as_dict(default_session)
|
||||
self._channel_sessions = dict(channel_sessions or {})
|
||||
self._connection_repo = connection_repo
|
||||
self._client = None # lazy init — langgraph_sdk async client
|
||||
self._skill_storage: SkillStorage | None = None
|
||||
self._csrf_token = generate_csrf_token()
|
||||
self._semaphore: asyncio.Semaphore | None = None
|
||||
self._running = False
|
||||
@@ -671,12 +773,17 @@ class ChannelManager:
|
||||
configurable["checkpoint_ns"] = ""
|
||||
configurable["thread_id"] = thread_id
|
||||
|
||||
# ``user_id`` drives user-scoped filesystem buckets that only accept
|
||||
# ``[A-Za-z0-9_-]``, so normalize the channel id and keep the raw value
|
||||
# under ``channel_user_id`` for platform-facing lookups.
|
||||
# ``user_id`` drives DeerFlow-owned memory, files, and thread buckets.
|
||||
# For browser-connected IM channels, prefer the DeerFlow account that
|
||||
# owns the connection. Preserve the raw platform user under
|
||||
# ``channel_user_id`` for platform-facing lookups and audits.
|
||||
run_context_identity: dict[str, Any] = {"thread_id": thread_id}
|
||||
if msg.user_id:
|
||||
owner_user_id = _effective_owner_user_id(msg)
|
||||
if owner_user_id:
|
||||
run_context_identity["user_id"] = make_safe_user_id(owner_user_id)
|
||||
elif msg.user_id:
|
||||
run_context_identity["user_id"] = make_safe_user_id(msg.user_id)
|
||||
if msg.user_id:
|
||||
run_context_identity["channel_user_id"] = msg.user_id
|
||||
|
||||
run_context = _merge_dicts(
|
||||
@@ -696,6 +803,21 @@ class ChannelManager:
|
||||
|
||||
return assistant_id, run_config, run_context
|
||||
|
||||
def _resolve_available_skill_names(self, msg: InboundMessage) -> set[str] | None:
|
||||
thread_id = self.store.get_thread_id(msg.channel_name, msg.chat_id, topic_id=msg.topic_id) or ""
|
||||
_, _, run_context = self._resolve_run_params(msg, thread_id)
|
||||
if run_context.get("is_bootstrap"):
|
||||
return {"bootstrap"}
|
||||
|
||||
agent_name = run_context.get("agent_name")
|
||||
if not isinstance(agent_name, str) or not agent_name.strip():
|
||||
return None
|
||||
|
||||
agent_config = load_agent_config(_normalize_custom_agent_name(agent_name))
|
||||
if agent_config and agent_config.skills is not None:
|
||||
return set(agent_config.skills)
|
||||
return None
|
||||
|
||||
# -- LangGraph SDK client (lazy) ----------------------------------------
|
||||
|
||||
def _get_client(self):
|
||||
@@ -713,6 +835,11 @@ class ChannelManager:
|
||||
)
|
||||
return self._client
|
||||
|
||||
def _get_skill_storage(self) -> SkillStorage:
|
||||
if self._skill_storage is None:
|
||||
self._skill_storage = get_or_new_skill_storage()
|
||||
return self._skill_storage
|
||||
|
||||
# -- lifecycle ---------------------------------------------------------
|
||||
|
||||
async def start(self) -> None:
|
||||
@@ -768,6 +895,7 @@ class ChannelManager:
|
||||
logger.error("[Manager] unhandled error in message task: %s", exc, exc_info=exc)
|
||||
|
||||
async def _handle_message(self, msg: InboundMessage) -> None:
|
||||
msg = _apply_effective_owner(msg)
|
||||
async with self._semaphore:
|
||||
try:
|
||||
if msg.msg_type == InboundMessageType.COMMAND:
|
||||
@@ -782,6 +910,14 @@ class ChannelManager:
|
||||
exc,
|
||||
)
|
||||
await self._send_error(msg, str(exc))
|
||||
except SlashSkillCommandResolutionError as exc:
|
||||
logger.warning(
|
||||
"Slash skill command resolution failed for %s (chat=%s): %s",
|
||||
msg.channel_name,
|
||||
msg.chat_id,
|
||||
exc,
|
||||
)
|
||||
await self._send_error(msg, str(exc))
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Error handling message from %s (chat=%s)",
|
||||
@@ -792,10 +928,27 @@ class ChannelManager:
|
||||
|
||||
# -- chat handling -----------------------------------------------------
|
||||
|
||||
async def _create_thread(self, client, msg: InboundMessage) -> str:
|
||||
"""Create a new thread through Gateway and store the mapping."""
|
||||
thread = await client.threads.create()
|
||||
thread_id = thread["thread_id"]
|
||||
async def _lookup_thread_id(self, msg: InboundMessage) -> str | None:
|
||||
if msg.connection_id and self._connection_repo is not None:
|
||||
return await self._connection_repo.get_thread_id(
|
||||
msg.connection_id,
|
||||
msg.chat_id,
|
||||
msg.topic_id,
|
||||
)
|
||||
return self.store.get_thread_id(msg.channel_name, msg.chat_id, topic_id=msg.topic_id)
|
||||
|
||||
async def _store_thread_id(self, msg: InboundMessage, thread_id: str) -> None:
|
||||
if msg.connection_id and msg.owner_user_id and self._connection_repo is not None:
|
||||
await self._connection_repo.set_thread_id(
|
||||
connection_id=msg.connection_id,
|
||||
owner_user_id=msg.owner_user_id,
|
||||
provider=msg.channel_name,
|
||||
external_conversation_id=msg.chat_id,
|
||||
external_topic_id=msg.topic_id,
|
||||
thread_id=thread_id,
|
||||
)
|
||||
return
|
||||
|
||||
self.store.set_thread_id(
|
||||
msg.channel_name,
|
||||
msg.chat_id,
|
||||
@@ -803,18 +956,40 @@ class ChannelManager:
|
||||
topic_id=msg.topic_id,
|
||||
user_id=msg.user_id,
|
||||
)
|
||||
|
||||
async def _create_thread(self, client, msg: InboundMessage) -> str:
|
||||
"""Create a new thread through Gateway and store the mapping."""
|
||||
metadata = _thread_channel_metadata(msg)
|
||||
owner_headers = _owner_headers(msg)
|
||||
if owner_headers:
|
||||
thread = await client.threads.create(metadata=metadata, headers=owner_headers)
|
||||
else:
|
||||
thread = await client.threads.create(metadata=metadata)
|
||||
thread_id = thread["thread_id"]
|
||||
await self._store_thread_id(msg, thread_id)
|
||||
logger.info("[Manager] new thread created through Gateway: thread_id=%s for chat_id=%s topic_id=%s", thread_id, msg.chat_id, msg.topic_id)
|
||||
return thread_id
|
||||
|
||||
async def _update_thread_channel_metadata(self, client, msg: InboundMessage, thread_id: str) -> None:
|
||||
"""Best-effort source metadata backfill for existing IM-created threads."""
|
||||
update_kwargs: dict[str, Any] = {"metadata": _thread_channel_metadata(msg)}
|
||||
if owner_headers := _owner_headers(msg):
|
||||
update_kwargs["headers"] = owner_headers
|
||||
try:
|
||||
await client.threads.update(thread_id, **update_kwargs)
|
||||
except Exception:
|
||||
logger.debug("[Manager] failed to update channel metadata for thread_id=%s", thread_id, exc_info=True)
|
||||
|
||||
async def _handle_chat(self, msg: InboundMessage, extra_context: dict[str, Any] | None = None) -> None:
|
||||
client = self._get_client()
|
||||
|
||||
# Look up existing DeerFlow thread.
|
||||
# topic_id may be None (e.g. Telegram private chats) — the store
|
||||
# handles this by using the "channel:chat_id" key without a topic suffix.
|
||||
thread_id = self.store.get_thread_id(msg.channel_name, msg.chat_id, topic_id=msg.topic_id)
|
||||
thread_id = await self._lookup_thread_id(msg)
|
||||
if thread_id:
|
||||
logger.info("[Manager] reusing thread: thread_id=%s for topic_id=%s", thread_id, msg.topic_id)
|
||||
await self._update_thread_channel_metadata(client, msg, thread_id)
|
||||
|
||||
# No existing thread found — create a new one
|
||||
if thread_id is None:
|
||||
@@ -836,9 +1011,11 @@ class ChannelManager:
|
||||
if extra_context:
|
||||
run_context.update(extra_context)
|
||||
|
||||
original_text = msg.text
|
||||
uploaded = await _ingest_inbound_files(thread_id, msg)
|
||||
if uploaded:
|
||||
msg.text = f"{_format_uploaded_files_block(uploaded)}\n\n{msg.text}".strip()
|
||||
human_message = _human_input_message(msg.text, original_content=original_text)
|
||||
|
||||
if self._channel_supports_streaming(msg.channel_name):
|
||||
await self._handle_streaming_chat(
|
||||
@@ -848,18 +1025,24 @@ class ChannelManager:
|
||||
assistant_id,
|
||||
run_config,
|
||||
run_context,
|
||||
human_message,
|
||||
)
|
||||
return
|
||||
|
||||
logger.info("[Manager] invoking runs.wait(thread_id=%s, text=%r)", thread_id, msg.text[:100])
|
||||
run_kwargs: dict[str, Any] = {
|
||||
"input": {"messages": [human_message]},
|
||||
"config": run_config,
|
||||
"context": run_context,
|
||||
"multitask_strategy": "reject",
|
||||
}
|
||||
if owner_headers := _owner_headers(msg):
|
||||
run_kwargs["headers"] = owner_headers
|
||||
try:
|
||||
result = await client.runs.wait(
|
||||
thread_id,
|
||||
assistant_id,
|
||||
input={"messages": [{"role": "human", "content": msg.text}]},
|
||||
config=run_config,
|
||||
context=run_context,
|
||||
multitask_strategy="reject",
|
||||
**run_kwargs,
|
||||
)
|
||||
except Exception as exc:
|
||||
if _is_thread_busy_error(exc):
|
||||
@@ -896,6 +1079,8 @@ class ChannelManager:
|
||||
artifacts=artifacts,
|
||||
attachments=attachments,
|
||||
thread_ts=msg.thread_ts,
|
||||
connection_id=msg.connection_id,
|
||||
owner_user_id=msg.owner_user_id,
|
||||
metadata=_response_metadata(msg.metadata, pending_clarification=pending_clarification),
|
||||
)
|
||||
logger.info("[Manager] publishing outbound message to bus: channel=%s, chat_id=%s", msg.channel_name, msg.chat_id)
|
||||
@@ -909,6 +1094,7 @@ class ChannelManager:
|
||||
assistant_id: str,
|
||||
run_config: dict[str, Any],
|
||||
run_context: dict[str, Any],
|
||||
human_message: dict[str, Any],
|
||||
) -> None:
|
||||
logger.info("[Manager] invoking runs.stream(thread_id=%s, text=%r)", thread_id, msg.text[:100])
|
||||
|
||||
@@ -919,16 +1105,21 @@ class ChannelManager:
|
||||
last_published_text = ""
|
||||
last_publish_at = 0.0
|
||||
stream_error: BaseException | None = None
|
||||
stream_kwargs: dict[str, Any] = {
|
||||
"input": {"messages": [human_message]},
|
||||
"config": run_config,
|
||||
"context": run_context,
|
||||
"stream_mode": ["messages-tuple", "values"],
|
||||
"multitask_strategy": "reject",
|
||||
}
|
||||
if owner_headers := _owner_headers(msg):
|
||||
stream_kwargs["headers"] = owner_headers
|
||||
|
||||
try:
|
||||
async for chunk in client.runs.stream(
|
||||
thread_id,
|
||||
assistant_id,
|
||||
input={"messages": [{"role": "human", "content": msg.text}]},
|
||||
config=run_config,
|
||||
context=run_context,
|
||||
stream_mode=["messages-tuple", "values"],
|
||||
multitask_strategy="reject",
|
||||
**stream_kwargs,
|
||||
):
|
||||
event = getattr(chunk, "event", "")
|
||||
data = getattr(chunk, "data", None)
|
||||
@@ -958,6 +1149,8 @@ class ChannelManager:
|
||||
text=latest_text,
|
||||
is_final=False,
|
||||
thread_ts=msg.thread_ts,
|
||||
connection_id=msg.connection_id,
|
||||
owner_user_id=msg.owner_user_id,
|
||||
metadata=_response_metadata(msg.metadata),
|
||||
)
|
||||
)
|
||||
@@ -1004,6 +1197,8 @@ class ChannelManager:
|
||||
attachments=attachments,
|
||||
is_final=True,
|
||||
thread_ts=msg.thread_ts,
|
||||
connection_id=msg.connection_id,
|
||||
owner_user_id=msg.owner_user_id,
|
||||
metadata=_response_metadata(msg.metadata, pending_clarification=pending_clarification),
|
||||
)
|
||||
)
|
||||
@@ -1011,11 +1206,20 @@ class ChannelManager:
|
||||
# -- command handling --------------------------------------------------
|
||||
|
||||
async def _handle_command(self, msg: InboundMessage) -> None:
|
||||
text = msg.text.strip()
|
||||
raw_text = msg.text
|
||||
text = raw_text.strip()
|
||||
parts = text.split(maxsplit=1)
|
||||
command = parts[0].lower().lstrip("/")
|
||||
reply: str | None = None
|
||||
if not parts:
|
||||
command = None
|
||||
reply = _unknown_command_reply()
|
||||
else:
|
||||
command = parts[0].lower().removeprefix("/")
|
||||
|
||||
if command == "bootstrap":
|
||||
if reply is None and not raw_text.startswith("/"):
|
||||
reply = _unknown_command_reply(command)
|
||||
|
||||
if reply is None and command == "bootstrap":
|
||||
from dataclasses import replace as _dc_replace
|
||||
|
||||
chat_text = parts[1] if len(parts) > 1 else "Initialize workspace"
|
||||
@@ -1023,27 +1227,19 @@ class ChannelManager:
|
||||
await self._handle_chat(chat_msg, extra_context={"is_bootstrap": True})
|
||||
return
|
||||
|
||||
if command == "new":
|
||||
if reply is None and command == "new":
|
||||
# Create a new thread through Gateway
|
||||
client = self._get_client()
|
||||
thread = await client.threads.create()
|
||||
new_thread_id = thread["thread_id"]
|
||||
self.store.set_thread_id(
|
||||
msg.channel_name,
|
||||
msg.chat_id,
|
||||
new_thread_id,
|
||||
topic_id=msg.topic_id,
|
||||
user_id=msg.user_id,
|
||||
)
|
||||
await self._create_thread(client, msg)
|
||||
reply = "New conversation started."
|
||||
elif command == "status":
|
||||
thread_id = self.store.get_thread_id(msg.channel_name, msg.chat_id, topic_id=msg.topic_id)
|
||||
elif reply is None and command == "status":
|
||||
thread_id = await self._lookup_thread_id(msg)
|
||||
reply = f"Active thread: {thread_id}" if thread_id else "No active conversation."
|
||||
elif command == "models":
|
||||
elif reply is None and command == "models":
|
||||
reply = await self._fetch_gateway("/api/models", "models")
|
||||
elif command == "memory":
|
||||
elif reply is None and command == "memory":
|
||||
reply = await self._fetch_gateway("/api/memory", "memory")
|
||||
elif command == "help":
|
||||
elif reply is None and command == "help":
|
||||
reply = (
|
||||
"Available commands:\n"
|
||||
"/bootstrap — Start a bootstrap session (enables agent setup)\n"
|
||||
@@ -1051,18 +1247,36 @@ class ChannelManager:
|
||||
"/status — Show current thread info\n"
|
||||
"/models — List available models\n"
|
||||
"/memory — Show memory status\n"
|
||||
"/<skill-name> <task> — Activate an enabled skill for one turn\n"
|
||||
"/help — Show this help"
|
||||
)
|
||||
elif reply is None:
|
||||
slash_resolution = await asyncio.to_thread(
|
||||
lambda: _resolve_slash_skill_command(
|
||||
raw_text,
|
||||
self._resolve_available_skill_names(msg),
|
||||
self._get_skill_storage,
|
||||
)
|
||||
)
|
||||
if slash_resolution and slash_resolution.failure_message:
|
||||
reply = slash_resolution.failure_message
|
||||
elif slash_resolution and slash_resolution.route_to_chat:
|
||||
from dataclasses import replace as _dc_replace
|
||||
|
||||
chat_msg = _dc_replace(msg, msg_type=InboundMessageType.CHAT)
|
||||
await self._handle_chat(chat_msg)
|
||||
return
|
||||
else:
|
||||
available = " | ".join(sorted(KNOWN_CHANNEL_COMMANDS))
|
||||
reply = f"Unknown command: /{command}. Available commands: {available}"
|
||||
reply = _unknown_command_reply(command)
|
||||
|
||||
outbound = OutboundMessage(
|
||||
channel_name=msg.channel_name,
|
||||
chat_id=msg.chat_id,
|
||||
thread_id=self.store.get_thread_id(msg.channel_name, msg.chat_id) or "",
|
||||
thread_id=await self._lookup_thread_id(msg) or "",
|
||||
text=reply,
|
||||
thread_ts=msg.thread_ts,
|
||||
connection_id=msg.connection_id,
|
||||
owner_user_id=msg.owner_user_id,
|
||||
metadata=_slim_metadata(msg.metadata),
|
||||
)
|
||||
await self.bus.publish_outbound(outbound)
|
||||
@@ -1098,9 +1312,11 @@ class ChannelManager:
|
||||
outbound = OutboundMessage(
|
||||
channel_name=msg.channel_name,
|
||||
chat_id=msg.chat_id,
|
||||
thread_id=self.store.get_thread_id(msg.channel_name, msg.chat_id) or "",
|
||||
thread_id=await self._lookup_thread_id(msg) or "",
|
||||
text=error_text,
|
||||
thread_ts=msg.thread_ts,
|
||||
connection_id=msg.connection_id,
|
||||
owner_user_id=msg.owner_user_id,
|
||||
metadata=_slim_metadata(msg.metadata),
|
||||
)
|
||||
await self.bus.publish_outbound(outbound)
|
||||
|
||||
@@ -44,6 +44,12 @@ class InboundMessage:
|
||||
Messages sharing the same ``topic_id`` within a ``chat_id`` will
|
||||
reuse the same DeerFlow thread. When ``None``, each message
|
||||
creates a new thread (one-shot Q&A).
|
||||
connection_id: Optional DeerFlow channel connection id. When present,
|
||||
conversation mapping is scoped by the connection instead of the
|
||||
legacy global ``channel_name:chat_id[:topic_id]`` key.
|
||||
owner_user_id: DeerFlow user id that owns the channel connection.
|
||||
Platform user ids stay in ``user_id``.
|
||||
workspace_id: Optional external workspace/guild/team id.
|
||||
files: Optional list of file attachments (platform-specific dicts).
|
||||
metadata: Arbitrary extra data from the channel.
|
||||
created_at: Unix timestamp when the message was created.
|
||||
@@ -56,6 +62,9 @@ class InboundMessage:
|
||||
msg_type: InboundMessageType = InboundMessageType.CHAT
|
||||
thread_ts: str | None = None
|
||||
topic_id: str | None = None
|
||||
connection_id: str | None = None
|
||||
owner_user_id: str | None = None
|
||||
workspace_id: str | None = None
|
||||
files: list[dict[str, Any]] = field(default_factory=list)
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
created_at: float = field(default_factory=time.time)
|
||||
@@ -95,6 +104,9 @@ class OutboundMessage:
|
||||
is_final: Whether this is the final message in the response stream.
|
||||
thread_ts: Optional platform thread identifier for threaded replies.
|
||||
metadata: Arbitrary extra data.
|
||||
connection_id: Optional DeerFlow channel connection id used for
|
||||
connection-specific outbound credentials.
|
||||
owner_user_id: DeerFlow user id that owns the channel connection.
|
||||
created_at: Unix timestamp.
|
||||
"""
|
||||
|
||||
@@ -106,6 +118,8 @@ class OutboundMessage:
|
||||
attachments: list[ResolvedAttachment] = field(default_factory=list)
|
||||
is_final: bool = True
|
||||
thread_ts: str | None = None
|
||||
connection_id: str | None = None
|
||||
owner_user_id: str | None = None
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
created_at: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Local persistence for runtime IM channel configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import tempfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ChannelRuntimeConfigStore:
|
||||
"""JSON-backed store for channel credentials entered from the UI.
|
||||
|
||||
This intentionally mirrors ``ChannelStore``: local/private deployments get
|
||||
durable runtime configuration without needing a public callback URL or a
|
||||
config.yaml edit.
|
||||
"""
|
||||
|
||||
def __init__(self, path: str | Path | None = None) -> None:
|
||||
if path is None:
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
path = Path(get_paths().base_dir) / "channels" / "runtime-config.json"
|
||||
self._path = Path(path)
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._data: dict[str, dict[str, Any]] = self._load()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _load(self) -> dict[str, dict[str, Any]]:
|
||||
if self._path.exists():
|
||||
try:
|
||||
raw = json.loads(self._path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
logger.warning("Corrupt channel runtime config store at %s, starting fresh", self._path)
|
||||
return {}
|
||||
if isinstance(raw, dict):
|
||||
return {str(name): dict(value) for name, value in raw.items() if isinstance(value, dict)}
|
||||
return {}
|
||||
|
||||
def _save(self) -> None:
|
||||
fd = tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
dir=self._path.parent,
|
||||
suffix=".tmp",
|
||||
delete=False,
|
||||
)
|
||||
try:
|
||||
json.dump(self._data, fd, indent=2, ensure_ascii=False)
|
||||
fd.close()
|
||||
Path(fd.name).replace(self._path)
|
||||
try:
|
||||
self._path.chmod(0o600)
|
||||
except OSError:
|
||||
logger.debug("Unable to chmod channel runtime config store at %s", self._path, exc_info=True)
|
||||
except BaseException:
|
||||
fd.close()
|
||||
Path(fd.name).unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
def load_all(self) -> dict[str, dict[str, Any]]:
|
||||
with self._lock:
|
||||
return {name: dict(config) for name, config in self._data.items()}
|
||||
|
||||
def get_provider_config(self, provider: str) -> dict[str, Any] | None:
|
||||
with self._lock:
|
||||
config = self._data.get(provider)
|
||||
return dict(config) if isinstance(config, dict) else None
|
||||
|
||||
def set_provider_config(self, provider: str, config: dict[str, Any]) -> None:
|
||||
with self._lock:
|
||||
self._data[provider] = dict(config)
|
||||
self._save()
|
||||
|
||||
def remove_provider_config(self, provider: str) -> bool:
|
||||
with self._lock:
|
||||
if provider not in self._data:
|
||||
return False
|
||||
del self._data[provider]
|
||||
self._save()
|
||||
return True
|
||||
|
||||
|
||||
def _provider_enabled(channel_connections_config: Any, provider: str) -> bool:
|
||||
provider_config = getattr(channel_connections_config, provider, None)
|
||||
return bool(getattr(provider_config, "enabled", False))
|
||||
|
||||
|
||||
def merge_runtime_channel_configs(
|
||||
channels_config: dict[str, Any],
|
||||
channel_connections_config: Any,
|
||||
*,
|
||||
store: ChannelRuntimeConfigStore | None = None,
|
||||
) -> None:
|
||||
"""Merge persisted runtime provider config into ``channels_config`` in-place."""
|
||||
if channel_connections_config is None or not getattr(channel_connections_config, "enabled", False):
|
||||
return
|
||||
|
||||
runtime_store = store or ChannelRuntimeConfigStore()
|
||||
for provider, runtime_config in runtime_store.load_all().items():
|
||||
if not _provider_enabled(channel_connections_config, provider):
|
||||
continue
|
||||
existing = channels_config.get(provider)
|
||||
merged = dict(runtime_config)
|
||||
if isinstance(existing, dict):
|
||||
merged.update(existing)
|
||||
channels_config[provider] = merged
|
||||
|
||||
|
||||
def apply_runtime_connection_config(
|
||||
channel_connections_config: Any,
|
||||
*,
|
||||
store: ChannelRuntimeConfigStore | None = None,
|
||||
) -> Any:
|
||||
"""Apply persisted connection metadata that lives outside ``channels``.
|
||||
|
||||
Telegram uses a bot username for deep links; UI-entered values are stored
|
||||
with the runtime channel config so local restarts keep the provider
|
||||
configured.
|
||||
"""
|
||||
if channel_connections_config is None or not getattr(channel_connections_config, "enabled", False):
|
||||
return channel_connections_config
|
||||
|
||||
runtime_store = store or ChannelRuntimeConfigStore()
|
||||
telegram_runtime_config = runtime_store.get_provider_config("telegram")
|
||||
bot_username = ""
|
||||
if isinstance(telegram_runtime_config, dict):
|
||||
bot_username = str(telegram_runtime_config.get("bot_username") or "").strip()
|
||||
if not bot_username or not _provider_enabled(channel_connections_config, "telegram"):
|
||||
return channel_connections_config
|
||||
|
||||
config = channel_connections_config.model_copy(deep=True)
|
||||
config.telegram.bot_username = bot_username
|
||||
return config
|
||||
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any
|
||||
from app.channels.base import Channel
|
||||
from app.channels.manager import DEFAULT_GATEWAY_URL, DEFAULT_LANGGRAPH_URL, ChannelManager
|
||||
from app.channels.message_bus import MessageBus
|
||||
from app.channels.runtime_config_store import merge_runtime_channel_configs
|
||||
from app.channels.store import ChannelStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -52,6 +53,30 @@ def _resolve_service_url(config: dict[str, Any], config_key: str, env_key: str,
|
||||
return default
|
||||
|
||||
|
||||
def _merge_channel_connection_runtime_config(channels_config: dict[str, Any], app_config: AppConfig) -> None:
|
||||
connection_config = getattr(app_config, "channel_connections", None)
|
||||
merge_runtime_channel_configs(channels_config, connection_config)
|
||||
|
||||
|
||||
def _make_connection_repo(app_config: AppConfig):
|
||||
connection_config = getattr(app_config, "channel_connections", None)
|
||||
if connection_config is None or not getattr(connection_config, "enabled", False):
|
||||
return None
|
||||
|
||||
try:
|
||||
from deerflow.persistence.channel_connections import ChannelConnectionRepository
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
except Exception:
|
||||
logger.exception("Failed to import channel connection repository")
|
||||
return None
|
||||
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
logger.warning("Channel connections are enabled but database persistence is not available")
|
||||
return None
|
||||
return ChannelConnectionRepository(session_factory)
|
||||
|
||||
|
||||
class ChannelService:
|
||||
"""Manages the lifecycle of all configured IM channels.
|
||||
|
||||
@@ -59,9 +84,10 @@ class ChannelService:
|
||||
instantiates enabled channels, and starts the ChannelManager dispatcher.
|
||||
"""
|
||||
|
||||
def __init__(self, channels_config: dict[str, Any] | None = None) -> None:
|
||||
def __init__(self, channels_config: dict[str, Any] | None = None, *, connection_repo: Any | None = None) -> None:
|
||||
self.bus = MessageBus()
|
||||
self.store = ChannelStore()
|
||||
self._connection_repo = connection_repo
|
||||
config = dict(channels_config or {})
|
||||
langgraph_url = _resolve_service_url(config, "langgraph_url", _CHANNELS_LANGGRAPH_URL_ENV, DEFAULT_LANGGRAPH_URL)
|
||||
gateway_url = _resolve_service_url(config, "gateway_url", _CHANNELS_GATEWAY_URL_ENV, DEFAULT_GATEWAY_URL)
|
||||
@@ -74,6 +100,7 @@ class ChannelService:
|
||||
gateway_url=gateway_url,
|
||||
default_session=default_session if isinstance(default_session, dict) else None,
|
||||
channel_sessions=channel_sessions,
|
||||
connection_repo=connection_repo,
|
||||
)
|
||||
self._channels: dict[str, Any] = {} # name -> Channel instance
|
||||
self._config = config
|
||||
@@ -90,8 +117,9 @@ class ChannelService:
|
||||
# extra fields are allowed by AppConfig (extra="allow")
|
||||
extra = app_config.model_extra or {}
|
||||
if "channels" in extra:
|
||||
channels_config = extra["channels"]
|
||||
return cls(channels_config=channels_config)
|
||||
channels_config = dict(extra["channels"] or {})
|
||||
_merge_channel_connection_runtime_config(channels_config, app_config)
|
||||
return cls(channels_config=channels_config, connection_repo=_make_connection_repo(app_config))
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Start the manager and all enabled channels."""
|
||||
@@ -151,6 +179,27 @@ class ChannelService:
|
||||
|
||||
return await self._start_channel(name, config)
|
||||
|
||||
async def configure_channel(self, name: str, config: dict[str, Any]) -> bool:
|
||||
"""Apply runtime config for a channel and restart it if the service is running."""
|
||||
self._config[name] = dict(config)
|
||||
if not self._running:
|
||||
return True
|
||||
return await self.restart_channel(name)
|
||||
|
||||
async def remove_channel(self, name: str) -> bool:
|
||||
"""Remove runtime config for a channel and stop it if currently running."""
|
||||
self._config.pop(name, None)
|
||||
channel = self._channels.pop(name, None)
|
||||
if channel is None:
|
||||
return True
|
||||
try:
|
||||
await channel.stop()
|
||||
logger.info("Channel %s stopped and removed", name)
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("Error stopping channel %s for removal", name)
|
||||
return False
|
||||
|
||||
async def _start_channel(self, name: str, config: dict[str, Any]) -> bool:
|
||||
"""Instantiate and start a single channel."""
|
||||
import_path = _CHANNEL_REGISTRY.get(name)
|
||||
@@ -169,6 +218,8 @@ class ChannelService:
|
||||
try:
|
||||
config = dict(config)
|
||||
config["channel_store"] = self.store
|
||||
if self._connection_repo is not None:
|
||||
config["connection_repo"] = self._connection_repo
|
||||
channel = channel_cls(bus=self.bus, config=config)
|
||||
self._channels[name] = channel
|
||||
await channel.start()
|
||||
|
||||
+164
-14
@@ -9,6 +9,8 @@ from typing import Any
|
||||
from markdown_to_mrkdwn import SlackMarkdownConverter
|
||||
|
||||
from app.channels.base import Channel
|
||||
from app.channels.commands import extract_connect_code, is_known_channel_command
|
||||
from app.channels.connection_identity import attach_connection_identity
|
||||
from app.channels.message_bus import InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -32,6 +34,20 @@ def _normalize_allowed_users(allowed_users: Any) -> set[str]:
|
||||
return {str(user_id) for user_id in values if str(user_id)}
|
||||
|
||||
|
||||
def _strip_leading_slack_bot_mention(text: str, bot_user_id: str | None) -> str:
|
||||
if not bot_user_id:
|
||||
return text
|
||||
if not text.startswith("<@"):
|
||||
return text
|
||||
end = text.find(">")
|
||||
if end <= 2:
|
||||
return text
|
||||
mentioned_user_id = text[2:end].split("|", 1)[0].lstrip("!")
|
||||
if mentioned_user_id != bot_user_id:
|
||||
return text
|
||||
return text[end + 1 :].lstrip()
|
||||
|
||||
|
||||
class SlackChannel(Channel):
|
||||
"""Slack IM channel using Socket Mode (WebSocket, no public IP).
|
||||
|
||||
@@ -49,6 +65,10 @@ class SlackChannel(Channel):
|
||||
self._web_client = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
self._allowed_users = _normalize_allowed_users(config.get("allowed_users", []))
|
||||
self._connection_repo = config.get("connection_repo")
|
||||
self._web_client_factory = config.get("web_client_factory")
|
||||
configured_bot_user_id = config.get("bot_user_id")
|
||||
self._bot_user_id = str(configured_bot_user_id).lstrip("@") if configured_bot_user_id else None
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._running:
|
||||
@@ -63,15 +83,28 @@ class SlackChannel(Channel):
|
||||
return
|
||||
|
||||
self._SocketModeResponse = SocketModeResponse
|
||||
if self._web_client_factory is None:
|
||||
self._web_client_factory = WebClient
|
||||
|
||||
bot_token = self.config.get("bot_token", "")
|
||||
app_token = self.config.get("app_token", "")
|
||||
|
||||
if self._connection_repo is not None and self.config.get("event_delivery") == "http":
|
||||
if not bot_token:
|
||||
logger.error("Slack HTTP Events mode requires bot_token")
|
||||
return
|
||||
await self._initialize_operator_web_client(str(bot_token))
|
||||
self._loop = asyncio.get_event_loop()
|
||||
self._running = True
|
||||
self.bus.subscribe_outbound(self._on_outbound)
|
||||
logger.info("Slack channel started in HTTP Events mode")
|
||||
return
|
||||
|
||||
if not bot_token or not app_token:
|
||||
logger.error("Slack channel requires bot_token and app_token")
|
||||
return
|
||||
|
||||
self._web_client = WebClient(token=bot_token)
|
||||
await self._initialize_operator_web_client(str(bot_token))
|
||||
self._socket_client = SocketModeClient(
|
||||
app_token=app_token,
|
||||
web_client=self._web_client,
|
||||
@@ -96,7 +129,8 @@ class SlackChannel(Channel):
|
||||
logger.info("Slack channel stopped")
|
||||
|
||||
async def send(self, msg: OutboundMessage, *, _max_retries: int = 3) -> None:
|
||||
if not self._web_client:
|
||||
web_client = await self._get_web_client_for_message(msg)
|
||||
if not web_client:
|
||||
return
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
@@ -109,11 +143,12 @@ class SlackChannel(Channel):
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(_max_retries):
|
||||
try:
|
||||
await asyncio.to_thread(self._web_client.chat_postMessage, **kwargs)
|
||||
await asyncio.to_thread(web_client.chat_postMessage, **kwargs)
|
||||
# Add a completion reaction to the thread root
|
||||
if msg.thread_ts:
|
||||
await asyncio.to_thread(
|
||||
self._add_reaction,
|
||||
self._add_reaction_with_client,
|
||||
web_client,
|
||||
msg.chat_id,
|
||||
msg.thread_ts,
|
||||
"white_check_mark",
|
||||
@@ -137,7 +172,8 @@ class SlackChannel(Channel):
|
||||
if msg.thread_ts:
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
self._add_reaction,
|
||||
self._add_reaction_with_client,
|
||||
web_client,
|
||||
msg.chat_id,
|
||||
msg.thread_ts,
|
||||
"x",
|
||||
@@ -149,7 +185,8 @@ class SlackChannel(Channel):
|
||||
raise last_exc
|
||||
|
||||
async def send_file(self, msg: OutboundMessage, attachment: ResolvedAttachment) -> bool:
|
||||
if not self._web_client:
|
||||
web_client = await self._get_web_client_for_message(msg)
|
||||
if not web_client:
|
||||
return False
|
||||
|
||||
try:
|
||||
@@ -162,7 +199,7 @@ class SlackChannel(Channel):
|
||||
if msg.thread_ts:
|
||||
kwargs["thread_ts"] = msg.thread_ts
|
||||
|
||||
await asyncio.to_thread(self._web_client.files_upload_v2, **kwargs)
|
||||
await asyncio.to_thread(web_client.files_upload_v2, **kwargs)
|
||||
logger.info("[Slack] file uploaded: %s to channel=%s", attachment.filename, msg.chat_id)
|
||||
return True
|
||||
except Exception:
|
||||
@@ -171,12 +208,38 @@ class SlackChannel(Channel):
|
||||
|
||||
# -- internal ----------------------------------------------------------
|
||||
|
||||
def _add_reaction(self, channel_id: str, timestamp: str, emoji: str) -> None:
|
||||
"""Add an emoji reaction to a message (best-effort, non-blocking)."""
|
||||
if not self._web_client:
|
||||
async def _initialize_operator_web_client(self, bot_token: str) -> None:
|
||||
self._web_client = self._web_client_factory(token=bot_token)
|
||||
if self._bot_user_id is not None:
|
||||
return
|
||||
try:
|
||||
self._web_client.reactions_add(
|
||||
auth_info = await asyncio.to_thread(self._web_client.auth_test)
|
||||
user_id = auth_info.get("user_id") if isinstance(auth_info, dict) else None
|
||||
if user_id is None:
|
||||
auth_get = getattr(auth_info, "get", None)
|
||||
user_id = auth_get("user_id") if callable(auth_get) else None
|
||||
if isinstance(user_id, str) and user_id:
|
||||
self._bot_user_id = user_id
|
||||
except Exception:
|
||||
logger.warning("[Slack] failed to resolve bot user id; app mention text may include the bot mention", exc_info=True)
|
||||
|
||||
async def _get_web_client_for_message(self, msg: OutboundMessage):
|
||||
if msg.connection_id and self._connection_repo is not None:
|
||||
credentials = await self._connection_repo.get_credentials(msg.connection_id)
|
||||
access_token = credentials.get("access_token") if credentials else None
|
||||
if not access_token:
|
||||
return self._web_client
|
||||
if self._web_client_factory is None:
|
||||
from slack_sdk import WebClient
|
||||
|
||||
self._web_client_factory = WebClient
|
||||
return self._web_client_factory(token=access_token)
|
||||
return self._web_client
|
||||
|
||||
@staticmethod
|
||||
def _add_reaction_with_client(web_client, channel_id: str, timestamp: str, emoji: str) -> None:
|
||||
try:
|
||||
web_client.reactions_add(
|
||||
channel=channel_id,
|
||||
timestamp=timestamp,
|
||||
name=emoji,
|
||||
@@ -185,6 +248,12 @@ class SlackChannel(Channel):
|
||||
if "already_reacted" not in str(exc):
|
||||
logger.warning("[Slack] failed to add reaction %s: %s", emoji, exc)
|
||||
|
||||
def _add_reaction(self, channel_id: str, timestamp: str, emoji: str) -> None:
|
||||
"""Add an emoji reaction to a message (best-effort, non-blocking)."""
|
||||
if not self._web_client:
|
||||
return
|
||||
self._add_reaction_with_client(self._web_client, channel_id, timestamp, emoji)
|
||||
|
||||
def _send_running_reply(self, channel_id: str, thread_ts: str) -> None:
|
||||
"""Send a 'Working on it......' reply in the thread (called from SDK thread)."""
|
||||
if not self._web_client:
|
||||
@@ -210,17 +279,26 @@ class SlackChannel(Channel):
|
||||
if event_type != "events_api":
|
||||
return
|
||||
|
||||
if self._bot_user_id is None:
|
||||
authorization = next((item for item in req.payload.get("authorizations", []) if isinstance(item, dict)), None)
|
||||
user_id = authorization.get("user_id") if authorization else None
|
||||
if isinstance(user_id, str) and user_id:
|
||||
self._bot_user_id = user_id
|
||||
|
||||
event = req.payload.get("event", {})
|
||||
etype = event.get("type", "")
|
||||
|
||||
# Handle message events (DM or @mention)
|
||||
if etype in ("message", "app_mention"):
|
||||
self._handle_message_event(event)
|
||||
self._handle_message_event(
|
||||
event,
|
||||
team_id=req.payload.get("team_id") or req.payload.get("team") or event.get("team"),
|
||||
)
|
||||
|
||||
except Exception:
|
||||
logger.exception("Error processing Slack event")
|
||||
|
||||
def _handle_message_event(self, event: dict) -> None:
|
||||
def _handle_message_event(self, event: dict, *, team_id: str | None = None) -> None:
|
||||
# Ignore bot messages
|
||||
if event.get("bot_id") or event.get("subtype"):
|
||||
return
|
||||
@@ -233,13 +311,28 @@ class SlackChannel(Channel):
|
||||
return
|
||||
|
||||
text = event.get("text", "").strip()
|
||||
if event.get("type") == "app_mention":
|
||||
text = _strip_leading_slack_bot_mention(text, self._bot_user_id)
|
||||
if not text:
|
||||
return
|
||||
|
||||
connect_code = extract_connect_code(text)
|
||||
if connect_code:
|
||||
if self._loop and self._loop.is_running():
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._bind_connection_from_connect_code(
|
||||
event=event,
|
||||
team_id=str(team_id or event.get("team") or ""),
|
||||
code=connect_code,
|
||||
),
|
||||
self._loop,
|
||||
)
|
||||
return
|
||||
|
||||
channel_id = event.get("channel", "")
|
||||
thread_ts = event.get("thread_ts") or event.get("ts", "")
|
||||
|
||||
if text.startswith("/"):
|
||||
if is_known_channel_command(text):
|
||||
msg_type = InboundMessageType.COMMAND
|
||||
else:
|
||||
msg_type = InboundMessageType.CHAT
|
||||
@@ -261,4 +354,61 @@ class SlackChannel(Channel):
|
||||
self._add_reaction(channel_id, event.get("ts", thread_ts), "eyes")
|
||||
# Send "running" reply first (fire-and-forget from SDK thread)
|
||||
self._send_running_reply(channel_id, thread_ts)
|
||||
if self._connection_repo is None:
|
||||
asyncio.run_coroutine_threadsafe(self.bus.publish_inbound(inbound), self._loop)
|
||||
else:
|
||||
asyncio.run_coroutine_threadsafe(self._publish_inbound_with_connection(inbound, team_id=team_id), self._loop)
|
||||
|
||||
async def _publish_inbound_with_connection(self, inbound, *, team_id: str | None = None) -> None:
|
||||
inbound = await self._attach_connection_identity(inbound, team_id=team_id)
|
||||
await self.bus.publish_inbound(inbound)
|
||||
|
||||
async def _attach_connection_identity(self, inbound, *, team_id: str | None = None):
|
||||
workspace_id = str(team_id or inbound.metadata.get("team_id") or "")
|
||||
return await attach_connection_identity(
|
||||
inbound,
|
||||
repo=self._connection_repo,
|
||||
provider="slack",
|
||||
workspace_id=workspace_id,
|
||||
)
|
||||
|
||||
async def _bind_connection_from_connect_code(self, *, event: dict, team_id: str, code: str) -> bool:
|
||||
if self._connection_repo is None or not code:
|
||||
return False
|
||||
|
||||
channel_id = str(event.get("channel") or "")
|
||||
thread_ts = str(event.get("thread_ts") or event.get("ts") or "")
|
||||
state = await self._connection_repo.consume_oauth_state(provider="slack", state=code)
|
||||
if state is None:
|
||||
self._post_connection_reply(channel_id, "Slack connection code is invalid or expired.", thread_ts)
|
||||
return True
|
||||
|
||||
user_id = str(event.get("user") or "")
|
||||
if not user_id or not team_id:
|
||||
self._post_connection_reply(channel_id, "Slack connection could not be completed from this message.", thread_ts)
|
||||
return True
|
||||
|
||||
await self._connection_repo.upsert_connection(
|
||||
owner_user_id=state["owner_user_id"],
|
||||
provider="slack",
|
||||
external_account_id=user_id,
|
||||
workspace_id=team_id,
|
||||
metadata={
|
||||
"team_id": team_id,
|
||||
"channel_id": channel_id,
|
||||
},
|
||||
status="connected",
|
||||
)
|
||||
self._post_connection_reply(channel_id, "Slack connected to DeerFlow.", thread_ts)
|
||||
return True
|
||||
|
||||
def _post_connection_reply(self, channel_id: str, text: str, thread_ts: str | None = None) -> None:
|
||||
if not self._web_client or not channel_id:
|
||||
return
|
||||
kwargs: dict[str, Any] = {"channel": channel_id, "text": text}
|
||||
if thread_ts:
|
||||
kwargs["thread_ts"] = thread_ts
|
||||
try:
|
||||
self._web_client.chat_postMessage(**kwargs)
|
||||
except Exception:
|
||||
logger.exception("[Slack] failed to send connection reply in channel=%s", channel_id)
|
||||
|
||||
@@ -8,6 +8,7 @@ import threading
|
||||
from typing import Any
|
||||
|
||||
from app.channels.base import Channel
|
||||
from app.channels.connection_identity import attach_connection_identity
|
||||
from app.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -35,6 +36,7 @@ class TelegramChannel(Channel):
|
||||
pass
|
||||
# chat_id -> last sent message_id for threaded replies
|
||||
self._last_bot_message: dict[str, int] = {}
|
||||
self._connection_repo = config.get("connection_repo")
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._running:
|
||||
@@ -60,12 +62,17 @@ class TelegramChannel(Channel):
|
||||
|
||||
# Command handlers
|
||||
app.add_handler(CommandHandler("start", self._cmd_start))
|
||||
app.add_handler(CommandHandler("bootstrap", self._cmd_generic))
|
||||
app.add_handler(CommandHandler("new", self._cmd_generic))
|
||||
app.add_handler(CommandHandler("status", self._cmd_generic))
|
||||
app.add_handler(CommandHandler("models", self._cmd_generic))
|
||||
app.add_handler(CommandHandler("memory", self._cmd_generic))
|
||||
app.add_handler(CommandHandler("help", self._cmd_generic))
|
||||
|
||||
# Slash skill commands are dynamic and cannot all be pre-registered
|
||||
# with Telegram, so route unknown slash commands through chat handling.
|
||||
app.add_handler(MessageHandler(filters.TEXT & filters.COMMAND, self._on_text))
|
||||
|
||||
# General message handler
|
||||
app.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, self._on_text))
|
||||
|
||||
@@ -171,6 +178,26 @@ class TelegramChannel(Channel):
|
||||
logger.exception("[Telegram] failed to send file: %s", attachment.filename)
|
||||
return False
|
||||
|
||||
async def process_webhook_update(self, payload: dict[str, Any]) -> bool:
|
||||
if not self._application:
|
||||
return False
|
||||
try:
|
||||
from telegram import Update
|
||||
except ImportError:
|
||||
logger.error("python-telegram-bot is not installed. Install it with: uv add python-telegram-bot")
|
||||
return False
|
||||
|
||||
update = Update.de_json(payload, self._application.bot)
|
||||
if update is None:
|
||||
return False
|
||||
|
||||
if self._tg_loop and self._tg_loop.is_running():
|
||||
future = asyncio.run_coroutine_threadsafe(self._application.process_update(update), self._tg_loop)
|
||||
await asyncio.wrap_future(future)
|
||||
else:
|
||||
await self._application.process_update(update)
|
||||
return True
|
||||
|
||||
# -- helpers -----------------------------------------------------------
|
||||
|
||||
async def _send_running_reply(self, chat_id: str, reply_to_message_id: int) -> None:
|
||||
@@ -228,10 +255,90 @@ class TelegramChannel(Channel):
|
||||
return True
|
||||
return user_id in self._allowed_users
|
||||
|
||||
@staticmethod
|
||||
def _telegram_display_name(user) -> str:
|
||||
full_name = getattr(user, "full_name", None)
|
||||
if isinstance(full_name, str) and full_name:
|
||||
return full_name
|
||||
username = getattr(user, "username", None)
|
||||
if isinstance(username, str) and username:
|
||||
return username
|
||||
return str(getattr(user, "id", ""))
|
||||
|
||||
async def _bind_connection_from_start_token(self, update, state_token: str) -> bool:
|
||||
if self._connection_repo is None or not state_token:
|
||||
return False
|
||||
|
||||
state = await self._connection_repo.consume_oauth_state(provider="telegram", state=state_token)
|
||||
if state is None:
|
||||
await update.message.reply_text("Telegram connection link is invalid or expired.")
|
||||
return True
|
||||
|
||||
owner_user_id = state["owner_user_id"]
|
||||
user_id = str(update.effective_user.id)
|
||||
chat_id = str(update.effective_chat.id)
|
||||
connection = await self._connection_repo.upsert_connection(
|
||||
owner_user_id=owner_user_id,
|
||||
provider="telegram",
|
||||
external_account_id=user_id,
|
||||
external_account_name=self._telegram_display_name(update.effective_user),
|
||||
workspace_id=chat_id,
|
||||
workspace_name=None,
|
||||
metadata={
|
||||
"chat_id": chat_id,
|
||||
"chat_type": update.effective_chat.type,
|
||||
"telegram_username": getattr(update.effective_user, "username", None),
|
||||
},
|
||||
status="connected",
|
||||
)
|
||||
logger.info("[Telegram] bound chat=%s user=%s to DeerFlow user=%s connection=%s", chat_id, user_id, owner_user_id, connection["id"])
|
||||
await update.message.reply_text("Telegram connected to DeerFlow.")
|
||||
return True
|
||||
|
||||
async def _attach_connection_identity(self, inbound: InboundMessage) -> InboundMessage:
|
||||
return await attach_connection_identity(
|
||||
inbound,
|
||||
repo=self._connection_repo,
|
||||
provider="telegram",
|
||||
workspace_id=inbound.chat_id,
|
||||
)
|
||||
|
||||
def _get_bot_username(self, context) -> str | None:
|
||||
bot = getattr(context, "bot", None)
|
||||
username = getattr(bot, "username", None)
|
||||
if not username and self._application is not None:
|
||||
username = getattr(getattr(self._application, "bot", None), "username", None)
|
||||
return str(username) if username else None
|
||||
|
||||
@staticmethod
|
||||
def _strip_bot_username_from_leading_command(text: str, bot_username: str | None) -> str:
|
||||
username = (bot_username or "").lstrip("@").lower()
|
||||
if not username or not text.startswith("/"):
|
||||
return text
|
||||
|
||||
parts = text.split(maxsplit=1)
|
||||
command_token = parts[0]
|
||||
if "@" not in command_token:
|
||||
return text
|
||||
|
||||
command_name, addressed_username = command_token[1:].rsplit("@", 1)
|
||||
if not command_name or addressed_username.lower() != username:
|
||||
return text
|
||||
|
||||
normalized = f"/{command_name}"
|
||||
if len(parts) > 1:
|
||||
normalized = f"{normalized} {parts[1]}"
|
||||
return normalized
|
||||
|
||||
async def _cmd_start(self, update, context) -> None:
|
||||
"""Handle /start command."""
|
||||
if not self._check_user(update.effective_user.id):
|
||||
return
|
||||
args = getattr(context, "args", []) if context is not None else []
|
||||
if args:
|
||||
handled = await self._bind_connection_from_start_token(update, str(args[0]))
|
||||
if handled:
|
||||
return
|
||||
await update.message.reply_text("Welcome to DeerFlow! Send me a message to start a conversation.\nType /help for available commands.")
|
||||
|
||||
async def _process_incoming_with_reply(self, chat_id: str, msg_id: int, inbound: InboundMessage) -> None:
|
||||
@@ -243,7 +350,7 @@ class TelegramChannel(Channel):
|
||||
if not self._check_user(update.effective_user.id):
|
||||
return
|
||||
|
||||
text = update.message.text
|
||||
text = self._strip_bot_username_from_leading_command(update.message.text.strip(), self._get_bot_username(context))
|
||||
chat_id = str(update.effective_chat.id)
|
||||
user_id = str(update.effective_user.id)
|
||||
msg_id = str(update.message.message_id)
|
||||
@@ -267,6 +374,7 @@ class TelegramChannel(Channel):
|
||||
thread_ts=msg_id,
|
||||
)
|
||||
inbound.topic_id = topic_id
|
||||
inbound = await self._attach_connection_identity(inbound)
|
||||
|
||||
if self._main_loop and self._main_loop.is_running():
|
||||
fut = asyncio.run_coroutine_threadsafe(self._process_incoming_with_reply(chat_id, update.message.message_id, inbound), self._main_loop)
|
||||
@@ -279,7 +387,7 @@ class TelegramChannel(Channel):
|
||||
if not self._check_user(update.effective_user.id):
|
||||
return
|
||||
|
||||
text = update.message.text.strip()
|
||||
text = self._strip_bot_username_from_leading_command(update.message.text.strip(), self._get_bot_username(context))
|
||||
if not text:
|
||||
return
|
||||
|
||||
@@ -309,6 +417,7 @@ class TelegramChannel(Channel):
|
||||
thread_ts=msg_id,
|
||||
)
|
||||
inbound.topic_id = topic_id
|
||||
inbound = await self._attach_connection_identity(inbound)
|
||||
|
||||
if self._main_loop and self._main_loop.is_running():
|
||||
fut = asyncio.run_coroutine_threadsafe(self._process_incoming_with_reply(chat_id, update.message.message_id, inbound), self._main_loop)
|
||||
|
||||
@@ -22,7 +22,9 @@ from cryptography.hazmat.primitives import padding
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
|
||||
from app.channels.base import Channel
|
||||
from app.channels.message_bus import InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment
|
||||
from app.channels.commands import extract_connect_code, is_known_channel_command
|
||||
from app.channels.connection_identity import attach_connection_identity
|
||||
from app.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -252,6 +254,7 @@ class WechatChannel(Channel):
|
||||
self._state_dir = self._resolve_state_dir(config.get("state_dir"))
|
||||
self._cursor_path = self._state_dir / "wechat-getupdates.json" if self._state_dir else None
|
||||
self._auth_path = self._state_dir / "wechat-auth.json" if self._state_dir else None
|
||||
self._connection_repo = config.get("connection_repo")
|
||||
self._load_state()
|
||||
|
||||
async def start(self) -> None:
|
||||
@@ -616,11 +619,21 @@ class WechatChannel(Channel):
|
||||
if thread_ts:
|
||||
self._context_tokens_by_thread[thread_ts] = context_token
|
||||
|
||||
connect_code = extract_connect_code(text)
|
||||
if connect_code and self._connection_repo is not None:
|
||||
handled = await self._bind_connection_from_connect_code(
|
||||
chat_id=chat_id,
|
||||
context_token=context_token,
|
||||
code=connect_code,
|
||||
)
|
||||
if handled:
|
||||
return
|
||||
|
||||
inbound = self._make_inbound(
|
||||
chat_id=chat_id,
|
||||
user_id=chat_id,
|
||||
text=text,
|
||||
msg_type=InboundMessageType.COMMAND if text.startswith("/") else InboundMessageType.CHAT,
|
||||
msg_type=InboundMessageType.COMMAND if is_known_channel_command(text) else InboundMessageType.CHAT,
|
||||
thread_ts=thread_ts,
|
||||
files=files,
|
||||
metadata={
|
||||
@@ -631,8 +644,54 @@ class WechatChannel(Channel):
|
||||
},
|
||||
)
|
||||
inbound.topic_id = None
|
||||
inbound = await self._attach_connection_identity(inbound)
|
||||
await self.bus.publish_inbound(inbound)
|
||||
|
||||
async def _attach_connection_identity(self, inbound: InboundMessage) -> InboundMessage:
|
||||
return await attach_connection_identity(
|
||||
inbound,
|
||||
repo=self._connection_repo,
|
||||
provider="wechat",
|
||||
workspace_id=inbound.chat_id,
|
||||
)
|
||||
|
||||
async def _bind_connection_from_connect_code(self, *, chat_id: str, context_token: str, code: str) -> bool:
|
||||
if self._connection_repo is None or not code:
|
||||
return False
|
||||
|
||||
state = await self._connection_repo.consume_oauth_state(provider="wechat", state=code)
|
||||
if state is None:
|
||||
await self._send_connection_reply(chat_id, context_token, "WeChat connection code is invalid or expired.")
|
||||
return True
|
||||
|
||||
if not chat_id:
|
||||
await self._send_connection_reply(chat_id, context_token, "WeChat connection could not be completed from this message.")
|
||||
return True
|
||||
|
||||
await self._connection_repo.upsert_connection(
|
||||
owner_user_id=state["owner_user_id"],
|
||||
provider="wechat",
|
||||
external_account_id=chat_id,
|
||||
workspace_id=chat_id,
|
||||
metadata={
|
||||
"context_token": context_token,
|
||||
},
|
||||
status="connected",
|
||||
)
|
||||
await self._send_connection_reply(chat_id, context_token, "WeChat connected to DeerFlow.")
|
||||
return True
|
||||
|
||||
async def _send_connection_reply(self, chat_id: str, context_token: str, text: str) -> None:
|
||||
if not context_token:
|
||||
return
|
||||
await self._send_text_message(
|
||||
chat_id=chat_id,
|
||||
context_token=context_token,
|
||||
text=text,
|
||||
client_id_prefix="deerflow-connect",
|
||||
max_retries=1,
|
||||
)
|
||||
|
||||
async def _ensure_authenticated(self) -> bool:
|
||||
async with self._auth_lock:
|
||||
if self._bot_token:
|
||||
|
||||
@@ -8,7 +8,10 @@ from collections.abc import Awaitable, Callable
|
||||
from typing import Any, cast
|
||||
|
||||
from app.channels.base import Channel
|
||||
from app.channels.commands import extract_connect_code, is_known_channel_command
|
||||
from app.channels.connection_identity import attach_connection_identity
|
||||
from app.channels.message_bus import (
|
||||
InboundMessage,
|
||||
InboundMessageType,
|
||||
MessageBus,
|
||||
OutboundMessage,
|
||||
@@ -28,6 +31,7 @@ class WeComChannel(Channel):
|
||||
self._ws_frames: dict[str, dict[str, Any]] = {}
|
||||
self._ws_stream_ids: dict[str, str] = {}
|
||||
self._working_message = "Working on it..."
|
||||
self._connection_repo = config.get("connection_repo")
|
||||
|
||||
@property
|
||||
def supports_streaming(self) -> bool:
|
||||
@@ -270,7 +274,17 @@ class WeComChannel(Channel):
|
||||
|
||||
user_id = (body.get("from") or {}).get("userid")
|
||||
|
||||
inbound_type = InboundMessageType.COMMAND if text.startswith("/") else InboundMessageType.CHAT
|
||||
connect_code = extract_connect_code(text)
|
||||
if connect_code and self._connection_repo is not None:
|
||||
handled = await self._bind_connection_from_connect_code(
|
||||
frame=frame,
|
||||
user_id=str(user_id or ""),
|
||||
code=connect_code,
|
||||
)
|
||||
if handled:
|
||||
return
|
||||
|
||||
inbound_type = InboundMessageType.COMMAND if is_known_channel_command(text) else InboundMessageType.CHAT
|
||||
inbound = self._make_inbound(
|
||||
chat_id=user_id, # keep user's conversation in memory
|
||||
user_id=user_id,
|
||||
@@ -291,8 +305,52 @@ class WeComChannel(Channel):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
inbound = await self._attach_connection_identity(inbound)
|
||||
await self.bus.publish_inbound(inbound)
|
||||
|
||||
async def _attach_connection_identity(self, inbound: InboundMessage) -> InboundMessage:
|
||||
return await attach_connection_identity(
|
||||
inbound,
|
||||
repo=self._connection_repo,
|
||||
provider="wecom",
|
||||
workspace_id=str(inbound.metadata.get("aibotid") or "") or None,
|
||||
fallback_without_workspace=True,
|
||||
)
|
||||
|
||||
async def _bind_connection_from_connect_code(self, *, frame: dict[str, Any], user_id: str, code: str) -> bool:
|
||||
if self._connection_repo is None or not code:
|
||||
return False
|
||||
|
||||
state = await self._connection_repo.consume_oauth_state(provider="wecom", state=code)
|
||||
if state is None:
|
||||
await self._send_connection_reply(frame, "WeCom connection code is invalid or expired.")
|
||||
return True
|
||||
|
||||
if not user_id:
|
||||
await self._send_connection_reply(frame, "WeCom connection could not be completed from this message.")
|
||||
return True
|
||||
|
||||
body = frame.get("body", {}) or {}
|
||||
workspace_id = str(body.get("aibotid") or "") or None
|
||||
await self._connection_repo.upsert_connection(
|
||||
owner_user_id=state["owner_user_id"],
|
||||
provider="wecom",
|
||||
external_account_id=user_id,
|
||||
workspace_id=workspace_id,
|
||||
metadata={
|
||||
"aibotid": workspace_id,
|
||||
"chattype": body.get("chattype"),
|
||||
},
|
||||
status="connected",
|
||||
)
|
||||
await self._send_connection_reply(frame, "WeCom connected to DeerFlow.")
|
||||
return True
|
||||
|
||||
async def _send_connection_reply(self, frame: dict[str, Any], text: str) -> None:
|
||||
if not self._ws_client:
|
||||
return
|
||||
await self._ws_client.reply(frame, {"msgtype": "text", "text": {"content": text}})
|
||||
|
||||
async def _send_ws(self, msg: OutboundMessage, *, _max_retries: int = 3) -> None:
|
||||
if not self._ws_client:
|
||||
return
|
||||
|
||||
@@ -6,6 +6,7 @@ from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.gateway.auth_disabled import warn_if_auth_disabled_enabled
|
||||
from app.gateway.auth_middleware import AuthMiddleware
|
||||
from app.gateway.config import get_gateway_config
|
||||
from app.gateway.csrf_middleware import CSRFMiddleware, get_configured_cors_origins
|
||||
@@ -15,6 +16,7 @@ from app.gateway.routers import (
|
||||
artifacts,
|
||||
assistants_compat,
|
||||
auth,
|
||||
channel_connections,
|
||||
channels,
|
||||
feedback,
|
||||
mcp,
|
||||
@@ -172,6 +174,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
startup_config = get_app_config()
|
||||
apply_logging_level(startup_config.log_level)
|
||||
logger.info("Configuration loaded successfully")
|
||||
warn_if_auth_disabled_enabled()
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to load configuration during gateway startup: {e}"
|
||||
logger.exception(error_msg)
|
||||
@@ -182,6 +185,12 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
# Pre-warm tiktoken encoding cache so the first memory-injection request
|
||||
# never blocks on the BPE data download (which hits an OpenAI/Azure URL
|
||||
# that may be unreachable in restricted networks — see issue #3402).
|
||||
# When memory.token_counting is "char", token counting never touches
|
||||
# tiktoken, so skip the warm-up entirely (avoids even the 5s probe in
|
||||
# network-restricted deployments — see issue #3429).
|
||||
if startup_config.memory.token_counting == "char":
|
||||
logger.info("memory.token_counting='char'; skipping tiktoken warm-up (network-free token estimation)")
|
||||
else:
|
||||
try:
|
||||
from deerflow.agents.memory.prompt import warm_tiktoken_cache
|
||||
|
||||
@@ -192,9 +201,9 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
if warmed:
|
||||
logger.info("tiktoken encoding cache warmed successfully")
|
||||
else:
|
||||
logger.warning("tiktoken encoding cache warm-up failed; token counting will use character-based fallback")
|
||||
logger.warning("tiktoken encoding cache warm-up failed; token counting will use character-based fallback until tiktoken loads successfully")
|
||||
except TimeoutError:
|
||||
logger.warning("tiktoken encoding cache warm-up timed out; token counting will use character-based fallback")
|
||||
logger.warning("tiktoken encoding cache warm-up timed out; token counting will use character-based fallback until tiktoken loads successfully")
|
||||
except Exception:
|
||||
logger.warning("tiktoken warm-up skipped", exc_info=True)
|
||||
|
||||
@@ -376,6 +385,9 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for
|
||||
# Suggestions API is mounted at /api/threads/{thread_id}/suggestions
|
||||
app.include_router(suggestions.router)
|
||||
|
||||
# User-facing IM channel connection API is mounted at /api/channels
|
||||
app.include_router(channel_connections.router)
|
||||
|
||||
# Channels API is mounted at /api/channels
|
||||
app.include_router(channels.router)
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Shared helpers for local/E2E auth-disabled mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
from deerflow.runtime.user_context import DEFAULT_USER_ID
|
||||
|
||||
AUTH_DISABLED_ENV_VAR = "DEER_FLOW_AUTH_DISABLED"
|
||||
AUTH_DISABLED_USER_ID = DEFAULT_USER_ID
|
||||
AUTH_DISABLED_USER_EMAIL = "default@test.local"
|
||||
|
||||
AUTH_SOURCE_SESSION = "session"
|
||||
AUTH_SOURCE_INTERNAL = "internal"
|
||||
AUTH_SOURCE_AUTH_DISABLED = "auth_disabled"
|
||||
|
||||
_PRODUCTION_ENV_VARS: tuple[str, ...] = ("DEER_FLOW_ENV", "ENVIRONMENT")
|
||||
_PRODUCTION_ENV_VALUES: frozenset[str] = frozenset({"prod", "production"})
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_explicit_production_environment() -> bool:
|
||||
return any(os.environ.get(name, "").strip().lower() in _PRODUCTION_ENV_VALUES for name in _PRODUCTION_ENV_VARS)
|
||||
|
||||
|
||||
def is_auth_disabled_requested() -> bool:
|
||||
return os.environ.get(AUTH_DISABLED_ENV_VAR) == "1"
|
||||
|
||||
|
||||
def is_auth_disabled() -> bool:
|
||||
return is_auth_disabled_requested() and not is_explicit_production_environment()
|
||||
|
||||
|
||||
def warn_if_auth_disabled_enabled() -> None:
|
||||
if not is_auth_disabled():
|
||||
return
|
||||
|
||||
logger.warning(
|
||||
"%s=1 is active: authentication is bypassed and anonymous requests run as synthetic admin user %r. Do not enable this in shared or production deployments.",
|
||||
AUTH_DISABLED_ENV_VAR,
|
||||
AUTH_DISABLED_USER_ID,
|
||||
)
|
||||
|
||||
|
||||
def get_auth_disabled_user():
|
||||
return SimpleNamespace(
|
||||
id=AUTH_DISABLED_USER_ID,
|
||||
email=AUTH_DISABLED_USER_EMAIL,
|
||||
password_hash=None,
|
||||
system_role="admin",
|
||||
needs_setup=False,
|
||||
token_version=0,
|
||||
)
|
||||
@@ -17,6 +17,13 @@ from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse
|
||||
from app.gateway.auth_disabled import (
|
||||
AUTH_SOURCE_AUTH_DISABLED,
|
||||
AUTH_SOURCE_INTERNAL,
|
||||
AUTH_SOURCE_SESSION,
|
||||
get_auth_disabled_user,
|
||||
is_auth_disabled,
|
||||
)
|
||||
from app.gateway.authz import _ALL_PERMISSIONS, AuthContext
|
||||
from app.gateway.internal_auth import INTERNAL_AUTH_HEADER_NAME, get_internal_user, is_valid_internal_auth_token
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
@@ -80,18 +87,14 @@ class AuthMiddleware(BaseHTTPMiddleware):
|
||||
if is_valid_internal_auth_token(request.headers.get(INTERNAL_AUTH_HEADER_NAME)):
|
||||
internal_user = get_internal_user()
|
||||
|
||||
# Non-public path: require session cookie
|
||||
if internal_user is None and not request.cookies.get("access_token"):
|
||||
return JSONResponse(
|
||||
status_code=401,
|
||||
content={
|
||||
"detail": AuthErrorResponse(
|
||||
code=AuthErrorCode.NOT_AUTHENTICATED,
|
||||
message="Authentication required",
|
||||
).model_dump()
|
||||
},
|
||||
)
|
||||
auth_source = AUTH_SOURCE_SESSION
|
||||
access_token = request.cookies.get("access_token")
|
||||
|
||||
# Non-public path: require session cookie
|
||||
if internal_user is not None:
|
||||
user = internal_user
|
||||
auth_source = AUTH_SOURCE_INTERNAL
|
||||
elif access_token:
|
||||
# Strict JWT validation: reject junk/expired tokens with 401
|
||||
# right here instead of silently passing through. This closes
|
||||
# the "junk cookie bypass" gap (AUTH_TEST_PLAN test 7.5.8):
|
||||
@@ -105,19 +108,33 @@ class AuthMiddleware(BaseHTTPMiddleware):
|
||||
# bubble up, so we catch and render it as JSONResponse here.
|
||||
from app.gateway.deps import get_current_user_from_request
|
||||
|
||||
if internal_user is not None:
|
||||
user = internal_user
|
||||
else:
|
||||
try:
|
||||
user = await get_current_user_from_request(request)
|
||||
except HTTPException as exc:
|
||||
if not is_auth_disabled():
|
||||
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
|
||||
user = get_auth_disabled_user()
|
||||
auth_source = AUTH_SOURCE_AUTH_DISABLED
|
||||
elif is_auth_disabled():
|
||||
user = get_auth_disabled_user()
|
||||
auth_source = AUTH_SOURCE_AUTH_DISABLED
|
||||
else:
|
||||
return JSONResponse(
|
||||
status_code=401,
|
||||
content={
|
||||
"detail": AuthErrorResponse(
|
||||
code=AuthErrorCode.NOT_AUTHENTICATED,
|
||||
message="Authentication required",
|
||||
).model_dump()
|
||||
},
|
||||
)
|
||||
|
||||
# Stamp both request.state.user (for the contextvar pattern)
|
||||
# and request.state.auth (so @require_permission's "auth is
|
||||
# None" branch short-circuits instead of running the entire
|
||||
# JWT-decode + DB-lookup pipeline a second time per request).
|
||||
request.state.user = user
|
||||
request.state.auth_source = auth_source
|
||||
request.state.auth = AuthContext(user=user, permissions=_ALL_PERMISSIONS)
|
||||
token = set_current_user(user)
|
||||
try:
|
||||
|
||||
@@ -276,6 +276,11 @@ def require_permission(
|
||||
# strict-deny rather than strict-allow — only an *existing*
|
||||
# row with a *different* user_id triggers 404.
|
||||
if owner_check:
|
||||
from app.gateway.internal_auth import INTERNAL_SYSTEM_ROLE
|
||||
|
||||
if getattr(auth.user, "system_role", None) == INTERNAL_SYSTEM_ROLE:
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
thread_id = kwargs.get("thread_id")
|
||||
if thread_id is None:
|
||||
raise ValueError("require_permission with owner_check=True requires 'thread_id' parameter")
|
||||
|
||||
@@ -14,6 +14,8 @@ from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from app.gateway.auth_disabled import is_auth_disabled
|
||||
|
||||
CSRF_COOKIE_NAME = "csrf_token"
|
||||
CSRF_HEADER_NAME = "X-CSRF-Token"
|
||||
CSRF_TOKEN_LENGTH = 64 # bytes
|
||||
@@ -38,6 +40,9 @@ def should_check_csrf(request: Request) -> bool:
|
||||
if request.method not in ("POST", "PUT", "DELETE", "PATCH"):
|
||||
return False
|
||||
|
||||
if is_auth_disabled():
|
||||
return False
|
||||
|
||||
path = request.url.path.rstrip("/")
|
||||
# Exempt /api/v1/auth/me endpoint
|
||||
if path == "/api/v1/auth/me":
|
||||
|
||||
@@ -331,6 +331,17 @@ async def get_current_user_from_request(request: Request):
|
||||
|
||||
Raises HTTPException 401 if not authenticated.
|
||||
"""
|
||||
state = getattr(request, "state", None)
|
||||
state_user = getattr(state, "user", None)
|
||||
from app.gateway.auth_disabled import AUTH_SOURCE_AUTH_DISABLED, AUTH_SOURCE_INTERNAL, AUTH_SOURCE_SESSION
|
||||
|
||||
if state_user is not None and getattr(state, "auth_source", None) in {
|
||||
AUTH_SOURCE_SESSION,
|
||||
AUTH_SOURCE_AUTH_DISABLED,
|
||||
AUTH_SOURCE_INTERNAL,
|
||||
}:
|
||||
return state_user
|
||||
|
||||
from app.gateway.auth import decode_token
|
||||
from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse, TokenError, token_error_to_code
|
||||
|
||||
|
||||
@@ -5,10 +5,12 @@ from __future__ import annotations
|
||||
import os
|
||||
import secrets
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from deerflow.runtime.user_context import DEFAULT_USER_ID
|
||||
|
||||
INTERNAL_AUTH_HEADER_NAME = "X-DeerFlow-Internal-Token"
|
||||
INTERNAL_OWNER_USER_ID_HEADER_NAME = "X-DeerFlow-Owner-User-Id"
|
||||
INTERNAL_AUTH_ENV_VAR = "DEER_FLOW_INTERNAL_AUTH_TOKEN"
|
||||
INTERNAL_SYSTEM_ROLE = "internal"
|
||||
|
||||
@@ -23,9 +25,12 @@ def _load_internal_auth_token() -> str:
|
||||
_INTERNAL_AUTH_TOKEN = _load_internal_auth_token()
|
||||
|
||||
|
||||
def create_internal_auth_headers() -> dict[str, str]:
|
||||
def create_internal_auth_headers(*, owner_user_id: str | None = None) -> dict[str, str]:
|
||||
"""Return headers that authenticate trusted Gateway internal calls."""
|
||||
return {INTERNAL_AUTH_HEADER_NAME: _INTERNAL_AUTH_TOKEN}
|
||||
headers = {INTERNAL_AUTH_HEADER_NAME: _INTERNAL_AUTH_TOKEN}
|
||||
if owner_user_id:
|
||||
headers[INTERNAL_OWNER_USER_ID_HEADER_NAME] = owner_user_id
|
||||
return headers
|
||||
|
||||
|
||||
def is_valid_internal_auth_token(token: str | None) -> bool:
|
||||
@@ -36,3 +41,21 @@ def is_valid_internal_auth_token(token: str | None) -> bool:
|
||||
def get_internal_user():
|
||||
"""Return the synthetic user used for trusted internal channel calls."""
|
||||
return SimpleNamespace(id=DEFAULT_USER_ID, system_role=INTERNAL_SYSTEM_ROLE)
|
||||
|
||||
|
||||
def get_trusted_internal_owner_user_id(request: Any) -> str | None:
|
||||
"""Return the owner override for a trusted internal request, if present.
|
||||
|
||||
The header is ignored for normal browser/API callers. It is only honored
|
||||
after ``AuthMiddleware`` has validated the internal auth token and stamped
|
||||
the synthetic internal user onto ``request.state.user``.
|
||||
"""
|
||||
user = getattr(getattr(request, "state", None), "user", None)
|
||||
if getattr(user, "system_role", None) != INTERNAL_SYSTEM_ROLE:
|
||||
return None
|
||||
|
||||
owner_user_id = request.headers.get(INTERNAL_OWNER_USER_ID_HEADER_NAME)
|
||||
if not owner_user_id:
|
||||
return None
|
||||
owner_user_id = owner_user_id.strip()
|
||||
return owner_user_id or None
|
||||
|
||||
@@ -20,6 +20,7 @@ from langgraph_sdk import Auth
|
||||
|
||||
from app.gateway.auth.errors import TokenError
|
||||
from app.gateway.auth.jwt import decode_token
|
||||
from app.gateway.auth_disabled import AUTH_DISABLED_USER_ID, is_auth_disabled
|
||||
from app.gateway.deps import get_local_provider
|
||||
|
||||
auth = Auth()
|
||||
@@ -38,6 +39,9 @@ def _check_csrf(request) -> None:
|
||||
if method.upper() not in _CSRF_METHODS:
|
||||
return
|
||||
|
||||
if is_auth_disabled():
|
||||
return
|
||||
|
||||
cookie_token = request.cookies.get("csrf_token")
|
||||
header_token = request.headers.get("x-csrf-token")
|
||||
|
||||
@@ -66,6 +70,9 @@ async def authenticate(request):
|
||||
# are rejected early, even if the cookie carries a valid JWT.
|
||||
_check_csrf(request)
|
||||
|
||||
if is_auth_disabled():
|
||||
return AUTH_DISABLED_USER_ID
|
||||
|
||||
token = request.cookies.get("access_token")
|
||||
if not token:
|
||||
raise Auth.exceptions.HTTPException(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""CRUD API for custom agents."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
@@ -213,15 +214,21 @@ async def create_agent_endpoint(request: AgentCreateRequest) -> AgentResponse:
|
||||
user_id = get_effective_user_id()
|
||||
paths = get_paths()
|
||||
|
||||
def _create_agent() -> AgentResponse | None:
|
||||
# Worker thread: base-dir resolution, existence checks, directory/file
|
||||
# creation, read-back, and failure cleanup are all blocking filesystem
|
||||
# IO that must stay off the event loop.
|
||||
agent_dir = paths.user_agent_dir(user_id, normalized_name)
|
||||
legacy_dir = paths.agent_dir(normalized_name)
|
||||
|
||||
if agent_dir.exists() or legacy_dir.exists():
|
||||
raise HTTPException(status_code=409, detail=f"Agent '{normalized_name}' already exists")
|
||||
if legacy_dir.exists():
|
||||
return None # signals 409 to the caller
|
||||
|
||||
try:
|
||||
agent_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
agent_dir.mkdir(parents=True, exist_ok=False)
|
||||
except FileExistsError:
|
||||
return None # signals 409 to the caller
|
||||
# Write config.yaml
|
||||
config_data: dict = {"name": normalized_name}
|
||||
if request.description:
|
||||
@@ -245,16 +252,23 @@ async def create_agent_endpoint(request: AgentCreateRequest) -> AgentResponse:
|
||||
|
||||
agent_cfg = load_agent_config(normalized_name, user_id=user_id)
|
||||
return _agent_config_to_response(agent_cfg, include_soul=True, user_id=user_id)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
# Clean up on failure
|
||||
except Exception:
|
||||
# Clean up partial state on failure before surfacing the error.
|
||||
if agent_dir.exists():
|
||||
shutil.rmtree(agent_dir)
|
||||
raise
|
||||
|
||||
try:
|
||||
response = await asyncio.to_thread(_create_agent)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create agent '{request.name}': {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to create agent: {str(e)}")
|
||||
|
||||
if response is None:
|
||||
raise HTTPException(status_code=409, detail=f"Agent '{normalized_name}' already exists")
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@router.put(
|
||||
"/agents/{name}",
|
||||
@@ -428,19 +442,30 @@ async def delete_agent(name: str) -> None:
|
||||
name = _normalize_agent_name(name)
|
||||
user_id = get_effective_user_id()
|
||||
paths = get_paths()
|
||||
agent_dir = paths.user_agent_dir(user_id, name)
|
||||
|
||||
def _remove_agent_dir() -> tuple[str, str]:
|
||||
# Runs in a worker thread: resolving the base dir, probing the directory
|
||||
# (`exists`), and removing it (`rmtree`) are all blocking filesystem IO
|
||||
# that must stay off the event loop.
|
||||
agent_dir = paths.user_agent_dir(user_id, name)
|
||||
if not agent_dir.exists():
|
||||
if paths.agent_dir(name).exists():
|
||||
outcome = "legacy" if paths.agent_dir(name).exists() else "missing"
|
||||
return outcome, str(agent_dir)
|
||||
shutil.rmtree(agent_dir)
|
||||
return "deleted", str(agent_dir)
|
||||
|
||||
try:
|
||||
outcome, agent_dir = await asyncio.to_thread(_remove_agent_dir)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete agent '{name}': {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to delete agent: {str(e)}")
|
||||
|
||||
if outcome == "legacy":
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(f"Agent '{name}' only exists in the legacy shared layout and is not scoped to a user. Run scripts/migrate_user_isolation.py to move legacy agents into the per-user layout before deleting."),
|
||||
)
|
||||
if outcome == "missing":
|
||||
raise HTTPException(status_code=404, detail=f"Agent '{name}' not found")
|
||||
|
||||
try:
|
||||
shutil.rmtree(agent_dir)
|
||||
logger.info(f"Deleted agent '{name}' from {agent_dir}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete agent '{name}': {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to delete agent: {str(e)}")
|
||||
|
||||
@@ -341,9 +341,19 @@ async def change_password(request: Request, response: Response, body: ChangePass
|
||||
- Re-issues session cookie with new token_version
|
||||
"""
|
||||
from app.gateway.auth.password import hash_password_async, verify_password_async
|
||||
from app.gateway.auth_disabled import AUTH_SOURCE_AUTH_DISABLED
|
||||
|
||||
user = await get_current_user_from_request(request)
|
||||
|
||||
if getattr(request.state, "auth_source", None) == AUTH_SOURCE_AUTH_DISABLED:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=AuthErrorResponse(
|
||||
code=AuthErrorCode.INVALID_CREDENTIALS,
|
||||
message="Password changes are not available when DEER_FLOW_AUTH_DISABLED=1.",
|
||||
).model_dump(),
|
||||
)
|
||||
|
||||
if user.password_hash is None:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=AuthErrorResponse(code=AuthErrorCode.INVALID_CREDENTIALS, message="OAuth users cannot change password").model_dump())
|
||||
|
||||
|
||||
@@ -0,0 +1,600 @@
|
||||
"""Browser-facing APIs for user-owned IM channel bindings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request, Response
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.channels.runtime_config_store import (
|
||||
ChannelRuntimeConfigStore,
|
||||
apply_runtime_connection_config,
|
||||
merge_runtime_channel_configs,
|
||||
)
|
||||
from deerflow.config.channel_connections_config import ChannelConnectionsConfig
|
||||
from deerflow.persistence.channel_connections import ChannelConnectionRepository
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
router = APIRouter(prefix="/api/channels", tags=["channel-connections"])
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_STATE_TTL_SECONDS = 600
|
||||
_MASKED_CREDENTIAL_VALUE = "********"
|
||||
|
||||
|
||||
class ChannelCredentialFieldResponse(BaseModel):
|
||||
name: str
|
||||
label: str
|
||||
type: str = "text"
|
||||
required: bool = True
|
||||
|
||||
|
||||
class ChannelProviderResponse(BaseModel):
|
||||
provider: str
|
||||
display_name: str
|
||||
enabled: bool
|
||||
configured: bool
|
||||
connectable: bool
|
||||
unavailable_reason: str | None = None
|
||||
auth_mode: str
|
||||
connection_status: str
|
||||
credential_fields: list[ChannelCredentialFieldResponse] = Field(default_factory=list)
|
||||
credential_values: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ChannelProvidersResponse(BaseModel):
|
||||
enabled: bool
|
||||
providers: list[ChannelProviderResponse]
|
||||
|
||||
|
||||
class ChannelConnectionResponse(BaseModel):
|
||||
id: str
|
||||
provider: str
|
||||
status: str
|
||||
external_account_id: str | None = None
|
||||
external_account_name: str | None = None
|
||||
workspace_id: str | None = None
|
||||
workspace_name: str | None = None
|
||||
scopes: list[str] = Field(default_factory=list)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ChannelConnectionsResponse(BaseModel):
|
||||
connections: list[ChannelConnectionResponse]
|
||||
|
||||
|
||||
class ChannelConnectResponse(BaseModel):
|
||||
provider: str
|
||||
mode: str
|
||||
url: str | None = None
|
||||
code: str
|
||||
instruction: str
|
||||
expires_in: int
|
||||
|
||||
|
||||
class ChannelRuntimeConfigRequest(BaseModel):
|
||||
values: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
_PROVIDER_META: dict[str, dict[str, str]] = {
|
||||
"telegram": {"display_name": "Telegram", "auth_mode": "deep_link"},
|
||||
"slack": {"display_name": "Slack", "auth_mode": "binding_code"},
|
||||
"discord": {"display_name": "Discord", "auth_mode": "binding_code"},
|
||||
"feishu": {"display_name": "Feishu", "auth_mode": "binding_code"},
|
||||
"dingtalk": {"display_name": "DingTalk", "auth_mode": "binding_code"},
|
||||
"wechat": {"display_name": "WeChat", "auth_mode": "binding_code"},
|
||||
"wecom": {"display_name": "WeCom", "auth_mode": "binding_code"},
|
||||
}
|
||||
|
||||
_CREDENTIAL_FIELDS: dict[str, tuple[dict[str, str], ...]] = {
|
||||
"telegram": (
|
||||
{"name": "bot_token", "label": "Bot token", "type": "password"},
|
||||
{"name": "bot_username", "label": "Bot username", "type": "text"},
|
||||
),
|
||||
"slack": (
|
||||
{"name": "bot_token", "label": "Bot token", "type": "password"},
|
||||
{"name": "app_token", "label": "App token", "type": "password"},
|
||||
),
|
||||
"discord": ({"name": "bot_token", "label": "Bot token", "type": "password"},),
|
||||
"feishu": (
|
||||
{"name": "app_id", "label": "App ID", "type": "text"},
|
||||
{"name": "app_secret", "label": "App secret", "type": "password"},
|
||||
),
|
||||
"dingtalk": (
|
||||
{"name": "client_id", "label": "Client ID", "type": "text"},
|
||||
{"name": "client_secret", "label": "Client secret", "type": "password"},
|
||||
),
|
||||
"wechat": ({"name": "bot_token", "label": "Bot token", "type": "password"},),
|
||||
"wecom": (
|
||||
{"name": "bot_id", "label": "Bot ID", "type": "text"},
|
||||
{"name": "bot_secret", "label": "Bot secret", "type": "password"},
|
||||
),
|
||||
}
|
||||
|
||||
_RUNTIME_REQUIREMENTS: dict[str, tuple[str, ...]] = {
|
||||
"telegram": ("bot_token",),
|
||||
"slack": ("bot_token", "app_token"),
|
||||
"discord": ("bot_token",),
|
||||
"feishu": ("app_id", "app_secret"),
|
||||
"dingtalk": ("client_id", "client_secret"),
|
||||
"wechat": ("bot_token",),
|
||||
"wecom": ("bot_id", "bot_secret"),
|
||||
}
|
||||
|
||||
|
||||
def _get_user_id(request: Request) -> str:
|
||||
user = getattr(request.state, "user", None)
|
||||
if user is None:
|
||||
raise HTTPException(status_code=401, detail="Authentication required")
|
||||
return str(user.id)
|
||||
|
||||
|
||||
def _get_app_config():
|
||||
from deerflow.config.app_config import get_app_config
|
||||
|
||||
return get_app_config()
|
||||
|
||||
|
||||
def _get_runtime_config_store(request: Request) -> ChannelRuntimeConfigStore:
|
||||
store = getattr(request.app.state, "channel_runtime_config_store", None)
|
||||
if isinstance(store, ChannelRuntimeConfigStore):
|
||||
return store
|
||||
store = ChannelRuntimeConfigStore()
|
||||
request.app.state.channel_runtime_config_store = store
|
||||
return store
|
||||
|
||||
|
||||
def _get_channel_connections_config(request: Request) -> ChannelConnectionsConfig:
|
||||
config = getattr(request.app.state, "channel_connections_config", None)
|
||||
if not isinstance(config, ChannelConnectionsConfig):
|
||||
config = _get_app_config().channel_connections
|
||||
config = apply_runtime_connection_config(config, store=_get_runtime_config_store(request))
|
||||
request.app.state.channel_connections_config = config
|
||||
return config
|
||||
|
||||
|
||||
def _get_channels_config(request: Request) -> dict[str, Any]:
|
||||
state_config = getattr(request.app.state, "channels_config", None)
|
||||
if isinstance(state_config, dict):
|
||||
return state_config
|
||||
|
||||
result = _load_channels_config(request, _get_channel_connections_config(request))
|
||||
request.app.state.channels_config = result
|
||||
return result
|
||||
|
||||
|
||||
def _load_channels_config(request: Request, config: ChannelConnectionsConfig) -> dict[str, Any]:
|
||||
app_config = _get_app_config()
|
||||
extra = app_config.model_extra or {}
|
||||
channels_config = extra.get("channels")
|
||||
result = dict(channels_config) if isinstance(channels_config, dict) else {}
|
||||
merge_runtime_channel_configs(
|
||||
result,
|
||||
config,
|
||||
store=_get_runtime_config_store(request),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _get_repository(request: Request, config: ChannelConnectionsConfig) -> ChannelConnectionRepository:
|
||||
repo = getattr(request.app.state, "channel_connection_repo", None)
|
||||
if isinstance(repo, ChannelConnectionRepository):
|
||||
return repo
|
||||
|
||||
sf = get_session_factory()
|
||||
if sf is None:
|
||||
raise HTTPException(status_code=503, detail="Channel connection persistence is not available")
|
||||
|
||||
repo = ChannelConnectionRepository(sf)
|
||||
request.app.state.channel_connection_repo = repo
|
||||
return repo
|
||||
|
||||
|
||||
def _provider_config(config: ChannelConnectionsConfig, provider: str):
|
||||
provider_config = getattr(config, provider, None)
|
||||
if provider_config is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel provider")
|
||||
return provider_config
|
||||
|
||||
|
||||
def _runtime_channel_configured(provider: str, channels_config: dict[str, Any]) -> bool:
|
||||
runtime_config = channels_config.get(provider)
|
||||
if not isinstance(runtime_config, dict) or not runtime_config.get("enabled", False):
|
||||
return False
|
||||
return all(str(runtime_config.get(key) or "").strip() for key in _RUNTIME_REQUIREMENTS[provider])
|
||||
|
||||
|
||||
def _runtime_unavailable_reason(provider: str) -> str:
|
||||
meta = _PROVIDER_META.get(provider)
|
||||
display_name = meta["display_name"] if meta else provider
|
||||
return f"Enter the required {display_name} credentials to connect this channel."
|
||||
|
||||
|
||||
def _runtime_not_running_reason(provider: str) -> str:
|
||||
meta = _PROVIDER_META.get(provider)
|
||||
display_name = meta["display_name"] if meta else provider
|
||||
return f"{display_name} channel is configured but is not running. Check the credentials and save this channel again."
|
||||
|
||||
|
||||
def _runtime_channel_running(provider: str) -> bool | None:
|
||||
try:
|
||||
from app.channels.service import get_channel_service
|
||||
except Exception:
|
||||
logger.debug("Unable to inspect channel service status", exc_info=True)
|
||||
return None
|
||||
|
||||
service = get_channel_service()
|
||||
if service is None:
|
||||
return None
|
||||
try:
|
||||
status = service.get_status()
|
||||
except Exception:
|
||||
logger.debug("Unable to read channel service status", exc_info=True)
|
||||
return None
|
||||
|
||||
if not status.get("service_running"):
|
||||
return False
|
||||
channel_status = status.get("channels", {}).get(provider)
|
||||
if not isinstance(channel_status, dict):
|
||||
return None
|
||||
return bool(channel_status.get("running"))
|
||||
|
||||
|
||||
def _provider_unavailable_reason(
|
||||
config: ChannelConnectionsConfig,
|
||||
channels_config: dict[str, Any],
|
||||
provider: str,
|
||||
) -> str | None:
|
||||
provider_config = _provider_config(config, provider)
|
||||
if not provider_config.enabled:
|
||||
return None
|
||||
if not provider_config.configured:
|
||||
return _runtime_unavailable_reason(provider)
|
||||
if not _runtime_channel_configured(provider, channels_config):
|
||||
return _runtime_unavailable_reason(provider)
|
||||
if _runtime_channel_running(provider) is False:
|
||||
return _runtime_not_running_reason(provider)
|
||||
return None
|
||||
|
||||
|
||||
def _provider_status(
|
||||
config: ChannelConnectionsConfig,
|
||||
channels_config: dict[str, Any],
|
||||
provider: str,
|
||||
) -> tuple[dict[str, bool], str | None]:
|
||||
declared = config.provider_status(provider)
|
||||
unavailable_reason = _provider_unavailable_reason(config, channels_config, provider)
|
||||
configured = declared["configured"] and _runtime_channel_configured(provider, channels_config)
|
||||
return {"enabled": declared["enabled"], "configured": configured}, unavailable_reason
|
||||
|
||||
|
||||
def _new_binding_code() -> str:
|
||||
return secrets.token_urlsafe(16)
|
||||
|
||||
|
||||
async def _create_state(
|
||||
repo: ChannelConnectionRepository,
|
||||
*,
|
||||
owner_user_id: str,
|
||||
provider: str,
|
||||
) -> str:
|
||||
state = _new_binding_code()
|
||||
await repo.create_oauth_state(
|
||||
owner_user_id=owner_user_id,
|
||||
provider=provider,
|
||||
state=state,
|
||||
expires_at=datetime.now(UTC) + timedelta(seconds=_STATE_TTL_SECONDS),
|
||||
)
|
||||
return state
|
||||
|
||||
|
||||
def _connect_instruction(provider: str, code: str) -> str:
|
||||
if provider == "telegram":
|
||||
return f"Send /start {code} to the DeerFlow Telegram bot."
|
||||
meta = _PROVIDER_META.get(provider)
|
||||
if meta is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel provider")
|
||||
return f"Send /connect {code} to the DeerFlow {meta['display_name']} bot."
|
||||
|
||||
|
||||
def _connect_url(config: ChannelConnectionsConfig, provider: str, code: str) -> str | None:
|
||||
if provider == "telegram":
|
||||
provider_config = _provider_config(config, provider)
|
||||
return f"https://t.me/{provider_config.bot_username}?start={code}"
|
||||
if _PROVIDER_META.get(provider, {}).get("auth_mode") == "binding_code":
|
||||
return None
|
||||
raise HTTPException(status_code=404, detail="Unknown channel provider")
|
||||
|
||||
|
||||
def _connection_updated_at(connection: dict[str, Any]) -> datetime:
|
||||
value = connection.get("updated_at")
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
pass
|
||||
return datetime.min.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def _newest_connection_by_provider(connections: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
by_provider: dict[str, dict[str, Any]] = {}
|
||||
for item in connections:
|
||||
existing = by_provider.get(item["provider"])
|
||||
if existing is None or _connection_updated_at(item) > _connection_updated_at(existing):
|
||||
by_provider[item["provider"]] = item
|
||||
return by_provider
|
||||
|
||||
|
||||
def _credential_fields(provider: str) -> list[ChannelCredentialFieldResponse]:
|
||||
fields = _CREDENTIAL_FIELDS.get(provider)
|
||||
if fields is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel provider")
|
||||
return [ChannelCredentialFieldResponse(**field) for field in fields]
|
||||
|
||||
|
||||
def _credential_values(provider: str, channels_config: dict[str, Any]) -> dict[str, str]:
|
||||
runtime_config = channels_config.get(provider)
|
||||
if not isinstance(runtime_config, dict):
|
||||
return {}
|
||||
|
||||
values: dict[str, str] = {}
|
||||
for field in _credential_fields(provider):
|
||||
value = str(runtime_config.get(field.name) or "").strip()
|
||||
if not value:
|
||||
continue
|
||||
values[field.name] = _MASKED_CREDENTIAL_VALUE if field.type == "password" else value
|
||||
return values
|
||||
|
||||
|
||||
def _provider_response(
|
||||
config: ChannelConnectionsConfig,
|
||||
channels_config: dict[str, Any],
|
||||
provider: str,
|
||||
meta: dict[str, str],
|
||||
connection: dict[str, Any] | None = None,
|
||||
) -> ChannelProviderResponse:
|
||||
status, unavailable_reason = _provider_status(config, channels_config, provider)
|
||||
if connection:
|
||||
connection_status = connection["status"]
|
||||
elif status["configured"] and unavailable_reason is None:
|
||||
connection_status = "connected"
|
||||
else:
|
||||
connection_status = "not_connected"
|
||||
credential_values = _credential_values(provider, channels_config)
|
||||
if provider == "telegram" and not credential_values.get("bot_username"):
|
||||
bot_username = str(_provider_config(config, provider).bot_username or "").strip()
|
||||
if bot_username:
|
||||
credential_values["bot_username"] = bot_username
|
||||
return ChannelProviderResponse(
|
||||
provider=provider,
|
||||
display_name=meta["display_name"],
|
||||
enabled=status["enabled"],
|
||||
configured=status["configured"],
|
||||
connectable=status["enabled"] and status["configured"] and unavailable_reason is None,
|
||||
unavailable_reason=unavailable_reason,
|
||||
auth_mode=meta["auth_mode"],
|
||||
connection_status=connection_status,
|
||||
credential_fields=_credential_fields(provider),
|
||||
credential_values=credential_values,
|
||||
)
|
||||
|
||||
|
||||
def _required_runtime_values(
|
||||
provider: str,
|
||||
values: dict[str, str],
|
||||
existing_config: dict[str, Any] | None = None,
|
||||
) -> dict[str, str]:
|
||||
fields = _credential_fields(provider)
|
||||
cleaned: dict[str, str] = {}
|
||||
missing: list[str] = []
|
||||
existing_config = existing_config or {}
|
||||
for field in fields:
|
||||
raw_value = values.get(field.name, "")
|
||||
if field.type == "password" and raw_value == _MASKED_CREDENTIAL_VALUE:
|
||||
existing_value = str(existing_config.get(field.name) or "").strip()
|
||||
if existing_value:
|
||||
cleaned[field.name] = existing_value
|
||||
continue
|
||||
value = raw_value.strip() if isinstance(raw_value, str) else str(raw_value or "").strip()
|
||||
if field.required and not value:
|
||||
missing.append(field.label)
|
||||
cleaned[field.name] = value
|
||||
if missing:
|
||||
raise HTTPException(status_code=400, detail=f"Missing required channel configuration: {', '.join(missing)}")
|
||||
return cleaned
|
||||
|
||||
|
||||
async def _restart_runtime_channel_if_available(provider: str, runtime_config: dict[str, Any]) -> bool | None:
|
||||
try:
|
||||
from app.channels.service import get_channel_service
|
||||
except Exception:
|
||||
logger.exception("Failed to import channel service while configuring %s", provider)
|
||||
return None
|
||||
|
||||
service = get_channel_service()
|
||||
if service is None:
|
||||
return None
|
||||
return await service.configure_channel(provider, runtime_config)
|
||||
|
||||
|
||||
async def _sync_runtime_channel_after_removal(provider: str, channels_config: dict[str, Any]) -> bool | None:
|
||||
try:
|
||||
from app.channels.service import get_channel_service
|
||||
except Exception:
|
||||
logger.exception("Failed to import channel service while disconnecting %s", provider)
|
||||
return None
|
||||
|
||||
service = get_channel_service()
|
||||
if service is None:
|
||||
return None
|
||||
|
||||
runtime_config = channels_config.get(provider)
|
||||
if isinstance(runtime_config, dict) and runtime_config.get("enabled", False):
|
||||
return await service.configure_channel(provider, runtime_config)
|
||||
return await service.remove_channel(provider)
|
||||
|
||||
|
||||
@router.get("/providers", response_model=ChannelProvidersResponse)
|
||||
async def get_channel_providers(request: Request) -> ChannelProvidersResponse:
|
||||
config = _get_channel_connections_config(request)
|
||||
channels_config = _get_channels_config(request)
|
||||
repo = None
|
||||
if config.enabled:
|
||||
try:
|
||||
repo = _get_repository(request, config)
|
||||
except HTTPException as exc:
|
||||
if exc.status_code != 503:
|
||||
raise
|
||||
owner_user_id = _get_user_id(request)
|
||||
connections = await repo.list_connections(owner_user_id) if repo is not None else []
|
||||
by_provider = _newest_connection_by_provider(connections)
|
||||
|
||||
providers: list[ChannelProviderResponse] = []
|
||||
for provider, meta in _PROVIDER_META.items():
|
||||
if not config.provider_status(provider)["enabled"]:
|
||||
continue
|
||||
connection = by_provider.get(provider)
|
||||
providers.append(_provider_response(config, channels_config, provider, meta, connection))
|
||||
return ChannelProvidersResponse(enabled=config.enabled, providers=providers)
|
||||
|
||||
|
||||
@router.get("/connections", response_model=ChannelConnectionsResponse)
|
||||
async def get_channel_connections(request: Request) -> ChannelConnectionsResponse:
|
||||
config = _get_channel_connections_config(request)
|
||||
if not config.enabled:
|
||||
return ChannelConnectionsResponse(connections=[])
|
||||
repo = _get_repository(request, config)
|
||||
rows = await repo.list_connections(_get_user_id(request))
|
||||
return ChannelConnectionsResponse(connections=[ChannelConnectionResponse(**row) for row in rows])
|
||||
|
||||
|
||||
@router.delete("/connections/{connection_id}", status_code=204)
|
||||
async def disconnect_channel_connection(connection_id: str, request: Request) -> Response:
|
||||
config = _get_channel_connections_config(request)
|
||||
if not config.enabled:
|
||||
raise HTTPException(status_code=400, detail="Channel connections are disabled")
|
||||
|
||||
repo = _get_repository(request, config)
|
||||
disconnected = await repo.disconnect_connection(
|
||||
connection_id=connection_id,
|
||||
owner_user_id=_get_user_id(request),
|
||||
)
|
||||
if not disconnected:
|
||||
raise HTTPException(status_code=404, detail="Channel connection not found")
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@router.delete("/{provider}/runtime-config", response_model=ChannelProviderResponse)
|
||||
async def disconnect_channel_provider_runtime(provider: str, request: Request) -> ChannelProviderResponse:
|
||||
config = _get_channel_connections_config(request)
|
||||
if not config.enabled:
|
||||
raise HTTPException(status_code=400, detail="Channel connections are disabled")
|
||||
|
||||
provider_config = _provider_config(config, provider)
|
||||
if not provider_config.enabled:
|
||||
raise HTTPException(status_code=400, detail="Channel provider is not enabled")
|
||||
|
||||
owner_user_id = _get_user_id(request)
|
||||
try:
|
||||
repo = _get_repository(request, config)
|
||||
except HTTPException as exc:
|
||||
if exc.status_code != 503:
|
||||
raise
|
||||
repo = None
|
||||
|
||||
if repo is not None:
|
||||
for connection in await repo.list_connections(owner_user_id):
|
||||
if connection["provider"] == provider and connection["status"] != "revoked":
|
||||
await repo.disconnect_connection(
|
||||
connection_id=connection["id"],
|
||||
owner_user_id=owner_user_id,
|
||||
)
|
||||
|
||||
_get_runtime_config_store(request).remove_provider_config(provider)
|
||||
channels_config = _load_channels_config(request, config)
|
||||
request.app.state.channels_config = channels_config
|
||||
|
||||
stopped = await _sync_runtime_channel_after_removal(provider, channels_config)
|
||||
if stopped is False:
|
||||
display_name = _PROVIDER_META[provider]["display_name"]
|
||||
raise HTTPException(status_code=400, detail=f"Failed to stop {display_name} channel. Try again.")
|
||||
|
||||
return _provider_response(config, channels_config, provider, _PROVIDER_META[provider])
|
||||
|
||||
|
||||
@router.post("/{provider}/connect", response_model=ChannelConnectResponse)
|
||||
async def connect_channel_provider(provider: str, request: Request) -> ChannelConnectResponse:
|
||||
config = _get_channel_connections_config(request)
|
||||
channels_config = _get_channels_config(request)
|
||||
if not config.enabled:
|
||||
raise HTTPException(status_code=400, detail="Channel connections are disabled")
|
||||
|
||||
status, unavailable_reason = _provider_status(config, channels_config, provider)
|
||||
if not status["enabled"]:
|
||||
raise HTTPException(status_code=400, detail="Channel provider is not enabled")
|
||||
if unavailable_reason:
|
||||
raise HTTPException(status_code=400, detail=unavailable_reason)
|
||||
if not status["configured"]:
|
||||
raise HTTPException(status_code=400, detail="Channel provider is not configured")
|
||||
|
||||
repo = _get_repository(request, config)
|
||||
code = await _create_state(
|
||||
repo,
|
||||
owner_user_id=_get_user_id(request),
|
||||
provider=provider,
|
||||
)
|
||||
return ChannelConnectResponse(
|
||||
provider=provider,
|
||||
mode=_PROVIDER_META[provider]["auth_mode"],
|
||||
url=_connect_url(config, provider, code),
|
||||
code=code,
|
||||
instruction=_connect_instruction(provider, code),
|
||||
expires_in=_STATE_TTL_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{provider}/runtime-config", response_model=ChannelProviderResponse)
|
||||
async def configure_channel_provider_runtime(
|
||||
provider: str,
|
||||
body: ChannelRuntimeConfigRequest,
|
||||
request: Request,
|
||||
) -> ChannelProviderResponse:
|
||||
config = _get_channel_connections_config(request)
|
||||
if not config.enabled:
|
||||
raise HTTPException(status_code=400, detail="Channel connections are disabled")
|
||||
|
||||
provider_config = _provider_config(config, provider)
|
||||
if not provider_config.enabled:
|
||||
raise HTTPException(status_code=400, detail="Channel provider is not enabled")
|
||||
|
||||
channels_config = _get_channels_config(request)
|
||||
existing = channels_config.get(provider)
|
||||
runtime_config = dict(existing) if isinstance(existing, dict) else {}
|
||||
values = _required_runtime_values(provider, body.values, runtime_config)
|
||||
runtime_config["enabled"] = True
|
||||
|
||||
for key in _RUNTIME_REQUIREMENTS[provider]:
|
||||
runtime_config[key] = values[key]
|
||||
|
||||
if provider == "telegram":
|
||||
runtime_config["bot_username"] = values["bot_username"]
|
||||
provider_config.bot_username = values["bot_username"]
|
||||
request.app.state.channel_connections_config = config
|
||||
|
||||
channels_config[provider] = runtime_config
|
||||
request.app.state.channels_config = channels_config
|
||||
|
||||
started = await _restart_runtime_channel_if_available(provider, runtime_config)
|
||||
if started is False:
|
||||
display_name = _PROVIDER_META[provider]["display_name"]
|
||||
raise HTTPException(status_code=400, detail=f"Failed to start {display_name} channel. Check the values and try again.")
|
||||
|
||||
_get_runtime_config_store(request).set_provider_config(provider, runtime_config)
|
||||
|
||||
return _provider_response(config, channels_config, provider, _PROVIDER_META[provider])
|
||||
@@ -98,6 +98,7 @@ class MemoryConfigResponse(BaseModel):
|
||||
fact_confidence_threshold: float = Field(..., description="Minimum confidence threshold for facts")
|
||||
injection_enabled: bool = Field(..., description="Whether memory injection is enabled")
|
||||
max_injection_tokens: int = Field(..., description="Maximum tokens for memory injection")
|
||||
token_counting: str = Field(..., description="Token counting strategy for memory injection ('tiktoken' or 'char')")
|
||||
|
||||
|
||||
class MemoryStatusResponse(BaseModel):
|
||||
@@ -310,7 +311,8 @@ async def get_memory_config_endpoint() -> MemoryConfigResponse:
|
||||
"max_facts": 100,
|
||||
"fact_confidence_threshold": 0.7,
|
||||
"injection_enabled": true,
|
||||
"max_injection_tokens": 2000
|
||||
"max_injection_tokens": 2000,
|
||||
"token_counting": "tiktoken"
|
||||
}
|
||||
```
|
||||
"""
|
||||
@@ -323,6 +325,7 @@ async def get_memory_config_endpoint() -> MemoryConfigResponse:
|
||||
fact_confidence_threshold=config.fact_confidence_threshold,
|
||||
injection_enabled=config.injection_enabled,
|
||||
max_injection_tokens=config.max_injection_tokens,
|
||||
token_counting=config.token_counting,
|
||||
)
|
||||
|
||||
|
||||
@@ -351,6 +354,7 @@ async def get_memory_status() -> MemoryStatusResponse:
|
||||
fact_confidence_threshold=config.fact_confidence_threshold,
|
||||
injection_enabled=config.injection_enabled,
|
||||
max_injection_tokens=config.max_injection_tokens,
|
||||
token_counting=config.token_counting,
|
||||
),
|
||||
data=MemoryResponse(**memory_data),
|
||||
)
|
||||
|
||||
@@ -22,6 +22,7 @@ from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.deps import get_checkpointer
|
||||
from app.gateway.internal_auth import get_trusted_internal_owner_user_id
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.config.paths import Paths, get_paths
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
@@ -257,11 +258,19 @@ async def create_thread(body: ThreadCreateRequest, request: Request) -> ThreadRe
|
||||
thread_store = get_thread_store(request)
|
||||
thread_id = body.thread_id or str(uuid.uuid4())
|
||||
now = now_iso()
|
||||
thread_owner_user_id = get_trusted_internal_owner_user_id(request)
|
||||
thread_owner_kwargs = {"user_id": thread_owner_user_id} if thread_owner_user_id else {}
|
||||
# ``body.metadata`` is already stripped of server-reserved keys by
|
||||
# ``ThreadCreateRequest._strip_reserved`` — see the model definition.
|
||||
|
||||
# Idempotency: return existing record when already present
|
||||
existing_record = await thread_store.get(thread_id)
|
||||
existing_record = await thread_store.get(thread_id, **thread_owner_kwargs)
|
||||
if existing_record is None and thread_owner_user_id:
|
||||
unscoped_record = await thread_store.get(thread_id, user_id=None)
|
||||
if unscoped_record is not None:
|
||||
if unscoped_record.get("user_id") != thread_owner_user_id:
|
||||
await thread_store.update_owner(thread_id, thread_owner_user_id, user_id=None)
|
||||
existing_record = await thread_store.get(thread_id, **thread_owner_kwargs)
|
||||
if existing_record is not None:
|
||||
return ThreadResponse(
|
||||
thread_id=thread_id,
|
||||
@@ -276,6 +285,7 @@ async def create_thread(body: ThreadCreateRequest, request: Request) -> ThreadRe
|
||||
await thread_store.create(
|
||||
thread_id,
|
||||
assistant_id=getattr(body, "assistant_id", None),
|
||||
**thread_owner_kwargs,
|
||||
metadata=body.metadata,
|
||||
)
|
||||
except Exception:
|
||||
|
||||
@@ -12,6 +12,7 @@ import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
@@ -19,7 +20,7 @@ from langchain_core.messages import BaseMessage
|
||||
from langchain_core.messages.utils import convert_to_messages
|
||||
|
||||
from app.gateway.deps import get_run_context, get_run_manager, get_stream_bridge
|
||||
from app.gateway.internal_auth import INTERNAL_SYSTEM_ROLE
|
||||
from app.gateway.internal_auth import INTERNAL_SYSTEM_ROLE, get_trusted_internal_owner_user_id
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.runtime import (
|
||||
@@ -35,6 +36,7 @@ from deerflow.runtime import (
|
||||
run_agent,
|
||||
)
|
||||
from deerflow.runtime.runs.naming import resolve_root_run_name
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -315,6 +317,24 @@ async def start_run(
|
||||
detail=f"Model {model_name!r} is not in the configured model allowlist",
|
||||
)
|
||||
|
||||
owner_user_id = get_trusted_internal_owner_user_id(request)
|
||||
# Stateless run endpoints carry thread_id in the request *body*, so the
|
||||
# @require_permission(owner_check=True) decorator -- which resolves ownership
|
||||
# from the path param -- cannot protect them. Enforce thread ownership here,
|
||||
# before any run is created, so one user cannot start runs on (or read /wait
|
||||
# checkpoint state from) another user's thread. Missing rows (auto-created
|
||||
# temp threads) and NULL-owner rows (shared / pre-auth data) stay accessible
|
||||
# via check_access; only a thread already owned by another user is rejected
|
||||
# with 404, matching thread_runs.py's anti-enumeration behaviour. Internal
|
||||
# channel runs act on behalf of IM users they do not own (see
|
||||
# inject_authenticated_user_context), so the internal system role is exempt.
|
||||
user = getattr(request.state, "user", None)
|
||||
if user is not None and getattr(user, "system_role", None) != INTERNAL_SYSTEM_ROLE:
|
||||
if not await run_ctx.thread_store.check_access(thread_id, str(user.id)):
|
||||
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
|
||||
|
||||
owner_context_token = set_current_user(SimpleNamespace(id=owner_user_id)) if owner_user_id else None
|
||||
try:
|
||||
try:
|
||||
record = await run_mgr.create_or_reject(
|
||||
thread_id,
|
||||
@@ -324,6 +344,7 @@ async def start_run(
|
||||
kwargs={"input": body.input, "config": body.config},
|
||||
multitask_strategy=body.multitask_strategy,
|
||||
model_name=model_name,
|
||||
user_id=owner_user_id,
|
||||
)
|
||||
except ConflictError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
@@ -335,6 +356,12 @@ async def start_run(
|
||||
# (e.g. stateless runs).
|
||||
try:
|
||||
existing = await run_ctx.thread_store.get(thread_id)
|
||||
if existing is None and owner_user_id:
|
||||
unscoped_existing = await run_ctx.thread_store.get(thread_id, user_id=None)
|
||||
if unscoped_existing is not None:
|
||||
if unscoped_existing.get("user_id") != owner_user_id:
|
||||
await run_ctx.thread_store.update_owner(thread_id, owner_user_id, user_id=None)
|
||||
existing = await run_ctx.thread_store.get(thread_id)
|
||||
if existing is None:
|
||||
await run_ctx.thread_store.create(
|
||||
thread_id,
|
||||
@@ -381,6 +408,9 @@ async def start_run(
|
||||
# after the run completes.
|
||||
|
||||
return record
|
||||
finally:
|
||||
if owner_context_token is not None:
|
||||
reset_current_user(owner_context_token)
|
||||
|
||||
|
||||
async def sse_consumer(
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
# IM Channel Connections
|
||||
|
||||
DeerFlow supports user-owned IM channel bindings for Telegram, Slack, Discord, Feishu/Lark, DingTalk, WeChat, and WeCom. The feature reuses the existing `channels.*` runtime configuration, so it works in local and private deployments with the same outbound transports already supported by DeerFlow.
|
||||
|
||||
No public IP, OAuth callback URL, or provider webhook is required in this implementation.
|
||||
|
||||
## Configuration
|
||||
|
||||
Configure the actual IM bots under the existing `channels` block:
|
||||
|
||||
```yaml
|
||||
channels:
|
||||
telegram:
|
||||
enabled: true
|
||||
bot_token: $TELEGRAM_BOT_TOKEN
|
||||
|
||||
slack:
|
||||
enabled: true
|
||||
bot_token: $SLACK_BOT_TOKEN
|
||||
app_token: $SLACK_APP_TOKEN
|
||||
|
||||
discord:
|
||||
enabled: true
|
||||
bot_token: $DISCORD_BOT_TOKEN
|
||||
|
||||
feishu:
|
||||
enabled: true
|
||||
app_id: $FEISHU_APP_ID
|
||||
app_secret: $FEISHU_APP_SECRET
|
||||
|
||||
dingtalk:
|
||||
enabled: true
|
||||
client_id: $DINGTALK_CLIENT_ID
|
||||
client_secret: $DINGTALK_CLIENT_SECRET
|
||||
|
||||
wechat:
|
||||
enabled: true
|
||||
bot_token: $WECHAT_BOT_TOKEN
|
||||
|
||||
wecom:
|
||||
enabled: true
|
||||
bot_id: $WECOM_BOT_ID
|
||||
bot_secret: $WECOM_BOT_SECRET
|
||||
```
|
||||
|
||||
Then enable user bindings in `channel_connections`:
|
||||
|
||||
```yaml
|
||||
channel_connections:
|
||||
enabled: true
|
||||
|
||||
telegram:
|
||||
enabled: true
|
||||
bot_username: $TELEGRAM_BOT_USERNAME
|
||||
|
||||
slack:
|
||||
enabled: true
|
||||
|
||||
discord:
|
||||
enabled: true
|
||||
|
||||
feishu:
|
||||
enabled: true
|
||||
|
||||
dingtalk:
|
||||
enabled: true
|
||||
|
||||
wechat:
|
||||
enabled: true
|
||||
|
||||
wecom:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
`channel_connections` does not duplicate provider secrets. It only controls the browser-facing connect UI and stores per-user binding records. Telegram needs `bot_username` only so the frontend can open a deep link.
|
||||
|
||||
## Connect Flow
|
||||
|
||||
Telegram:
|
||||
|
||||
- The frontend creates a short one-time code.
|
||||
- The Connect button opens `https://t.me/<bot_username>?start=<code>`.
|
||||
- The existing Telegram long-polling worker receives `/start <code>` and binds that Telegram chat/user to the current DeerFlow user.
|
||||
|
||||
Slack:
|
||||
|
||||
- The frontend creates a short one-time code.
|
||||
- The UI shows `Send /connect <code> to the DeerFlow Slack bot.`
|
||||
- The existing Slack Socket Mode worker receives the message and binds the Slack user/team to the current DeerFlow user.
|
||||
|
||||
Discord:
|
||||
|
||||
- The frontend creates a short one-time code.
|
||||
- The UI shows `Send /connect <code> to the DeerFlow Discord bot.`
|
||||
- The existing Discord Gateway worker receives the message and binds the Discord user/guild to the current DeerFlow user.
|
||||
|
||||
Feishu/Lark, DingTalk, WeChat, and WeCom:
|
||||
|
||||
- The frontend creates a short one-time code.
|
||||
- The UI shows `Send /connect <code> to the DeerFlow <Provider> bot.`
|
||||
- The already-running long-connection or polling worker receives the message and binds the platform user/workspace identity to the current DeerFlow user.
|
||||
|
||||
Codes expire after 10 minutes and are single-use.
|
||||
|
||||
## Runtime Model
|
||||
|
||||
Connection records live in SQL tables under `deerflow.persistence.channel_connections`:
|
||||
|
||||
- `channel_connections`: owner user, provider identity, workspace/guild/team, status, metadata.
|
||||
- `channel_oauth_states`: one-time connect codes and Telegram deep-link state.
|
||||
- `channel_conversations`: connection-scoped IM conversation to DeerFlow thread mapping.
|
||||
- `channel_credentials`: reserved for future provider-token flows, not used by the local/private binding flow.
|
||||
|
||||
Incoming messages that resolve to a connection carry `connection_id`, `owner_user_id`, and `workspace_id`. `ChannelManager` uses `owner_user_id` as the DeerFlow run user id and preserves the raw platform user id as `channel_user_id`.
|
||||
|
||||
## Security Notes
|
||||
|
||||
- Browser APIs remain authenticated and CSRF-protected.
|
||||
- Connect codes are random, short-lived, and single-use.
|
||||
- Provider bot tokens remain in `channels.*` and are never returned to the browser.
|
||||
- This implementation does not add public provider callback or webhook routes.
|
||||
@@ -31,7 +31,8 @@ Current injection format:
|
||||
|
||||
Token counting:
|
||||
- Uses `tiktoken` (`cl100k_base`) when available
|
||||
- Falls back to `len(text) // 4` if tokenizer import fails
|
||||
- Falls back to a network-free CJK-aware character estimate if tokenizer import or encoding load fails
|
||||
(CJK characters count as ~2 chars/token, other characters as ~4 chars/token)
|
||||
|
||||
## Known Gap
|
||||
|
||||
|
||||
@@ -50,18 +50,22 @@ gateway's own run/event stores using the request's auth context, so the real
|
||||
## How replay works
|
||||
|
||||
`tests/replay_provider.py::ReplayChatModel` returns recorded assistant turns keyed
|
||||
by a **normalized hash of the conversation** (human / ai / tool messages — role,
|
||||
text, tool-call name+args; with `<system-reminder>`, dates, UUIDs, tmp paths
|
||||
stripped). A miss raises loudly rather than passing silently.
|
||||
by a **normalized hash of the model caller + conversation**. The conversation is
|
||||
human / ai / tool messages — role, text, tool-call name+args; with
|
||||
`<system-reminder>`, dates, UUIDs, tmp paths stripped. The caller is the stable
|
||||
source of the model call (`lead_agent`, `middleware:title`, `suggest_agent`,
|
||||
`subagent:*`, etc.). A miss raises loudly rather than passing silently.
|
||||
|
||||
**The system prompt is excluded from the match key.** The lead-agent system
|
||||
prompt is a living, frequently-edited implementation detail — its wording changes
|
||||
across PRs (e.g. #3195 added a "File Editing Workflow" section). Hashing it would
|
||||
make every fixture go stale and red-fail unrelated PRs the moment anyone edits the
|
||||
prompt. The conversation flow (user input → tool calls → results → answer) is the
|
||||
stable contract that identifies a recorded turn. (This mirrors how open-design's
|
||||
mock picker keys on the user prompt, not the system internals.) Combined with
|
||||
pinning skills + extensions empty and disabling memory/summarization
|
||||
stable contract that identifies a recorded turn. The caller still stays in the
|
||||
key so two different model users with identical conversation text do not compete
|
||||
for the same replay bucket. (This mirrors how open-design's mock picker keys on
|
||||
the user prompt, not the system internals.) Combined with pinning skills +
|
||||
extensions empty and disabling memory/summarization
|
||||
(`tests/_replay_fixture.py::build_config_yaml`), a fixture replays the same across
|
||||
machines, days, prompt edits, and CI. Replaying needs **no API key**.
|
||||
|
||||
|
||||
@@ -127,8 +127,8 @@ complex_agent = create_agent_for_task("high")
|
||||
## How It Works
|
||||
|
||||
1. When `make_lead_agent(config)` is called, it extracts `is_plan_mode` from `config.configurable`
|
||||
2. The config is passed to `_build_middlewares(config)`
|
||||
3. `_build_middlewares()` reads `is_plan_mode` and calls `_create_todo_list_middleware(is_plan_mode)`
|
||||
2. The config is passed to `build_middlewares(config)`
|
||||
3. `build_middlewares()` reads `is_plan_mode` and calls `_create_todo_list_middleware(is_plan_mode)`
|
||||
4. If `is_plan_mode=True`, a `TodoListMiddleware` instance is created and added to the middleware chain
|
||||
5. The middleware automatically adds a `write_todos` tool to the agent's toolset
|
||||
6. The agent can use this tool to manage tasks during execution
|
||||
@@ -141,7 +141,7 @@ make_lead_agent(config)
|
||||
│
|
||||
├─> Extracts: is_plan_mode = config.configurable.get("is_plan_mode", False)
|
||||
│
|
||||
└─> _build_middlewares(config)
|
||||
└─> build_middlewares(config)
|
||||
│
|
||||
├─> ThreadDataMiddleware
|
||||
├─> SandboxMiddleware
|
||||
@@ -156,7 +156,7 @@ make_lead_agent(config)
|
||||
### Agent Module
|
||||
- **Location**: `packages/harness/deerflow/agents/lead_agent/agent.py`
|
||||
- **Function**: `_create_todo_list_middleware(is_plan_mode: bool)` - Creates TodoListMiddleware if plan mode is enabled
|
||||
- **Function**: `_build_middlewares(config: RunnableConfig)` - Builds middleware chain based on runtime config
|
||||
- **Function**: `build_middlewares(config: RunnableConfig)` - Builds middleware chain based on runtime config
|
||||
- **Function**: `make_lead_agent(config: RunnableConfig)` - Creates agent with appropriate middlewares
|
||||
|
||||
### Runtime Configuration
|
||||
|
||||
@@ -49,6 +49,8 @@ from deerflow.tracing import build_tracing_callbacks
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BOOTSTRAP_SKILL_NAMES = {"bootstrap"}
|
||||
|
||||
|
||||
def _get_runtime_config(config: RunnableConfig) -> dict:
|
||||
"""Merge legacy configurable options with LangGraph runtime context."""
|
||||
@@ -265,21 +267,31 @@ Being proactive with task management demonstrates thoroughness and ensures all r
|
||||
# ViewImageMiddleware should be before ClarificationMiddleware to inject image details before LLM
|
||||
# ToolErrorHandlingMiddleware should be before ClarificationMiddleware to convert tool exceptions to ToolMessages
|
||||
# ClarificationMiddleware should be last to intercept clarification requests after model calls
|
||||
def _build_middlewares(
|
||||
def build_middlewares(
|
||||
config: RunnableConfig,
|
||||
model_name: str | None,
|
||||
agent_name: str | None = None,
|
||||
custom_middlewares: list[AgentMiddleware] | None = None,
|
||||
*,
|
||||
available_skills: set[str] | None = None,
|
||||
app_config: AppConfig | None = None,
|
||||
deferred_setup=None,
|
||||
):
|
||||
"""Build middleware chain based on runtime configuration.
|
||||
"""Build the lead-agent middleware chain based on runtime configuration.
|
||||
|
||||
Public entry point for the lead agent's full middleware composition. Used by
|
||||
``make_lead_agent`` and by the embedded ``DeerFlowClient`` (a lead-agent variant
|
||||
that needs the identical chain). Keep this name stable: it is imported across a
|
||||
module boundary, so renames/signature changes ripple into ``client.py``.
|
||||
|
||||
Args:
|
||||
config: Runtime configuration containing configurable options like is_plan_mode.
|
||||
model_name: Resolved runtime model name; gates vision-only middleware.
|
||||
agent_name: If provided, MemoryMiddleware will use per-agent memory storage.
|
||||
custom_middlewares: Optional list of custom middlewares to inject into the chain.
|
||||
app_config: Explicit AppConfig; falls back to ``get_app_config()`` when omitted.
|
||||
deferred_setup: Optional deferred-MCP-tool setup that attaches
|
||||
``DeferredToolFilterMiddleware`` when ``tool_search`` is enabled.
|
||||
|
||||
Returns:
|
||||
List of middleware instances.
|
||||
@@ -293,6 +305,13 @@ def _build_middlewares(
|
||||
|
||||
middlewares.append(DynamicContextMiddleware(agent_name=agent_name, app_config=resolved_app_config))
|
||||
|
||||
# Deterministically load a full SKILL.md when the user starts the turn with
|
||||
# /skill-name. This keeps the base system prompt metadata-only while giving
|
||||
# explicit user activation priority over model-side relevance guessing.
|
||||
from deerflow.agents.middlewares.skill_activation_middleware import SkillActivationMiddleware
|
||||
|
||||
middlewares.append(SkillActivationMiddleware(available_skills=available_skills, app_config=resolved_app_config))
|
||||
|
||||
# Add summarization middleware if enabled
|
||||
summarization_middleware = _create_summarization_middleware(app_config=resolved_app_config)
|
||||
if summarization_middleware is not None:
|
||||
@@ -360,7 +379,7 @@ def _build_middlewares(
|
||||
|
||||
def _available_skill_names(agent_config, is_bootstrap: bool) -> set[str] | None:
|
||||
if is_bootstrap:
|
||||
return {"bootstrap"}
|
||||
return set(_BOOTSTRAP_SKILL_NAMES)
|
||||
if agent_config and agent_config.skills is not None:
|
||||
return set(agent_config.skills)
|
||||
return None
|
||||
@@ -466,17 +485,25 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
|
||||
if is_bootstrap:
|
||||
# Special bootstrap agent with minimal prompt for initial custom agent creation flow
|
||||
# Keep the bootstrap skill set intentionally narrow so agent creation
|
||||
# remains deterministic before the custom agent's own config exists.
|
||||
raw_tools = get_available_tools(model_name=model_name, subagent_enabled=subagent_enabled, app_config=resolved_app_config) + [setup_agent]
|
||||
filtered = filter_tools_by_skill_allowed_tools(raw_tools, skills_for_tool_policy)
|
||||
final_tools, setup = assemble_deferred_tools(filtered, enabled=resolved_app_config.tool_search.enabled)
|
||||
return create_agent(
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=resolved_app_config, attach_tracing=False),
|
||||
tools=final_tools,
|
||||
middleware=_build_middlewares(config, model_name=model_name, app_config=resolved_app_config, deferred_setup=setup),
|
||||
middleware=build_middlewares(
|
||||
config,
|
||||
model_name=model_name,
|
||||
available_skills=set(_BOOTSTRAP_SKILL_NAMES),
|
||||
app_config=resolved_app_config,
|
||||
deferred_setup=setup,
|
||||
),
|
||||
system_prompt=apply_prompt_template(
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
available_skills=set(["bootstrap"]),
|
||||
available_skills=set(_BOOTSTRAP_SKILL_NAMES),
|
||||
app_config=resolved_app_config,
|
||||
deferred_names=setup.deferred_names,
|
||||
),
|
||||
@@ -493,12 +520,19 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
return create_agent(
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, app_config=resolved_app_config, attach_tracing=False),
|
||||
tools=final_tools,
|
||||
middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_name, app_config=resolved_app_config, deferred_setup=setup),
|
||||
middleware=build_middlewares(
|
||||
config,
|
||||
model_name=model_name,
|
||||
agent_name=agent_name,
|
||||
available_skills=available_skills,
|
||||
app_config=resolved_app_config,
|
||||
deferred_setup=setup,
|
||||
),
|
||||
system_prompt=apply_prompt_template(
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
agent_name=agent_name,
|
||||
available_skills=set(agent_config.skills) if agent_config and agent_config.skills is not None else None,
|
||||
available_skills=available_skills,
|
||||
app_config=resolved_app_config,
|
||||
deferred_names=setup.deferred_names,
|
||||
),
|
||||
|
||||
@@ -586,7 +586,11 @@ def _get_memory_context(agent_name: str | None = None, *, app_config: AppConfig
|
||||
return ""
|
||||
|
||||
memory_data = get_memory_data(agent_name, user_id=get_effective_user_id())
|
||||
memory_content = format_memory_for_injection(memory_data, max_tokens=config.max_injection_tokens)
|
||||
memory_content = format_memory_for_injection(
|
||||
memory_data,
|
||||
max_tokens=config.max_injection_tokens,
|
||||
use_tiktoken=(config.token_counting == "tiktoken"),
|
||||
)
|
||||
|
||||
if not memory_content.strip():
|
||||
return ""
|
||||
@@ -625,6 +629,11 @@ You have access to skills that provide optimized workflows for specific tasks. E
|
||||
4. Load referenced resources only when needed during execution
|
||||
5. Follow the skill's instructions precisely
|
||||
|
||||
**Explicit Slash Skill Activation:**
|
||||
- If the user starts a request with `/<skill-name>`, that skill was explicitly requested for the current turn.
|
||||
- Follow the activated skill before choosing a general workflow.
|
||||
- The runtime injects the activated skill content for explicit slash activations; do not call `read_file` for that SKILL.md again unless the injected skill references supporting resources you need.
|
||||
|
||||
**Skills are located at:** {container_base_path}
|
||||
{skill_evolution_section}
|
||||
{skills_list}
|
||||
|
||||
@@ -5,7 +5,9 @@ from __future__ import annotations
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, cast
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -169,7 +171,26 @@ Return ONLY valid JSON."""
|
||||
# subsequent calls are a dict lookup (no network I/O). Pre-warming at
|
||||
# startup via :func:`warm_tiktoken_cache` avoids blocking a request on the
|
||||
# (potentially slow) first ``get_encoding`` call.
|
||||
_tiktoken_encoding_cache: dict[str, tiktoken.Encoding] = {}
|
||||
#
|
||||
# A *failed* load is cached as a ``(None, monotonic_timestamp)`` tuple so that
|
||||
# a network-restricted environment does not re-attempt the blocking BPE
|
||||
# download on every subsequent call. After ``_TIKTOKEN_RETRY_COOLDOWN_S`` the
|
||||
# failure is allowed to expire so a transient network outage can self-heal back
|
||||
# to accurate tiktoken counting without a process restart. A load already in
|
||||
# progress is cached as ``_TIKTOKEN_ENCODING_LOADING`` so concurrent callers
|
||||
# fall back immediately instead of spawning more blocking
|
||||
# ``tiktoken.get_encoding`` threads. Use the ``memory.token_counting: char``
|
||||
# config to skip tiktoken entirely.
|
||||
_TIKTOKEN_ENCODING_MISSING = object()
|
||||
_TIKTOKEN_ENCODING_LOADING = object()
|
||||
# Cooldown before a *failed* tiktoken load is re-attempted. This is an internal
|
||||
# tuning constant rather than a user-facing config: it only affects how quickly
|
||||
# the default ``tiktoken`` mode self-heals after a transient network outage.
|
||||
# Deployments that want to avoid tiktoken's network dependency entirely should
|
||||
# set ``memory.token_counting: char`` instead of tuning this value.
|
||||
_TIKTOKEN_RETRY_COOLDOWN_S = 600.0
|
||||
_tiktoken_encoding_cache: dict[str, Any] = {}
|
||||
_tiktoken_encoding_cache_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_tiktoken_encoding(encoding_name: str = "cl100k_base") -> tiktoken.Encoding | None:
|
||||
@@ -181,44 +202,91 @@ def _get_tiktoken_encoding(encoding_name: str = "cl100k_base") -> tiktoken.Encod
|
||||
download can block for tens of minutes before the OS TCP timeout kicks in.
|
||||
The caller must therefore be prepared for this to block and should run it
|
||||
off the event loop (e.g. via ``asyncio.to_thread``).
|
||||
|
||||
A failed load is remembered (with a timestamp) so subsequent calls fall
|
||||
back immediately to character-based estimation instead of re-triggering the
|
||||
blocking download. The failure expires after ``_TIKTOKEN_RETRY_COOLDOWN_S``
|
||||
so a transient outage can self-heal without a restart. A load already in
|
||||
progress is also remembered so that a timed-out caller does not leave a
|
||||
window where later requests start more blocking ``get_encoding`` calls.
|
||||
"""
|
||||
if not TIKTOKEN_AVAILABLE:
|
||||
return None
|
||||
|
||||
cached = _tiktoken_encoding_cache.get(encoding_name)
|
||||
if cached is not None:
|
||||
return cached
|
||||
with _tiktoken_encoding_cache_lock:
|
||||
cached = _tiktoken_encoding_cache.get(encoding_name, _TIKTOKEN_ENCODING_MISSING)
|
||||
if cached is _TIKTOKEN_ENCODING_LOADING:
|
||||
return None
|
||||
if isinstance(cached, tuple):
|
||||
# Cached failure: (None, failed_at). Retry only after cooldown.
|
||||
_, failed_at = cached
|
||||
if time.monotonic() - failed_at < _TIKTOKEN_RETRY_COOLDOWN_S:
|
||||
return None
|
||||
cached = _TIKTOKEN_ENCODING_MISSING
|
||||
if cached is not _TIKTOKEN_ENCODING_MISSING:
|
||||
return cast("tiktoken.Encoding", cached)
|
||||
_tiktoken_encoding_cache[encoding_name] = _TIKTOKEN_ENCODING_LOADING
|
||||
|
||||
try:
|
||||
encoding = tiktoken.get_encoding(encoding_name)
|
||||
_tiktoken_encoding_cache[encoding_name] = encoding
|
||||
return encoding
|
||||
except Exception:
|
||||
logger.warning("Failed to load tiktoken encoding %r; falling back to char-based estimation", encoding_name, exc_info=True)
|
||||
with _tiktoken_encoding_cache_lock:
|
||||
_tiktoken_encoding_cache[encoding_name] = (None, time.monotonic())
|
||||
return None
|
||||
|
||||
with _tiktoken_encoding_cache_lock:
|
||||
_tiktoken_encoding_cache[encoding_name] = encoding
|
||||
return encoding
|
||||
|
||||
def _count_tokens(text: str, encoding_name: str = "cl100k_base") -> int:
|
||||
|
||||
def _char_based_token_estimate(text: str) -> int:
|
||||
"""Network-free token estimate that accounts for CJK density.
|
||||
|
||||
The plain ``len(text) // 4`` heuristic is reasonable for English/code
|
||||
(~4 chars per token) but significantly under-estimates token counts for
|
||||
Chinese, Japanese, and Korean text, where the ratio is closer to 1.5-2
|
||||
characters per token. Counting CJK characters separately (~2 chars per
|
||||
token) avoids over-filling the injection budget for CJK-heavy memory
|
||||
content.
|
||||
"""
|
||||
cjk = sum(
|
||||
1
|
||||
for ch in text
|
||||
if "\u4e00" <= ch <= "\u9fff" # CJK Unified Ideographs
|
||||
or "\u3040" <= ch <= "\u30ff" # Hiragana + Katakana
|
||||
or "\uac00" <= ch <= "\ud7a3" # Hangul syllables
|
||||
)
|
||||
return (len(text) - cjk) // 4 + cjk // 2
|
||||
|
||||
|
||||
def _count_tokens(text: str, encoding_name: str = "cl100k_base", *, use_tiktoken: bool = True) -> int:
|
||||
"""Count tokens in text using tiktoken.
|
||||
|
||||
Args:
|
||||
text: The text to count tokens for.
|
||||
encoding_name: The encoding to use (default: cl100k_base for GPT-4/3.5).
|
||||
use_tiktoken: When ``False``, skip tiktoken entirely and use the
|
||||
network-free character-based estimate. This guarantees no BPE
|
||||
download is attempted (see ``memory.token_counting`` config).
|
||||
|
||||
Returns:
|
||||
The number of tokens in the text.
|
||||
"""
|
||||
if not use_tiktoken:
|
||||
return _char_based_token_estimate(text)
|
||||
|
||||
encoding = _get_tiktoken_encoding(encoding_name)
|
||||
if encoding is None:
|
||||
# Fallback to character-based estimation if tiktoken is not available
|
||||
# or the encoding failed to load.
|
||||
return len(text) // 4
|
||||
# Fallback to CJK-aware character estimation if tiktoken is not
|
||||
# available or the encoding failed to load.
|
||||
return _char_based_token_estimate(text)
|
||||
|
||||
try:
|
||||
return len(encoding.encode(text))
|
||||
except Exception:
|
||||
# Fallback to character-based estimation on error
|
||||
return len(text) // 4
|
||||
# Fallback to CJK-aware character estimation on error.
|
||||
return _char_based_token_estimate(text)
|
||||
|
||||
|
||||
def warm_tiktoken_cache() -> bool:
|
||||
@@ -248,12 +316,15 @@ def _coerce_confidence(value: Any, default: float = 0.0) -> float:
|
||||
return max(0.0, min(1.0, confidence))
|
||||
|
||||
|
||||
def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2000) -> str:
|
||||
def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2000, *, use_tiktoken: bool = True) -> str:
|
||||
"""Format memory data for injection into system prompt.
|
||||
|
||||
Args:
|
||||
memory_data: The memory data dictionary.
|
||||
max_tokens: Maximum tokens to use (counted via tiktoken for accuracy).
|
||||
use_tiktoken: When ``False``, all token counting uses the network-free
|
||||
character-based estimate instead of tiktoken (see
|
||||
``memory.token_counting`` config). Defaults to ``True``.
|
||||
|
||||
Returns:
|
||||
Formatted memory string for system prompt injection.
|
||||
@@ -315,10 +386,10 @@ def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2
|
||||
# Compute token count for existing sections once, then account
|
||||
# incrementally for each fact line to avoid full-string re-tokenization.
|
||||
base_text = "\n\n".join(sections)
|
||||
base_tokens = _count_tokens(base_text) if base_text else 0
|
||||
base_tokens = _count_tokens(base_text, use_tiktoken=use_tiktoken) if base_text else 0
|
||||
# Account for the separator between existing sections and the facts section.
|
||||
facts_header = "Facts:\n"
|
||||
separator_tokens = _count_tokens("\n\n" + facts_header) if base_text else _count_tokens(facts_header)
|
||||
separator_tokens = _count_tokens("\n\n" + facts_header, use_tiktoken=use_tiktoken) if base_text else _count_tokens(facts_header, use_tiktoken=use_tiktoken)
|
||||
running_tokens = base_tokens + separator_tokens
|
||||
|
||||
fact_lines: list[str] = []
|
||||
@@ -339,7 +410,7 @@ def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2
|
||||
|
||||
# Each additional line is preceded by a newline (except the first).
|
||||
line_text = ("\n" + line) if fact_lines else line
|
||||
line_tokens = _count_tokens(line_text)
|
||||
line_tokens = _count_tokens(line_text, use_tiktoken=use_tiktoken)
|
||||
|
||||
if running_tokens + line_tokens <= max_tokens:
|
||||
fact_lines.append(line)
|
||||
@@ -355,8 +426,9 @@ def format_memory_for_injection(memory_data: dict[str, Any], max_tokens: int = 2
|
||||
|
||||
result = "\n\n".join(sections)
|
||||
|
||||
# Use accurate token counting with tiktoken
|
||||
token_count = _count_tokens(result)
|
||||
# Use accurate token counting with tiktoken (or the char-based estimate
|
||||
# when use_tiktoken is False).
|
||||
token_count = _count_tokens(result, use_tiktoken=use_tiktoken)
|
||||
if token_count > max_tokens:
|
||||
# Truncate to fit within token limit
|
||||
# Estimate characters to remove based on token ratio
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
"""Middleware for explicit slash skill activation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import html
|
||||
import logging
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, override
|
||||
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain.agents.middleware.types import ModelRequest, ModelResponse
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
|
||||
from deerflow.skills.slash import parse_slash_skill_reference, resolve_slash_skill
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.storage.skill_storage import SkillStorage
|
||||
from deerflow.skills.types import SKILL_MD_FILE
|
||||
from deerflow.utils.messages import get_original_user_content_text
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.config.app_config import AppConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SLASH_SKILL_ACTIVATION_KEY = "slash_skill_activation"
|
||||
_SLASH_SKILL_ACTIVATION_TARGET_ID_KEY = "slash_skill_activation_target_id"
|
||||
_SUMMARY_MESSAGE_NAME = "summary"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Activation:
|
||||
skill_name: str
|
||||
category: str
|
||||
container_file_path: str
|
||||
skill_content: str
|
||||
content_hash: str
|
||||
remaining_text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ActivationResolution:
|
||||
activation: _Activation | None = None
|
||||
failure_message: str | None = None
|
||||
|
||||
|
||||
def is_slash_skill_activation_reminder(message: object) -> bool:
|
||||
"""Return whether a message is hidden slash-skill activation context."""
|
||||
return isinstance(message, HumanMessage) and bool(message.additional_kwargs.get(_SLASH_SKILL_ACTIVATION_KEY))
|
||||
|
||||
|
||||
def _is_user_activation_target(message: object) -> bool:
|
||||
if not isinstance(message, HumanMessage):
|
||||
return False
|
||||
if message.name == _SUMMARY_MESSAGE_NAME:
|
||||
return False
|
||||
if message.additional_kwargs.get("hide_from_ui"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class SkillActivationMiddleware(AgentMiddleware):
|
||||
"""Inject full SKILL.md content when the user explicitly types /skill-name."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
available_skills: set[str] | None = None,
|
||||
app_config: AppConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._available_skills = set(available_skills) if available_skills is not None else None
|
||||
self._app_config = app_config
|
||||
|
||||
def _storage(self) -> SkillStorage:
|
||||
if self._app_config is not None:
|
||||
return get_or_new_skill_storage(app_config=self._app_config)
|
||||
return get_or_new_skill_storage()
|
||||
|
||||
@staticmethod
|
||||
def _read_skill_content(skill_file: Path, skills_root: Path) -> str:
|
||||
if skill_file.name != SKILL_MD_FILE:
|
||||
raise ValueError(f"Expected {SKILL_MD_FILE}, got {skill_file.name}")
|
||||
resolved_root = skills_root.resolve()
|
||||
resolved_file = skill_file.resolve()
|
||||
try:
|
||||
resolved_file.relative_to(resolved_root)
|
||||
except ValueError as exc:
|
||||
raise ValueError("Resolved skill file must stay within the configured skills root.") from exc
|
||||
if not resolved_file.is_file():
|
||||
raise FileNotFoundError(resolved_file)
|
||||
return resolved_file.read_text(encoding="utf-8")
|
||||
|
||||
def _resolve_activation(self, text: str) -> _ActivationResolution | None:
|
||||
reference = parse_slash_skill_reference(text)
|
||||
if reference is None:
|
||||
return None
|
||||
|
||||
storage = self._storage()
|
||||
skills = storage.load_skills(enabled_only=False)
|
||||
skill = next((candidate for candidate in skills if candidate.name == reference.name), None)
|
||||
if skill is None:
|
||||
return _ActivationResolution(failure_message=f"Skill `/{reference.name}` is not installed.")
|
||||
if not skill.enabled:
|
||||
return _ActivationResolution(failure_message=f"Skill `/{reference.name}` is installed but disabled. Enable it before using slash activation.")
|
||||
if self._available_skills is not None and reference.name not in self._available_skills:
|
||||
return _ActivationResolution(failure_message=f"Skill `/{reference.name}` is not available for this agent.")
|
||||
|
||||
resolved = resolve_slash_skill(
|
||||
text,
|
||||
skills,
|
||||
available_skills=self._available_skills,
|
||||
container_base_path=storage.get_container_root(),
|
||||
)
|
||||
if resolved is None:
|
||||
return _ActivationResolution(failure_message=f"Skill `/{reference.name}` could not be resolved.")
|
||||
|
||||
try:
|
||||
skill_content = self._read_skill_content(resolved.skill.skill_file, storage.get_skills_root_path())
|
||||
except (OSError, ValueError):
|
||||
logger.exception("Failed to read slash-activated skill %s", resolved.skill.name)
|
||||
return _ActivationResolution(failure_message=f"Skill `/{reference.name}` could not be loaded safely. Please check the skill installation.")
|
||||
|
||||
content_hash = hashlib.sha256(skill_content.encode("utf-8")).hexdigest()
|
||||
return _ActivationResolution(
|
||||
activation=_Activation(
|
||||
skill_name=resolved.skill.name,
|
||||
category=str(resolved.skill.category),
|
||||
container_file_path=resolved.container_file_path,
|
||||
skill_content=skill_content,
|
||||
content_hash=content_hash,
|
||||
remaining_text=resolved.remaining_text,
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_activation_reminder(activation: _Activation) -> str:
|
||||
user_request = activation.remaining_text or ("No additional task text was provided after the slash skill command. Ask the user what they want to do with this skill if the next step is unclear.")
|
||||
escaped_user_request = html.escape(user_request, quote=False)
|
||||
escaped_skill_content = html.escape(activation.skill_content, quote=False)
|
||||
escaped_skill_name = html.escape(activation.skill_name, quote=True)
|
||||
escaped_category = html.escape(activation.category, quote=True)
|
||||
escaped_path = html.escape(activation.container_file_path, quote=True)
|
||||
escaped_content_hash = html.escape(activation.content_hash, quote=True)
|
||||
return f"""<slash_skill_activation>
|
||||
The user explicitly activated the `{activation.skill_name}` skill for this turn.
|
||||
Treat the task text as:
|
||||
<user_request>
|
||||
{escaped_user_request}
|
||||
</user_request>
|
||||
|
||||
Follow this skill before choosing a general workflow. Load supporting resources from the same skill directory only when needed.
|
||||
|
||||
<skill name="{escaped_skill_name}" category="{escaped_category}" path="{escaped_path}" sha256="{escaped_content_hash}">
|
||||
<skill_content encoding="xml-escaped">
|
||||
{escaped_skill_content}
|
||||
</skill_content>
|
||||
</skill>
|
||||
</slash_skill_activation>"""
|
||||
|
||||
@staticmethod
|
||||
def _has_existing_activation_for_target(messages: list, target_index: int, target: HumanMessage) -> bool:
|
||||
if target_index <= 0:
|
||||
return False
|
||||
|
||||
if target.id:
|
||||
for previous in messages[:target_index]:
|
||||
if not is_slash_skill_activation_reminder(previous):
|
||||
continue
|
||||
target_id = previous.additional_kwargs.get(_SLASH_SKILL_ACTIVATION_TARGET_ID_KEY)
|
||||
if target_id == target.id or previous.id == f"{target.id}__slash_activation":
|
||||
return True
|
||||
|
||||
previous = messages[target_index - 1]
|
||||
return is_slash_skill_activation_reminder(previous)
|
||||
|
||||
def _find_activation_target(self, messages: list) -> tuple[int, HumanMessage, _ActivationResolution] | None:
|
||||
if not messages:
|
||||
return None
|
||||
|
||||
target_index = next((idx for idx in range(len(messages) - 1, -1, -1) if _is_user_activation_target(messages[idx])), None)
|
||||
if target_index is None:
|
||||
return None
|
||||
|
||||
target = messages[target_index]
|
||||
if target is None:
|
||||
return None
|
||||
if self._has_existing_activation_for_target(messages, target_index, target):
|
||||
return None
|
||||
|
||||
content = get_original_user_content_text(target.content, target.additional_kwargs)
|
||||
resolution = self._resolve_activation(content)
|
||||
if resolution is None:
|
||||
return None
|
||||
return target_index, target, resolution
|
||||
|
||||
@staticmethod
|
||||
def _record_activation(request: ModelRequest, activation: _Activation, *, hook: str) -> None:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
context = getattr(runtime, "context", None)
|
||||
journal = context.get("__run_journal") if isinstance(context, dict) else None
|
||||
if journal is None:
|
||||
return
|
||||
try:
|
||||
journal.record_middleware(
|
||||
"skill_activation",
|
||||
name="SkillActivationMiddleware",
|
||||
hook=hook,
|
||||
action="activate",
|
||||
changes={
|
||||
"skill_name": activation.skill_name,
|
||||
"category": activation.category,
|
||||
"path": activation.container_file_path,
|
||||
"content_hash": activation.content_hash,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to record slash skill activation audit event", exc_info=True)
|
||||
|
||||
def _prepare_model_request(self, request: ModelRequest, *, hook: str) -> ModelRequest | AIMessage | None:
|
||||
target_and_resolution = self._find_activation_target(list(request.messages))
|
||||
if target_and_resolution is None:
|
||||
return None
|
||||
|
||||
target_index, target, resolution = target_and_resolution
|
||||
if resolution.failure_message:
|
||||
return AIMessage(content=resolution.failure_message)
|
||||
|
||||
activation = resolution.activation
|
||||
if activation is None:
|
||||
return None
|
||||
|
||||
logger.info(
|
||||
"SkillActivationMiddleware: activating slash skill %s category=%s path=%s hash=%s",
|
||||
activation.skill_name,
|
||||
activation.category,
|
||||
activation.container_file_path,
|
||||
activation.content_hash,
|
||||
)
|
||||
self._record_activation(request, activation, hook=hook)
|
||||
activation_msg = self._make_activation_message(target, self._build_activation_reminder(activation))
|
||||
messages = list(request.messages)
|
||||
messages.insert(target_index, activation_msg)
|
||||
return request.override(messages=messages)
|
||||
|
||||
@staticmethod
|
||||
def _make_activation_message(target: HumanMessage, activation_content: str) -> HumanMessage:
|
||||
stable_id = target.id or str(uuid.uuid4())
|
||||
additional_kwargs = {
|
||||
"hide_from_ui": True,
|
||||
_SLASH_SKILL_ACTIVATION_KEY: True,
|
||||
}
|
||||
if target.id:
|
||||
additional_kwargs[_SLASH_SKILL_ACTIVATION_TARGET_ID_KEY] = target.id
|
||||
return HumanMessage(
|
||||
content=activation_content,
|
||||
id=f"{stable_id}__slash_activation",
|
||||
additional_kwargs=additional_kwargs,
|
||||
)
|
||||
|
||||
@override
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelResponse | AIMessage:
|
||||
prepared = self._prepare_model_request(request, hook="wrap_model_call")
|
||||
if prepared is None:
|
||||
return handler(request)
|
||||
if isinstance(prepared, AIMessage):
|
||||
return prepared
|
||||
return handler(prepared)
|
||||
|
||||
@override
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelResponse | AIMessage:
|
||||
prepared = await asyncio.to_thread(self._prepare_model_request, request, hook="awrap_model_call")
|
||||
if prepared is None:
|
||||
return await handler(request)
|
||||
if isinstance(prepared, AIMessage):
|
||||
return prepared
|
||||
return await handler(prepared)
|
||||
@@ -13,6 +13,7 @@ from langgraph.runtime import Runtime
|
||||
from deerflow.config.paths import Paths, get_paths
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.utils.file_conversion import extract_outline
|
||||
from deerflow.utils.messages import ORIGINAL_USER_CONTENT_KEY, message_content_to_text
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -265,6 +266,8 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
|
||||
|
||||
# Extract original content - handle both string and list formats
|
||||
original_content = last_message.content
|
||||
additional_kwargs = dict(last_message.additional_kwargs or {})
|
||||
additional_kwargs.setdefault(ORIGINAL_USER_CONTENT_KEY, message_content_to_text(original_content))
|
||||
if isinstance(original_content, str):
|
||||
# Simple case: string content, just prepend files message
|
||||
updated_content = f"{files_message}\n\n{original_content}"
|
||||
@@ -285,7 +288,7 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]):
|
||||
content=updated_content,
|
||||
id=last_message.id,
|
||||
name=last_message.name,
|
||||
additional_kwargs=last_message.additional_kwargs,
|
||||
additional_kwargs=additional_kwargs,
|
||||
)
|
||||
|
||||
messages[last_message_index] = updated_message
|
||||
|
||||
@@ -33,7 +33,7 @@ from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from deerflow.agents.lead_agent.agent import _build_middlewares
|
||||
from deerflow.agents.lead_agent.agent import build_middlewares
|
||||
from deerflow.agents.lead_agent.prompt import apply_prompt_template
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.config.agents_config import AGENT_NAME_PATTERN
|
||||
@@ -247,7 +247,15 @@ class DeerFlowClient:
|
||||
# Attaching them again on the model would emit duplicate spans.
|
||||
"model": create_chat_model(name=model_name, thinking_enabled=thinking_enabled, attach_tracing=False),
|
||||
"tools": final_tools,
|
||||
"middleware": _build_middlewares(config, model_name=model_name, agent_name=self._agent_name, custom_middlewares=self._middlewares, deferred_setup=deferred_setup),
|
||||
"middleware": build_middlewares(
|
||||
config,
|
||||
model_name=model_name,
|
||||
agent_name=self._agent_name,
|
||||
available_skills=self._available_skills,
|
||||
custom_middlewares=self._middlewares,
|
||||
app_config=self._app_config,
|
||||
deferred_setup=deferred_setup,
|
||||
),
|
||||
"system_prompt": apply_prompt_template(
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
@@ -1133,6 +1141,7 @@ class DeerFlowClient:
|
||||
"fact_confidence_threshold": config.fact_confidence_threshold,
|
||||
"injection_enabled": config.injection_enabled,
|
||||
"max_injection_tokens": config.max_injection_tokens,
|
||||
"token_counting": config.token_counting,
|
||||
}
|
||||
|
||||
def get_memory_status(self) -> dict:
|
||||
|
||||
@@ -9,7 +9,7 @@ _api_key_warned = False
|
||||
|
||||
|
||||
class JinaClient:
|
||||
async def crawl(self, url: str, return_format: str = "html", timeout: int = 10) -> str:
|
||||
async def crawl(self, url: str, return_format: str = "html", timeout: int = 10, proxy: str | None = None, trust_env: bool = True) -> str:
|
||||
global _api_key_warned
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
@@ -23,7 +23,10 @@ class JinaClient:
|
||||
logger.warning("Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information.")
|
||||
data = {"url": url}
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
client_kwargs: dict[str, object] = {"trust_env": trust_env}
|
||||
if proxy:
|
||||
client_kwargs["proxy"] = proxy
|
||||
async with httpx.AsyncClient(**client_kwargs) as client:
|
||||
response = await client.post("https://r.jina.ai/", headers=headers, json=data, timeout=timeout)
|
||||
|
||||
if response.status_code != 200:
|
||||
|
||||
@@ -9,6 +9,38 @@ from deerflow.utils.readability import ReadabilityExtractor
|
||||
readability_extractor = ReadabilityExtractor()
|
||||
|
||||
|
||||
def _coerce_bool(value: object, default: bool) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if normalized in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def _coerce_timeout(value: object, default: int) -> int:
|
||||
if isinstance(value, bool):
|
||||
return default
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
return default
|
||||
return default
|
||||
|
||||
|
||||
def _coerce_proxy(value: object) -> str | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
proxy = value.strip()
|
||||
return proxy or None
|
||||
|
||||
|
||||
@tool("web_fetch", parse_docstring=True)
|
||||
async def web_fetch_tool(url: str) -> str:
|
||||
"""Fetch the contents of a web page at a given URL.
|
||||
@@ -22,10 +54,14 @@ async def web_fetch_tool(url: str) -> str:
|
||||
"""
|
||||
jina_client = JinaClient()
|
||||
timeout = 10
|
||||
proxy = None
|
||||
trust_env = True
|
||||
config = get_app_config().get_tool_config("web_fetch")
|
||||
if config is not None and "timeout" in config.model_extra:
|
||||
timeout = config.model_extra.get("timeout")
|
||||
html_content = await jina_client.crawl(url, return_format="html", timeout=timeout)
|
||||
if config is not None:
|
||||
timeout = _coerce_timeout(config.model_extra.get("timeout"), timeout)
|
||||
proxy = _coerce_proxy(config.model_extra.get("proxy"))
|
||||
trust_env = _coerce_bool(config.model_extra.get("trust_env"), trust_env)
|
||||
html_content = await jina_client.crawl(url, return_format="html", timeout=timeout, proxy=proxy, trust_env=trust_env)
|
||||
if isinstance(html_content, str) and html_content.startswith("Error:"):
|
||||
return html_content
|
||||
article = await asyncio.to_thread(readability_extractor.extract_article, html_content)
|
||||
|
||||
@@ -67,11 +67,13 @@ def resolve_agent_dir(name: str, *, user_id: str | None = None) -> Path:
|
||||
paths = get_paths()
|
||||
effective_user = user_id or get_effective_user_id()
|
||||
user_path = paths.user_agent_dir(effective_user, name)
|
||||
if user_path.exists():
|
||||
# Require config.yaml to confirm this is a genuine agent directory,
|
||||
# not a leftover from memory/storage writes (see #3390).
|
||||
if user_path.exists() and (user_path / "config.yaml").exists():
|
||||
return user_path
|
||||
|
||||
legacy_path = paths.agent_dir(name)
|
||||
if legacy_path.exists():
|
||||
if legacy_path.exists() and (legacy_path / "config.yaml").exists():
|
||||
return legacy_path
|
||||
|
||||
return user_path
|
||||
|
||||
@@ -7,10 +7,11 @@ from typing import Any, Self
|
||||
|
||||
import yaml
|
||||
from dotenv import load_dotenv
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
|
||||
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
|
||||
from deerflow.config.channel_connections_config import ChannelConnectionsConfig
|
||||
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
|
||||
from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
@@ -116,6 +117,7 @@ class AppConfig(BaseModel):
|
||||
subagents: SubagentsAppConfig = Field(default_factory=SubagentsAppConfig, description="Subagent runtime configuration")
|
||||
guardrails: GuardrailsConfig = Field(default_factory=GuardrailsConfig, description="Guardrail middleware configuration")
|
||||
circuit_breaker: CircuitBreakerConfig = Field(default_factory=CircuitBreakerConfig, description="LLM circuit breaker configuration")
|
||||
channel_connections: ChannelConnectionsConfig = Field(default_factory=ChannelConnectionsConfig, description="User-facing IM channel connection configuration")
|
||||
loop_detection: LoopDetectionConfig = Field(default_factory=LoopDetectionConfig, description="Loop detection middleware configuration")
|
||||
safety_finish_reason: SafetyFinishReasonConfig = Field(default_factory=SafetyFinishReasonConfig, description="Provider safety-filter finish_reason interception middleware configuration")
|
||||
model_config = ConfigDict(extra="allow")
|
||||
@@ -148,6 +150,21 @@ class AppConfig(BaseModel):
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("models", "tools", "tool_groups", mode="before")
|
||||
@classmethod
|
||||
def _coerce_null_list_sections(cls, value: Any) -> Any:
|
||||
"""Treat a present-but-empty config section as an empty list.
|
||||
|
||||
Commenting out every entry under a top-level YAML key — e.g. ``models:``
|
||||
with only comments beneath it, exactly as shipped in
|
||||
``config.example.yaml`` — makes PyYAML parse the value as ``None``.
|
||||
Without this, the documented ``cp config.example.yaml config.yaml``
|
||||
first-run flow crashes with an opaque ``Input should be a valid list``
|
||||
pydantic error. Coercing ``None`` to ``[]`` keeps that flow working and
|
||||
matches the field's own ``default_factory=list``.
|
||||
"""
|
||||
return [] if value is None else value
|
||||
|
||||
@classmethod
|
||||
def resolve_config_path(cls, config_path: str | None = None) -> Path:
|
||||
"""Resolve the config file path.
|
||||
@@ -209,6 +226,11 @@ class AppConfig(BaseModel):
|
||||
config_data["extensions"] = extensions_config.model_dump()
|
||||
|
||||
result = cls.model_validate(config_data)
|
||||
if not result.models:
|
||||
logger.warning(
|
||||
"No models are configured in %s. Add at least one entry under `models:` (see the commented examples in config.example.yaml) or run `make setup`.",
|
||||
resolved_path,
|
||||
)
|
||||
acp_agents = cls._validate_acp_agents(config_data.get("acp_agents", {}))
|
||||
cls._apply_singleton_configs(result, acp_agents)
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Configuration for user-owned IM channel connections."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SlackChannelConnectionConfig(BaseModel):
|
||||
enabled: bool = False
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class TelegramChannelConnectionConfig(BaseModel):
|
||||
enabled: bool = False
|
||||
bot_username: str = ""
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self.bot_username)
|
||||
|
||||
|
||||
class DiscordChannelConnectionConfig(BaseModel):
|
||||
enabled: bool = False
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class BindingCodeChannelConnectionConfig(BaseModel):
|
||||
enabled: bool = False
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class ChannelConnectionsConfig(BaseModel):
|
||||
"""Top-level config for browser-connectable IM channels."""
|
||||
|
||||
enabled: bool = False
|
||||
slack: SlackChannelConnectionConfig = Field(default_factory=SlackChannelConnectionConfig)
|
||||
telegram: TelegramChannelConnectionConfig = Field(default_factory=TelegramChannelConnectionConfig)
|
||||
discord: DiscordChannelConnectionConfig = Field(default_factory=DiscordChannelConnectionConfig)
|
||||
feishu: BindingCodeChannelConnectionConfig = Field(default_factory=BindingCodeChannelConnectionConfig)
|
||||
dingtalk: BindingCodeChannelConnectionConfig = Field(default_factory=BindingCodeChannelConnectionConfig)
|
||||
wechat: BindingCodeChannelConnectionConfig = Field(default_factory=BindingCodeChannelConnectionConfig)
|
||||
wecom: BindingCodeChannelConnectionConfig = Field(default_factory=BindingCodeChannelConnectionConfig)
|
||||
|
||||
def provider_status(self, provider: str) -> dict[str, bool]:
|
||||
config = getattr(self, provider, None)
|
||||
if config is None:
|
||||
return {"enabled": False, "configured": False}
|
||||
enabled = bool(config.enabled)
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"configured": enabled and bool(config.configured),
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Configuration for memory mechanism."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
@@ -60,6 +62,17 @@ class MemoryConfig(BaseModel):
|
||||
le=8000,
|
||||
description="Maximum tokens to use for memory injection",
|
||||
)
|
||||
token_counting: Literal["tiktoken", "char"] = Field(
|
||||
default="tiktoken",
|
||||
description=(
|
||||
"Token counting strategy for memory-injection budgeting. "
|
||||
"'tiktoken' is accurate but the encoding's BPE data may be "
|
||||
"downloaded from a public network endpoint on first use, which "
|
||||
"can block for a long time in network-restricted environments "
|
||||
"(see issue #3402/#3429). 'char' uses a network-free "
|
||||
"CJK-aware character-based estimate and never touches tiktoken."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# Global configuration instance
|
||||
|
||||
@@ -47,7 +47,7 @@ def make_safe_user_id(raw: str) -> str:
|
||||
sanitized = _UNSAFE_USER_ID_CHAR_RE.sub("-", raw)
|
||||
if sanitized == raw:
|
||||
return raw
|
||||
digest = hashlib.sha1(raw.encode("utf-8")).hexdigest()[:_SAFE_USER_ID_DIGEST_HEX_LEN]
|
||||
digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:_SAFE_USER_ID_DIGEST_HEX_LEN]
|
||||
return f"{sanitized}-{digest}"
|
||||
|
||||
|
||||
|
||||
@@ -4,7 +4,20 @@ from pydantic import BaseModel, ConfigDict, Field
|
||||
class VolumeMountConfig(BaseModel):
|
||||
"""Configuration for a volume mount."""
|
||||
|
||||
host_path: str = Field(..., description="Path on the host machine")
|
||||
host_path: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"Source path for the mount. Resolution depends on the active provider: "
|
||||
"``LocalSandboxProvider`` checks this path from the gateway process — in "
|
||||
"``make dev`` that is the host machine, but in Docker deployments "
|
||||
"(``make up`` / docker-compose) it is the path *inside* the "
|
||||
"``deer-flow-gateway`` container, so the host directory must also be "
|
||||
"bind-mounted into the gateway service for the mount to take effect. "
|
||||
"``AioSandboxProvider`` (DooD) passes this value straight to ``docker -v`` "
|
||||
"for the sandbox container, where it is resolved by the host Docker daemon "
|
||||
"from the host machine's perspective."
|
||||
),
|
||||
)
|
||||
container_path: str = Field(..., description="Path inside the container")
|
||||
read_only: bool = Field(default=False, description="Whether the mount is read-only")
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Patched ChatOpenAI adapter for StepFun reasoning models.
|
||||
|
||||
StepFun returns ``reasoning`` (or ``reasoning_content`` with deepseek-style) in
|
||||
both streaming deltas and non-streaming responses. Standard ``ChatOpenAI``
|
||||
ignores these non-standard fields, so reasoning content is silently dropped.
|
||||
This adapter captures reasoning from all response paths and replays it on
|
||||
historical assistant messages for multi-turn tool-call conversations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.language_models import LanguageModelInput
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
from deerflow.models.assistant_payload_replay import (
|
||||
restore_assistant_payloads,
|
||||
restore_reasoning_content,
|
||||
)
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
def _extract_reasoning(value: Any) -> str | object:
|
||||
"""Return reasoning content from a dict/Pydantic object.
|
||||
|
||||
StepFun may return reasoning via ``reasoning`` (default) or
|
||||
``reasoning_content`` (deepseek-style). Check both fields.
|
||||
"""
|
||||
if isinstance(value, Mapping):
|
||||
# Check reasoning_content first (deepseek-style), then reasoning (default)
|
||||
for field in ("reasoning_content", "reasoning"):
|
||||
if field in value and value[field] is not None:
|
||||
return value[field]
|
||||
return _MISSING
|
||||
|
||||
# Pydantic / SDK object attributes
|
||||
for field in ("reasoning_content", "reasoning"):
|
||||
attr = getattr(value, field, _MISSING)
|
||||
if attr is not _MISSING and attr is not None:
|
||||
return attr
|
||||
|
||||
# Some SDK versions store extra fields in model_extra
|
||||
model_extra = getattr(value, "model_extra", None)
|
||||
if isinstance(model_extra, Mapping):
|
||||
for field in ("reasoning_content", "reasoning"):
|
||||
if field in model_extra and model_extra[field] is not None:
|
||||
return model_extra[field]
|
||||
|
||||
return _MISSING
|
||||
|
||||
|
||||
def _with_reasoning_content(message: AIMessage | AIMessageChunk, reasoning: str) -> AIMessage | AIMessageChunk:
|
||||
"""Return a copy of *message* with reasoning_content stored in additional_kwargs."""
|
||||
additional_kwargs = dict(message.additional_kwargs)
|
||||
if additional_kwargs.get("reasoning_content") != reasoning:
|
||||
additional_kwargs["reasoning_content"] = reasoning
|
||||
return message.model_copy(update={"additional_kwargs": additional_kwargs})
|
||||
|
||||
|
||||
def _get_typed_choice_message(response: Any, index: int) -> Any:
|
||||
"""Extract the SDK-typed choice message at *index*, if available."""
|
||||
choices = getattr(response, "choices", None)
|
||||
if choices is None:
|
||||
return None
|
||||
try:
|
||||
return choices[index].message
|
||||
except (AttributeError, IndexError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
class PatchedChatStepFun(ChatOpenAI):
|
||||
"""ChatOpenAI with full reasoning support for StepFun models.
|
||||
|
||||
Captures ``reasoning`` / ``reasoning_content`` from both streaming and
|
||||
non-streaming responses and replays it on historical assistant messages in
|
||||
multi-turn tool-call conversations.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def is_lc_serializable(cls) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def lc_secrets(self) -> dict[str, str]:
|
||||
return {"api_key": "STEPFUN_API_KEY", "openai_api_key": "STEPFUN_API_KEY"}
|
||||
|
||||
# --- Request payload replay ---
|
||||
|
||||
def _get_request_payload(
|
||||
self,
|
||||
input_: LanguageModelInput,
|
||||
*,
|
||||
stop: list[str] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> dict:
|
||||
"""Restore ``reasoning_content`` on historical assistant messages."""
|
||||
original_messages = self._convert_input(input_).to_messages()
|
||||
payload = super()._get_request_payload(input_, stop=stop, **kwargs)
|
||||
|
||||
restore_assistant_payloads(
|
||||
payload.get("messages", []),
|
||||
original_messages,
|
||||
restore_reasoning_content,
|
||||
)
|
||||
|
||||
return payload
|
||||
|
||||
# --- Streaming reasoning capture ---
|
||||
|
||||
def _convert_chunk_to_generation_chunk(
|
||||
self,
|
||||
chunk: dict,
|
||||
default_chunk_class: type,
|
||||
base_generation_info: dict | None,
|
||||
) -> ChatGenerationChunk | None:
|
||||
"""Capture ``reasoning`` / ``reasoning_content`` from streaming deltas."""
|
||||
generation_chunk = super()._convert_chunk_to_generation_chunk(
|
||||
chunk,
|
||||
default_chunk_class,
|
||||
base_generation_info,
|
||||
)
|
||||
if generation_chunk is None:
|
||||
return None
|
||||
|
||||
choices = chunk.get("choices", [])
|
||||
if choices:
|
||||
delta = choices[0].get("delta") or {}
|
||||
reasoning = _extract_reasoning(delta)
|
||||
if reasoning is not _MISSING and isinstance(generation_chunk.message, AIMessageChunk):
|
||||
generation_chunk = ChatGenerationChunk(
|
||||
message=_with_reasoning_content(generation_chunk.message, reasoning),
|
||||
generation_info=generation_chunk.generation_info,
|
||||
)
|
||||
|
||||
return generation_chunk
|
||||
|
||||
# --- Non-streaming reasoning capture ---
|
||||
|
||||
def _create_chat_result(
|
||||
self,
|
||||
response: dict | Any,
|
||||
generation_info: dict | None = None,
|
||||
) -> ChatResult:
|
||||
"""Extract ``reasoning`` / ``reasoning_content`` from non-streaming responses."""
|
||||
result = super()._create_chat_result(response, generation_info)
|
||||
response_dict = response if isinstance(response, dict) else response.model_dump()
|
||||
choices = response_dict.get("choices", [])
|
||||
|
||||
patched_generations: list[ChatGeneration] | None = None
|
||||
for index, generation in enumerate(result.generations):
|
||||
choice = choices[index] if index < len(choices) else {}
|
||||
choice_message = choice.get("message", {}) if isinstance(choice, Mapping) else {}
|
||||
reasoning = _extract_reasoning(choice_message)
|
||||
|
||||
if reasoning is _MISSING and not isinstance(response, dict):
|
||||
reasoning = _extract_reasoning(_get_typed_choice_message(response, index))
|
||||
|
||||
message = generation.message
|
||||
if reasoning is not _MISSING and isinstance(message, AIMessage):
|
||||
if patched_generations is None:
|
||||
patched_generations = list(result.generations)
|
||||
patched_generations[index] = ChatGeneration(
|
||||
message=_with_reasoning_content(message, reasoning),
|
||||
generation_info=generation.generation_info,
|
||||
)
|
||||
|
||||
return ChatResult(
|
||||
generations=patched_generations or result.generations,
|
||||
llm_output=result.llm_output,
|
||||
)
|
||||
@@ -0,0 +1,21 @@
|
||||
"""User-owned IM channel connection persistence."""
|
||||
|
||||
from deerflow.persistence.channel_connections.model import (
|
||||
ChannelConnectionRow,
|
||||
ChannelConversationRow,
|
||||
ChannelCredentialRow,
|
||||
ChannelOAuthStateRow,
|
||||
)
|
||||
from deerflow.persistence.channel_connections.sql import (
|
||||
ChannelConnectionRepository,
|
||||
ChannelCredentialCipher,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ChannelConnectionRepository",
|
||||
"ChannelConnectionRow",
|
||||
"ChannelConversationRow",
|
||||
"ChannelCredentialCipher",
|
||||
"ChannelCredentialRow",
|
||||
"ChannelOAuthStateRow",
|
||||
]
|
||||
@@ -0,0 +1,111 @@
|
||||
"""ORM models for user-owned IM channel connections."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class ChannelConnectionRow(Base):
|
||||
__tablename__ = "channel_connections"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
provider: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="connected")
|
||||
|
||||
external_account_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
external_account_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
workspace_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
workspace_name: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
bot_user_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
scopes_json: Mapped[list] = mapped_column(JSON, default=list)
|
||||
capabilities_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now, onupdate=_utc_now)
|
||||
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"owner_user_id",
|
||||
"provider",
|
||||
"external_account_id",
|
||||
"workspace_id",
|
||||
name="uq_channel_connection_owner_provider_identity",
|
||||
),
|
||||
Index("idx_channel_connections_event_lookup", "provider", "workspace_id", "bot_user_id"),
|
||||
)
|
||||
|
||||
|
||||
class ChannelCredentialRow(Base):
|
||||
__tablename__ = "channel_credentials"
|
||||
|
||||
connection_id: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
ForeignKey("channel_connections.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
encrypted_access_token: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
encrypted_refresh_token: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
token_type: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
refresh_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
encrypted_extra_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now, onupdate=_utc_now)
|
||||
|
||||
|
||||
class ChannelOAuthStateRow(Base):
|
||||
__tablename__ = "channel_oauth_states"
|
||||
|
||||
state_hash: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
provider: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
code_verifier_encrypted: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
nonce_hash: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
redirect_after: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
requested_scopes_json: Mapped[list] = mapped_column(JSON, default=list)
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
consumed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now)
|
||||
|
||||
|
||||
class ChannelConversationRow(Base):
|
||||
__tablename__ = "channel_conversations"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
connection_id: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
ForeignKey("channel_connections.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
owner_user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
provider: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
external_conversation_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
external_topic_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utc_now, onupdate=_utc_now)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"connection_id",
|
||||
"external_conversation_id",
|
||||
"external_topic_id",
|
||||
name="uq_channel_conversation_connection_external",
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,349 @@
|
||||
"""SQL repository for user-owned IM channel connections."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from cryptography.fernet import Fernet
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.channel_connections.model import (
|
||||
ChannelConnectionRow,
|
||||
ChannelConversationRow,
|
||||
ChannelCredentialRow,
|
||||
ChannelOAuthStateRow,
|
||||
)
|
||||
from deerflow.utils.time import coerce_iso
|
||||
|
||||
|
||||
class ChannelCredentialCipher:
|
||||
"""Encrypts provider credentials before they are persisted."""
|
||||
|
||||
def __init__(self, fernet: Fernet) -> None:
|
||||
self._fernet = fernet
|
||||
|
||||
@classmethod
|
||||
def from_key(cls, key: str) -> ChannelCredentialCipher:
|
||||
digest = hashlib.sha256(key.encode("utf-8")).digest()
|
||||
return cls(Fernet(base64.urlsafe_b64encode(digest)))
|
||||
|
||||
def encrypt_text(self, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return "fernet:v1:" + self._fernet.encrypt(value.encode("utf-8")).decode("ascii")
|
||||
|
||||
def decrypt_text(self, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
token = value.removeprefix("fernet:v1:")
|
||||
return self._fernet.decrypt(token.encode("ascii")).decode("utf-8")
|
||||
|
||||
|
||||
class ChannelConnectionRepository:
|
||||
"""Persistence facade for channel connections, credentials, and conversations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
*,
|
||||
cipher: ChannelCredentialCipher | None = None,
|
||||
) -> None:
|
||||
self.session_factory = session_factory
|
||||
self._cipher = cipher
|
||||
|
||||
async def close(self) -> None:
|
||||
from deerflow.persistence.engine import close_engine
|
||||
|
||||
await close_engine()
|
||||
|
||||
@staticmethod
|
||||
def _new_id() -> str:
|
||||
return uuid.uuid4().hex
|
||||
|
||||
@staticmethod
|
||||
def _normalize_optional_identity(value: str | None) -> str:
|
||||
return value or ""
|
||||
|
||||
@staticmethod
|
||||
def _coerce_datetime(value: datetime | None) -> datetime | None:
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=UTC)
|
||||
|
||||
def _encrypt_optional_secret(self, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if self._cipher is None:
|
||||
raise RuntimeError("channel connection encryption key is required")
|
||||
return self._cipher.encrypt_text(value)
|
||||
|
||||
@staticmethod
|
||||
def _connection_to_dict(row: ChannelConnectionRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["external_account_id"] = data["external_account_id"] or None
|
||||
data["workspace_id"] = data["workspace_id"] or None
|
||||
data["scopes"] = data.pop("scopes_json") or []
|
||||
data["capabilities"] = data.pop("capabilities_json") or {}
|
||||
data["metadata"] = data.pop("metadata_json") or {}
|
||||
for key in ("created_at", "updated_at", "last_seen_at", "last_error_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = coerce_iso(value)
|
||||
return data
|
||||
|
||||
async def upsert_connection(
|
||||
self,
|
||||
*,
|
||||
owner_user_id: str,
|
||||
provider: str,
|
||||
external_account_id: str | None = None,
|
||||
external_account_name: str | None = None,
|
||||
workspace_id: str | None = None,
|
||||
workspace_name: str | None = None,
|
||||
bot_user_id: str | None = None,
|
||||
scopes: list[str] | None = None,
|
||||
capabilities: dict[str, Any] | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
status: str = "connected",
|
||||
) -> dict[str, Any]:
|
||||
external_account_id_value = self._normalize_optional_identity(external_account_id)
|
||||
workspace_id_value = self._normalize_optional_identity(workspace_id)
|
||||
async with self.session_factory() as session:
|
||||
stmt = select(ChannelConnectionRow).where(
|
||||
ChannelConnectionRow.owner_user_id == owner_user_id,
|
||||
ChannelConnectionRow.provider == provider,
|
||||
ChannelConnectionRow.external_account_id == external_account_id_value,
|
||||
ChannelConnectionRow.workspace_id == workspace_id_value,
|
||||
)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = ChannelConnectionRow(
|
||||
id=self._new_id(),
|
||||
owner_user_id=owner_user_id,
|
||||
provider=provider,
|
||||
external_account_id=external_account_id_value,
|
||||
workspace_id=workspace_id_value,
|
||||
)
|
||||
session.add(row)
|
||||
|
||||
row.status = status
|
||||
row.external_account_name = external_account_name
|
||||
row.workspace_name = workspace_name
|
||||
row.bot_user_id = bot_user_id
|
||||
row.scopes_json = list(scopes or [])
|
||||
row.capabilities_json = dict(capabilities or {})
|
||||
row.metadata_json = dict(metadata or {})
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._connection_to_dict(row)
|
||||
|
||||
async def list_connections(self, owner_user_id: str) -> list[dict[str, Any]]:
|
||||
async with self.session_factory() as session:
|
||||
result = await session.execute(select(ChannelConnectionRow).where(ChannelConnectionRow.owner_user_id == owner_user_id).order_by(ChannelConnectionRow.updated_at.desc(), ChannelConnectionRow.id.desc()))
|
||||
return [self._connection_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def disconnect_connection(self, *, connection_id: str, owner_user_id: str) -> bool:
|
||||
async with self.session_factory() as session:
|
||||
row = await session.get(ChannelConnectionRow, connection_id)
|
||||
if row is None or row.owner_user_id != owner_user_id:
|
||||
return False
|
||||
|
||||
row.status = "revoked"
|
||||
credential = await session.get(ChannelCredentialRow, connection_id)
|
||||
if credential is not None:
|
||||
await session.delete(credential)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def store_credentials(
|
||||
self,
|
||||
connection_id: str,
|
||||
*,
|
||||
access_token: str | None,
|
||||
refresh_token: str | None = None,
|
||||
token_type: str | None = None,
|
||||
expires_at: datetime | None = None,
|
||||
refresh_expires_at: datetime | None = None,
|
||||
extra: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
if self._cipher is None:
|
||||
raise RuntimeError("channel connection encryption key is required")
|
||||
async with self.session_factory() as session:
|
||||
row = await session.get(ChannelCredentialRow, connection_id)
|
||||
if row is None:
|
||||
row = ChannelCredentialRow(connection_id=connection_id)
|
||||
session.add(row)
|
||||
row.encrypted_access_token = self._cipher.encrypt_text(access_token)
|
||||
row.encrypted_refresh_token = self._cipher.encrypt_text(refresh_token)
|
||||
row.token_type = token_type
|
||||
row.expires_at = expires_at
|
||||
row.refresh_expires_at = refresh_expires_at
|
||||
row.encrypted_extra_json = self._cipher.encrypt_text(json.dumps(extra or {}, ensure_ascii=False))
|
||||
row.version = (row.version or 0) + 1
|
||||
await session.commit()
|
||||
|
||||
async def get_credentials(self, connection_id: str) -> dict[str, Any] | None:
|
||||
if self._cipher is None:
|
||||
return None
|
||||
async with self.session_factory() as session:
|
||||
row = await session.get(ChannelCredentialRow, connection_id)
|
||||
if row is None:
|
||||
return None
|
||||
extra_raw = self._cipher.decrypt_text(row.encrypted_extra_json)
|
||||
return {
|
||||
"connection_id": row.connection_id,
|
||||
"access_token": self._cipher.decrypt_text(row.encrypted_access_token),
|
||||
"refresh_token": self._cipher.decrypt_text(row.encrypted_refresh_token),
|
||||
"token_type": row.token_type,
|
||||
"expires_at": self._coerce_datetime(row.expires_at),
|
||||
"refresh_expires_at": self._coerce_datetime(row.refresh_expires_at),
|
||||
"extra": json.loads(extra_raw) if extra_raw else {},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def hash_state(state: str) -> str:
|
||||
return hashlib.sha256(state.encode("utf-8")).hexdigest()
|
||||
|
||||
async def create_oauth_state(
|
||||
self,
|
||||
*,
|
||||
owner_user_id: str,
|
||||
provider: str,
|
||||
state: str,
|
||||
expires_at: datetime,
|
||||
code_verifier: str | None = None,
|
||||
nonce_hash: str | None = None,
|
||||
redirect_after: str | None = None,
|
||||
requested_scopes: list[str] | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
row = ChannelOAuthStateRow(
|
||||
state_hash=self.hash_state(state),
|
||||
owner_user_id=owner_user_id,
|
||||
provider=provider,
|
||||
code_verifier_encrypted=self._encrypt_optional_secret(code_verifier),
|
||||
nonce_hash=nonce_hash,
|
||||
redirect_after=redirect_after,
|
||||
requested_scopes_json=list(requested_scopes or []),
|
||||
metadata_json=dict(metadata or {}),
|
||||
expires_at=expires_at,
|
||||
)
|
||||
async with self.session_factory() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
|
||||
async def count_oauth_states(self, *, owner_user_id: str, provider: str) -> int:
|
||||
async with self.session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(ChannelOAuthStateRow).where(
|
||||
ChannelOAuthStateRow.owner_user_id == owner_user_id,
|
||||
ChannelOAuthStateRow.provider == provider,
|
||||
)
|
||||
)
|
||||
return len(list(result.scalars()))
|
||||
|
||||
async def consume_oauth_state(
|
||||
self,
|
||||
*,
|
||||
provider: str,
|
||||
state: str,
|
||||
now: datetime | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
current_time = now or datetime.now(UTC)
|
||||
async with self.session_factory() as session:
|
||||
await session.execute(delete(ChannelOAuthStateRow).where(ChannelOAuthStateRow.expires_at < current_time))
|
||||
row = await session.get(ChannelOAuthStateRow, self.hash_state(state))
|
||||
if row is None or row.provider != provider or row.consumed_at is not None:
|
||||
await session.commit()
|
||||
return None
|
||||
expires_at = self._coerce_datetime(row.expires_at)
|
||||
if expires_at is not None and expires_at < current_time:
|
||||
await session.commit()
|
||||
return None
|
||||
|
||||
row.consumed_at = current_time
|
||||
await session.commit()
|
||||
return {
|
||||
"owner_user_id": row.owner_user_id,
|
||||
"provider": row.provider,
|
||||
"requested_scopes": row.requested_scopes_json or [],
|
||||
"metadata": row.metadata_json or {},
|
||||
"redirect_after": row.redirect_after,
|
||||
}
|
||||
|
||||
async def find_connection_by_external_identity(
|
||||
self,
|
||||
*,
|
||||
provider: str,
|
||||
external_account_id: str,
|
||||
workspace_id: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self.session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(ChannelConnectionRow)
|
||||
.where(
|
||||
ChannelConnectionRow.provider == provider,
|
||||
ChannelConnectionRow.external_account_id == self._normalize_optional_identity(external_account_id),
|
||||
ChannelConnectionRow.workspace_id == self._normalize_optional_identity(workspace_id),
|
||||
ChannelConnectionRow.status == "connected",
|
||||
)
|
||||
.order_by(ChannelConnectionRow.updated_at.desc(), ChannelConnectionRow.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return self._connection_to_dict(row) if row is not None else None
|
||||
|
||||
async def set_thread_id(
|
||||
self,
|
||||
*,
|
||||
connection_id: str,
|
||||
owner_user_id: str,
|
||||
provider: str,
|
||||
external_conversation_id: str,
|
||||
thread_id: str,
|
||||
external_topic_id: str | None = None,
|
||||
) -> None:
|
||||
topic_id = external_topic_id or ""
|
||||
async with self.session_factory() as session:
|
||||
stmt = select(ChannelConversationRow).where(
|
||||
ChannelConversationRow.connection_id == connection_id,
|
||||
ChannelConversationRow.external_conversation_id == external_conversation_id,
|
||||
ChannelConversationRow.external_topic_id == topic_id,
|
||||
)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = ChannelConversationRow(
|
||||
id=self._new_id(),
|
||||
connection_id=connection_id,
|
||||
owner_user_id=owner_user_id,
|
||||
provider=provider,
|
||||
external_conversation_id=external_conversation_id,
|
||||
external_topic_id=topic_id,
|
||||
thread_id=thread_id,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.thread_id = thread_id
|
||||
row.owner_user_id = owner_user_id
|
||||
row.provider = provider
|
||||
await session.commit()
|
||||
|
||||
async def get_thread_id(
|
||||
self,
|
||||
connection_id: str,
|
||||
external_conversation_id: str,
|
||||
external_topic_id: str | None = None,
|
||||
) -> str | None:
|
||||
async with self.session_factory() as session:
|
||||
stmt = select(ChannelConversationRow.thread_id).where(
|
||||
ChannelConversationRow.connection_id == connection_id,
|
||||
ChannelConversationRow.external_conversation_id == external_conversation_id,
|
||||
ChannelConversationRow.external_topic_id == (external_topic_id or ""),
|
||||
)
|
||||
return (await session.execute(stmt)).scalar_one_or_none()
|
||||
@@ -14,10 +14,26 @@ its storage implementation lives in ``deerflow.runtime.events.store.db`` and
|
||||
there is no matching entity directory.
|
||||
"""
|
||||
|
||||
from deerflow.persistence.channel_connections.model import (
|
||||
ChannelConnectionRow,
|
||||
ChannelConversationRow,
|
||||
ChannelCredentialRow,
|
||||
ChannelOAuthStateRow,
|
||||
)
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
|
||||
__all__ = ["FeedbackRow", "RunEventRow", "RunRow", "ThreadMetaRow", "UserRow"]
|
||||
__all__ = [
|
||||
"ChannelConnectionRow",
|
||||
"ChannelConversationRow",
|
||||
"ChannelCredentialRow",
|
||||
"ChannelOAuthStateRow",
|
||||
"FeedbackRow",
|
||||
"RunEventRow",
|
||||
"RunRow",
|
||||
"ThreadMetaRow",
|
||||
"UserRow",
|
||||
]
|
||||
|
||||
@@ -71,6 +71,15 @@ class ThreadMetaStore(abc.ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def update_owner(self, thread_id: str, owner_user_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
"""Move a thread metadata row to a new owner.
|
||||
|
||||
Intended for trusted internal repair/migration paths. No-op if the
|
||||
row does not exist or the caller fails the owner check.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def check_access(self, thread_id: str, user_id: str, *, require_existing: bool = False) -> bool:
|
||||
"""Check if ``user_id`` has access to ``thread_id``."""
|
||||
|
||||
@@ -127,6 +127,14 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
|
||||
async def update_owner(self, thread_id: str, owner_user_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.update_owner")
|
||||
if record is None:
|
||||
return
|
||||
record["user_id"] = owner_user_id
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
|
||||
async def delete(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.delete")
|
||||
if record is None:
|
||||
|
||||
@@ -211,6 +211,21 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
|
||||
async def update_owner(
|
||||
self,
|
||||
thread_id: str,
|
||||
owner_user_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> None:
|
||||
"""Move a thread metadata row to ``owner_user_id``."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.update_owner")
|
||||
async with self._sf() as session:
|
||||
if not await self._check_ownership(session, thread_id, resolved_user_id):
|
||||
return
|
||||
await session.execute(update(ThreadMetaRow).where(ThreadMetaRow.thread_id == thread_id).values(user_id=owner_user_id, updated_at=datetime.now(UTC)))
|
||||
await session.commit()
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
thread_id: str,
|
||||
|
||||
@@ -164,7 +164,18 @@ class RunJournal(BaseCallbackHandler):
|
||||
metadata={"caller": caller, **(metadata or {})},
|
||||
)
|
||||
|
||||
def on_chain_end(self, outputs: Any, *, run_id: UUID, **kwargs: Any) -> None:
|
||||
def on_chain_end(
|
||||
self,
|
||||
outputs: Any,
|
||||
*,
|
||||
run_id: UUID,
|
||||
parent_run_id: UUID | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
# Nested chain ends fire for internal graph nodes; only the root chain
|
||||
# represents the user-visible run lifecycle.
|
||||
if parent_run_id is not None:
|
||||
return
|
||||
self._put(event_type="run.end", category="outputs", content=outputs, metadata={"status": "success"})
|
||||
self._flush_sync()
|
||||
|
||||
|
||||
@@ -83,6 +83,7 @@ class RunRecord:
|
||||
multitask_strategy: str = "reject"
|
||||
metadata: dict = field(default_factory=dict)
|
||||
kwargs: dict = field(default_factory=dict)
|
||||
user_id: str | None = None
|
||||
created_at: str = ""
|
||||
updated_at: str = ""
|
||||
task: asyncio.Task | None = field(default=None, repr=False)
|
||||
@@ -124,7 +125,7 @@ class RunManager:
|
||||
|
||||
@staticmethod
|
||||
def _store_put_payload(record: RunRecord, *, error: str | None = None) -> dict[str, Any]:
|
||||
return {
|
||||
payload = {
|
||||
"thread_id": record.thread_id,
|
||||
"assistant_id": record.assistant_id,
|
||||
"status": record.status.value,
|
||||
@@ -135,6 +136,9 @@ class RunManager:
|
||||
"created_at": record.created_at,
|
||||
"model_name": record.model_name,
|
||||
}
|
||||
if record.user_id is not None:
|
||||
payload["user_id"] = record.user_id
|
||||
return payload
|
||||
|
||||
async def _call_store_with_retry(
|
||||
self,
|
||||
@@ -241,6 +245,7 @@ class RunManager:
|
||||
kwargs=row.get("kwargs") or {},
|
||||
created_at=row.get("created_at") or "",
|
||||
updated_at=row.get("updated_at") or "",
|
||||
user_id=row.get("user_id"),
|
||||
error=row.get("error"),
|
||||
model_name=row.get("model_name"),
|
||||
store_only=True,
|
||||
@@ -320,6 +325,7 @@ class RunManager:
|
||||
metadata: dict | None = None,
|
||||
kwargs: dict | None = None,
|
||||
multitask_strategy: str = "reject",
|
||||
user_id: str | None = None,
|
||||
) -> RunRecord:
|
||||
"""Create a new pending run and register it."""
|
||||
run_id = str(uuid.uuid4())
|
||||
@@ -333,6 +339,7 @@ class RunManager:
|
||||
multitask_strategy=multitask_strategy,
|
||||
metadata=metadata or {},
|
||||
kwargs=kwargs or {},
|
||||
user_id=user_id,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
@@ -504,6 +511,7 @@ class RunManager:
|
||||
kwargs: dict | None = None,
|
||||
multitask_strategy: str = "reject",
|
||||
model_name: str | None = None,
|
||||
user_id: str | None = None,
|
||||
) -> RunRecord:
|
||||
"""Atomically check for inflight runs and create a new one.
|
||||
|
||||
@@ -546,6 +554,7 @@ class RunManager:
|
||||
multitask_strategy=multitask_strategy,
|
||||
metadata=metadata or {},
|
||||
kwargs=kwargs or {},
|
||||
user_id=user_id,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
model_name=model_name,
|
||||
|
||||
@@ -147,7 +147,17 @@ class LocalSandboxProvider(SandboxProvider):
|
||||
mount.container_path,
|
||||
)
|
||||
continue
|
||||
# Ensure the host path exists before adding mapping
|
||||
# Ensure the host path exists before adding mapping.
|
||||
#
|
||||
# ``host_path`` is resolved against the filesystem of the
|
||||
# process running this provider — for ``make dev`` that is
|
||||
# the host machine, but for ``make up`` it is the
|
||||
# ``deer-flow-gateway`` container, so any host path that
|
||||
# isn't bind-mounted into the gateway image will be missing
|
||||
# here. Skipping silently makes this a high-cost-to-debug
|
||||
# silent failure (sandbox skill / tool reads an empty dir
|
||||
# instead of the configured mount), so escalate to ERROR
|
||||
# and include actionable guidance. See #3244.
|
||||
if host_path.exists():
|
||||
mappings.append(
|
||||
PathMapping(
|
||||
@@ -157,10 +167,16 @@ class LocalSandboxProvider(SandboxProvider):
|
||||
)
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Mount host_path does not exist, skipping: %s -> %s",
|
||||
logger.error(
|
||||
"sandbox.mounts entry %s -> %s ignored: host_path %s does not exist from the "
|
||||
"perspective of the gateway process. In Docker deployments (make up / docker-compose), "
|
||||
"this path must also be bind-mounted into the gateway container — add a matching "
|
||||
"volume entry under services.gateway.volumes in docker/docker-compose.yaml (and use "
|
||||
"the in-container path here), or run in local mode (make dev) where the gateway sees "
|
||||
"the host filesystem directly.",
|
||||
mount.host_path,
|
||||
mount.container_path,
|
||||
mount.host_path,
|
||||
)
|
||||
except Exception as e:
|
||||
# Log but don't fail if config loading fails
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from deerflow.skills.types import Skill
|
||||
|
||||
RESERVED_SLASH_SKILL_NAMES = frozenset({"bootstrap", "help", "memory", "models", "new", "status"})
|
||||
_SLASH_SKILL_RE = re.compile(r"^/([a-z0-9]+(?:-[a-z0-9]+)*)(?:\s+|$)")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SlashSkillReference:
|
||||
"""Parsed slash-skill command with the skill name and remaining task text."""
|
||||
|
||||
name: str
|
||||
remaining_text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ResolvedSlashSkill:
|
||||
"""Slash-skill activation resolved against enabled runtime-visible skills."""
|
||||
|
||||
skill: Skill
|
||||
remaining_text: str
|
||||
container_file_path: str
|
||||
|
||||
|
||||
def parse_slash_skill_reference(text: str) -> SlashSkillReference | None:
|
||||
"""Parse strict `/skill-name task` syntax, ignoring reserved control commands."""
|
||||
match = _SLASH_SKILL_RE.match(text)
|
||||
if not match:
|
||||
return None
|
||||
name = match.group(1)
|
||||
if name in RESERVED_SLASH_SKILL_NAMES:
|
||||
return None
|
||||
return SlashSkillReference(
|
||||
name=name,
|
||||
remaining_text=text[match.end() :].lstrip(),
|
||||
)
|
||||
|
||||
|
||||
def resolve_slash_skill(
|
||||
text: str,
|
||||
skills: list[Skill],
|
||||
*,
|
||||
available_skills: set[str] | None = None,
|
||||
container_base_path: str = "/mnt/skills",
|
||||
) -> ResolvedSlashSkill | None:
|
||||
"""Resolve text into an enabled, whitelisted skill activation if possible."""
|
||||
reference = parse_slash_skill_reference(text)
|
||||
if reference is None:
|
||||
return None
|
||||
if available_skills is not None and reference.name not in available_skills:
|
||||
return None
|
||||
|
||||
skill = next((candidate for candidate in skills if candidate.name == reference.name and candidate.enabled), None)
|
||||
if skill is None:
|
||||
return None
|
||||
|
||||
return ResolvedSlashSkill(
|
||||
skill=skill,
|
||||
remaining_text=reference.remaining_text,
|
||||
container_file_path=skill.get_container_file_path(container_base_path),
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
ORIGINAL_USER_CONTENT_KEY = "original_user_content"
|
||||
|
||||
|
||||
def message_content_to_text(content: Any) -> str:
|
||||
"""Extract text from LangChain message content shapes."""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
parts.append(item)
|
||||
elif isinstance(item, dict):
|
||||
text = item.get("text")
|
||||
if isinstance(text, str):
|
||||
parts.append(text)
|
||||
return "\n".join(part for part in parts if part)
|
||||
return str(content)
|
||||
|
||||
|
||||
def get_original_user_content_text(content: Any, additional_kwargs: Mapping[str, Any] | None) -> str:
|
||||
"""Return pre-middleware user text when available, otherwise content text."""
|
||||
original_content = (additional_kwargs or {}).get(ORIGINAL_USER_CONTENT_KEY)
|
||||
if isinstance(original_content, str):
|
||||
return original_content
|
||||
return message_content_to_text(content)
|
||||
@@ -36,6 +36,7 @@ dependencies = [
|
||||
"sqlalchemy[asyncio]>=2.0,<3.0",
|
||||
"aiosqlite>=0.19",
|
||||
"alembic>=1.13",
|
||||
"cryptography>=43.0.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
@@ -36,7 +36,8 @@ def main() -> int:
|
||||
for index, turn in enumerate(turns):
|
||||
data = turn["output"].get("data", {})
|
||||
tool_calls = [tc.get("name") for tc in (data.get("tool_calls") or [])]
|
||||
print(f" turn {index}: hash={turn['input_hash'][:12]} tool_calls={tool_calls} content={str(data.get('content'))[:50]!r}")
|
||||
caller = turn.get("caller", "legacy")
|
||||
print(f" turn {index}: caller={caller} hash={turn['input_hash'][:12]} tool_calls={tool_calls} content={str(data.get('content'))[:50]!r}")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -28,27 +28,45 @@ sys.path.insert(0, str(_BACKEND / "tests"))
|
||||
def _install_capture(out_path: Path) -> None:
|
||||
from langchain_core.callbacks import BaseCallbackHandler
|
||||
from langchain_core.messages import messages_to_dict
|
||||
from replay_provider import hash_messages
|
||||
from replay_provider import caller_identity, hash_messages, hash_replay_input
|
||||
|
||||
import deerflow.models.factory as factory_mod
|
||||
|
||||
class Capture(BaseCallbackHandler):
|
||||
def __init__(self) -> None:
|
||||
self.inputs: dict[str, list] = {}
|
||||
self.inputs: dict[str, tuple[list, str]] = {}
|
||||
|
||||
def on_chat_model_start(self, serialized, messages, *, run_id=None, **kwargs): # noqa: ANN001
|
||||
self.inputs[str(run_id)] = messages[0] if messages else []
|
||||
def on_chat_model_start( # noqa: ANN001
|
||||
self,
|
||||
serialized,
|
||||
messages,
|
||||
*,
|
||||
run_id=None,
|
||||
tags=None,
|
||||
name=None,
|
||||
**kwargs,
|
||||
):
|
||||
self.inputs[str(run_id)] = (
|
||||
messages[0] if messages else [],
|
||||
caller_identity(name=name, tags=tags),
|
||||
)
|
||||
|
||||
def on_llm_end(self, response, *, run_id=None, **kwargs): # noqa: ANN001
|
||||
inp = self.inputs.pop(str(run_id), None)
|
||||
if inp is None:
|
||||
captured = self.inputs.pop(str(run_id), None)
|
||||
if captured is None:
|
||||
return
|
||||
inp, caller = captured
|
||||
for batch in response.generations:
|
||||
for gen in batch:
|
||||
message = getattr(gen, "message", None)
|
||||
if message is None:
|
||||
continue
|
||||
record = {"input_hash": hash_messages(inp), "output": messages_to_dict([message])[0]}
|
||||
record = {
|
||||
"caller": caller,
|
||||
"conversation_hash": hash_messages(inp),
|
||||
"input_hash": hash_replay_input(inp, caller=caller),
|
||||
"output": messages_to_dict([message])[0],
|
||||
}
|
||||
with open(out_path, "a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
handle.flush()
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Process-wide Python startup customizations for backend entrypoints.
|
||||
|
||||
When ``backend/`` is on ``sys.path``, Python imports this module during
|
||||
interpreter startup. Keep changes here suitable for all gateway, script,
|
||||
migration, and test entrypoints that run in that environment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
|
||||
def _configure_windows_event_loop_policy() -> None:
|
||||
if sys.platform != "win32":
|
||||
return
|
||||
|
||||
selector_policy = getattr(asyncio, "WindowsSelectorEventLoopPolicy", None)
|
||||
if selector_policy is None:
|
||||
return
|
||||
|
||||
if not isinstance(asyncio.get_event_loop_policy(), selector_policy):
|
||||
asyncio.set_event_loop_policy(selector_policy())
|
||||
|
||||
|
||||
_configure_windows_event_loop_policy()
|
||||
@@ -32,7 +32,8 @@ REPLAY_MODEL_BLOCK = """\
|
||||
- name: scenario-model
|
||||
display_name: Scenario Model
|
||||
use: replay_provider:ReplayChatModel
|
||||
model: replay"""
|
||||
model: replay
|
||||
supports_thinking: true"""
|
||||
|
||||
|
||||
def real_model_block(model: str) -> str:
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Regression anchors: the custom-agent router must not block the event loop.
|
||||
|
||||
``app.gateway.routers.agents.create_agent_endpoint`` and ``delete_agent`` are
|
||||
async route handlers that resolve the agent directory (``Paths.base_dir`` calls
|
||||
``Path.resolve``), probe it (``Path.exists``), and create/remove it (``mkdir``,
|
||||
config/SOUL writes, ``shutil.rmtree``) — all blocking IO. Both offload that work
|
||||
via ``asyncio.to_thread``; if any of it regresses back onto the event loop, the
|
||||
strict Blockbuster gate raises ``BlockingError`` and these tests fail.
|
||||
|
||||
Imports live at module scope so the one-time FastAPI app construction (which
|
||||
reads files while building OpenAPI schemas) happens at collection time, not on
|
||||
the event loop under test. Test-side path resolution is itself offloaded with
|
||||
``asyncio.to_thread`` (matching ``test_uploads_middleware``) so only the
|
||||
handlers' own filesystem access is exercised on the loop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gateway.routers.agents import AgentCreateRequest, create_agent_endpoint, delete_agent
|
||||
from deerflow.config.agents_api_config import load_agents_api_config_from_dict
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
async def test_create_agent_does_not_block_event_loop(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path))
|
||||
monkeypatch.setattr("deerflow.config.paths._paths", None)
|
||||
load_agents_api_config_from_dict({"enabled": True})
|
||||
try:
|
||||
response = await create_agent_endpoint(AgentCreateRequest(name="loop-make-agent", soul="You are a test agent."))
|
||||
assert response is not None
|
||||
|
||||
user_id = get_effective_user_id()
|
||||
# test-side check (resolution offloaded; not exercised on the loop)
|
||||
agent_dir = await asyncio.to_thread(get_paths().user_agent_dir, user_id, "loop-make-agent")
|
||||
assert await asyncio.to_thread((agent_dir / "config.yaml").exists)
|
||||
finally:
|
||||
load_agents_api_config_from_dict({})
|
||||
|
||||
|
||||
async def test_delete_agent_does_not_block_event_loop(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path))
|
||||
monkeypatch.setattr("deerflow.config.paths._paths", None)
|
||||
load_agents_api_config_from_dict({"enabled": True})
|
||||
try:
|
||||
user_id = get_effective_user_id()
|
||||
user_id = get_effective_user_id()
|
||||
# test-side seeding (resolution offloaded; not exercised on the loop)
|
||||
agent_dir = await asyncio.to_thread(get_paths().user_agent_dir, user_id, "loop-test-agent")
|
||||
await asyncio.to_thread(agent_dir.mkdir, parents=True, exist_ok=True)
|
||||
await asyncio.to_thread((agent_dir / "config.yaml").write_text, "name: loop-test-agent\n", encoding="utf-8")
|
||||
|
||||
await delete_agent("loop-test-agent")
|
||||
|
||||
assert not await asyncio.to_thread(agent_dir.exists)
|
||||
finally:
|
||||
load_agents_api_config_from_dict({})
|
||||
+15
-5
@@ -12,7 +12,9 @@
|
||||
},
|
||||
"turns": [
|
||||
{
|
||||
"input_hash": "9c50eda6ab7e8593dabccbdeadc70a4a7bf778b2c0c3f275f1f96cf2c8ab58db",
|
||||
"caller": "lead_agent",
|
||||
"conversation_hash": "9c50eda6ab7e8593dabccbdeadc70a4a7bf778b2c0c3f275f1f96cf2c8ab58db",
|
||||
"input_hash": "27aeb4c11bff2c3ebc182fe52a06556823c21928620a400c7f26be9733c31f3f",
|
||||
"output": {
|
||||
"type": "ai",
|
||||
"data": {
|
||||
@@ -56,7 +58,9 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"input_hash": "3598aeb87e221ca8f554e4d61ce6d5e8801754606fa5c95a89c38bd6cb623045",
|
||||
"caller": "middleware:title",
|
||||
"conversation_hash": "3598aeb87e221ca8f554e4d61ce6d5e8801754606fa5c95a89c38bd6cb623045",
|
||||
"input_hash": "75101f9faa453b1a35deff920b1e3c1a9f0b013a7627fbbaa03436752776b953",
|
||||
"output": {
|
||||
"type": "ai",
|
||||
"data": {
|
||||
@@ -89,7 +93,9 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"input_hash": "6af134379b2a9efa01b4f63032f88211d5f38f459f8bed621eb6c65e8e05c1f9",
|
||||
"caller": "lead_agent",
|
||||
"conversation_hash": "6af134379b2a9efa01b4f63032f88211d5f38f459f8bed621eb6c65e8e05c1f9",
|
||||
"input_hash": "f7468603a43d301fcc0167c2f7cd10e53137bfc584f1b3d776614b7a612ed7a6",
|
||||
"output": {
|
||||
"type": "ai",
|
||||
"data": {
|
||||
@@ -132,7 +138,9 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"input_hash": "04751c4f7b0107b78b5c97d417063883fd586f5ebcbc4acf79be6cb3c0cdaec1",
|
||||
"caller": "lead_agent",
|
||||
"conversation_hash": "04751c4f7b0107b78b5c97d417063883fd586f5ebcbc4acf79be6cb3c0cdaec1",
|
||||
"input_hash": "218645dabc6926a1dbdf45dd20fba8a41e1e690cef78d7752566db3acf5a36ce",
|
||||
"output": {
|
||||
"type": "ai",
|
||||
"data": {
|
||||
@@ -165,7 +173,9 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"input_hash": "8b98ebdbb53e88f000556c4753adede8eaa076ff6fd7b8a1285bfd18aee8144d",
|
||||
"caller": "suggest_agent",
|
||||
"conversation_hash": "8b98ebdbb53e88f000556c4753adede8eaa076ff6fd7b8a1285bfd18aee8144d",
|
||||
"input_hash": "dcd855d389d7179a1e4bc7074fa9ba7ce697570af8947225d6bacb538f14a0cb",
|
||||
"output": {
|
||||
"type": "ai",
|
||||
"data": {
|
||||
|
||||
@@ -2,14 +2,19 @@
|
||||
record/replay e2e (mirrors open-design's ``mocks/`` golden traces).
|
||||
|
||||
A fixture is a JSON file capturing the *real* model calls of one scenario,
|
||||
keyed by a normalized hash of the **input** each call received::
|
||||
keyed by a normalized hash of the **caller + input** each call received::
|
||||
|
||||
{
|
||||
"scenario": "write_read_file",
|
||||
"mode": "ultra",
|
||||
"model": "gpt-5.5",
|
||||
"turns": [
|
||||
{"input_hash": "<sha256>", "input_preview": "...", "output": <message dict>},
|
||||
{
|
||||
"caller": "lead_agent",
|
||||
"conversation_hash": "<sha256>",
|
||||
"input_hash": "<sha256>",
|
||||
"output": <message dict>,
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
@@ -21,8 +26,11 @@ A real run makes model calls from several callers — the lead agent's own turns
|
||||
and their count/order is not something we want a replay to depend on. Matching by
|
||||
a normalized hash of the *input messages* means each call gets back exactly the
|
||||
output that was recorded for that input, regardless of order or which middleware
|
||||
issued it. That keeps the in-graph, deterministic title call part of the
|
||||
recording; memory/summarization, by contrast, are disabled in the replay config
|
||||
issued it. The caller name (``lead_agent``, ``middleware:title``,
|
||||
``suggest_agent``, ``subagent:*``, ...) is included so two different model
|
||||
callers with the same conversation text do not compete for the same replay
|
||||
bucket. That keeps the in-graph, deterministic title call part of the recording;
|
||||
memory/summarization, by contrast, are disabled in the replay config
|
||||
(``_replay_fixture.py``) because their background, debounced timing is not
|
||||
reproducible across runs.
|
||||
|
||||
@@ -67,7 +75,7 @@ from collections import deque
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.callbacks import CallbackManagerForLLMRun
|
||||
from langchain_core.callbacks import BaseCallbackHandler, CallbackManagerForLLMRun
|
||||
from langchain_core.language_models.chat_models import BaseChatModel
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage, messages_from_dict
|
||||
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
|
||||
@@ -75,6 +83,14 @@ from langchain_core.runnables import Runnable
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
_FIXTURE_ENV = "DEERFLOW_REPLAY_FIXTURE"
|
||||
_DEFAULT_CALLER = "lead_agent"
|
||||
_CALLER_TAG_PREFIXES = ("middleware:", "subagent:")
|
||||
_CALLER_NAME_ALIASES = {
|
||||
# TitleMiddleware uses this run_name and tags the call as middleware:title.
|
||||
# Some execution paths do not preserve the tag down to the model callback,
|
||||
# so keep the run_name and tag in the same replay namespace.
|
||||
"title_agent": "middleware:title",
|
||||
}
|
||||
|
||||
# Process-wide record of replay misses. A miss raises inside the model, but the
|
||||
# gateway's LLMErrorHandlingMiddleware swallows it into a normal assistant error
|
||||
@@ -94,6 +110,30 @@ def reset_replay_misses() -> None:
|
||||
_replay_misses.clear()
|
||||
|
||||
|
||||
def _normalize_caller(caller: str | None) -> str:
|
||||
value = _normalize_text(str(caller or "").strip())
|
||||
if not value:
|
||||
return _DEFAULT_CALLER
|
||||
return _CALLER_NAME_ALIASES.get(value, value)
|
||||
|
||||
|
||||
def _caller_from_tags(tags: list[str] | None) -> str | None:
|
||||
for tag in tags or []:
|
||||
if isinstance(tag, str) and (tag == _DEFAULT_CALLER or tag.startswith(_CALLER_TAG_PREFIXES)):
|
||||
return tag
|
||||
return None
|
||||
|
||||
|
||||
def caller_identity(*, name: str | None = None, tags: list[str] | None = None) -> str:
|
||||
"""Stable model-caller identity shared by record and replay.
|
||||
|
||||
Tags win because graph middleware and subagents already use them as the
|
||||
explicit caller marker. ``run_name`` is exposed to callbacks as ``name`` and
|
||||
covers route-level callers such as ``suggest_agent``.
|
||||
"""
|
||||
return _normalize_caller(_caller_from_tags(tags) or name)
|
||||
|
||||
|
||||
# Volatile substrings that differ between a recording run and a replay run but
|
||||
# carry no semantic weight for matching. Normalized to stable placeholders
|
||||
# before hashing so the same logical input hashes identically across processes.
|
||||
@@ -172,10 +212,30 @@ def _canonical_messages(messages: list[BaseMessage]) -> str:
|
||||
|
||||
|
||||
def hash_messages(messages: list[BaseMessage]) -> str:
|
||||
"""Stable hash of a model call's input. Shared by recorder and replayer."""
|
||||
"""Legacy stable hash of only a model call's conversation input."""
|
||||
return hashlib.sha256(_canonical_messages(messages).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def hash_replay_input(messages: list[BaseMessage], *, caller: str | None) -> str:
|
||||
"""Stable replay key for a caller-specific model input."""
|
||||
return hash_input_key(hash_messages(messages), caller=caller)
|
||||
|
||||
|
||||
def hash_input_key(conversation_hash: str, *, caller: str | None) -> str:
|
||||
"""Namespace a conversation hash by caller identity.
|
||||
|
||||
Keeping this as ``hash(caller + legacy_conversation_hash)`` lets existing
|
||||
fixtures migrate without a live-model re-record: their old ``input_hash`` is
|
||||
exactly the conversation hash.
|
||||
"""
|
||||
payload = json.dumps(
|
||||
{"caller": _normalize_caller(caller), "conversation_hash": conversation_hash},
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _load_fixture(fixture_path: str) -> dict[str, deque[AIMessage]]:
|
||||
with open(fixture_path, encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
@@ -199,24 +259,54 @@ class ReplayChatModel(BaseChatModel):
|
||||
|
||||
_table: dict[str, deque] = PrivateAttr(default_factory=dict)
|
||||
_fixture_path: str = PrivateAttr(default="")
|
||||
_run_callers: dict[str, str] = PrivateAttr(default_factory=dict)
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
# Ignore provider noise the factory forwards from config (model, api_key,
|
||||
# base_url, ...). Fixture path comes from the ``fixture`` kwarg or env.
|
||||
fixture_path = kwargs.pop("fixture", None) or os.environ.get(_FIXTURE_ENV)
|
||||
super().__init__()
|
||||
callbacks = kwargs.pop("callbacks", None)
|
||||
super().__init__(callbacks=callbacks)
|
||||
if not fixture_path:
|
||||
raise ValueError(f"ReplayChatModel needs a fixture path via the ``fixture`` kwarg or ${_FIXTURE_ENV}")
|
||||
self._fixture_path = fixture_path
|
||||
self._table = _load_fixture(fixture_path)
|
||||
self.callbacks = [*(self.callbacks or []), _ReplayCallerCapture(self._run_callers)]
|
||||
|
||||
@property
|
||||
def _llm_type(self) -> str:
|
||||
return "deerflow-replay"
|
||||
|
||||
def _match(self, messages: list[BaseMessage]) -> AIMessage:
|
||||
key = hash_messages(messages)
|
||||
def _caller_from_run_manager(self, run_manager: CallbackManagerForLLMRun | None) -> str:
|
||||
if run_manager is None:
|
||||
if len(self._run_callers) == 1:
|
||||
# Some async LangGraph paths fire on_chat_model_start with the
|
||||
# caller metadata but invoke the model implementation without a
|
||||
# run_manager. When there is only one pending start event, it is
|
||||
# the current call; use it so record/replay share the same
|
||||
# caller key.
|
||||
return self._run_callers.pop(next(iter(self._run_callers)))
|
||||
return _DEFAULT_CALLER
|
||||
run_id = str(getattr(run_manager, "run_id", ""))
|
||||
caller = self._run_callers.pop(run_id, None)
|
||||
if caller:
|
||||
return caller
|
||||
return caller_identity(
|
||||
name=getattr(run_manager, "run_name", None) or getattr(run_manager, "name", None),
|
||||
tags=getattr(run_manager, "tags", None),
|
||||
)
|
||||
|
||||
def _match(self, messages: list[BaseMessage], run_manager: CallbackManagerForLLMRun | None = None) -> AIMessage:
|
||||
caller = self._caller_from_run_manager(run_manager)
|
||||
key = hash_replay_input(messages, caller=caller)
|
||||
bucket = self._table.get(key)
|
||||
if not bucket:
|
||||
# Backward compatibility for fixtures recorded before caller-aware
|
||||
# keys. New recordings write caller-aware ``input_hash`` values.
|
||||
legacy_key = hash_messages(messages)
|
||||
bucket = self._table.get(legacy_key)
|
||||
if bucket:
|
||||
key = legacy_key
|
||||
if not bucket:
|
||||
_replay_misses.append(key)
|
||||
preview = _canonical_messages(messages)
|
||||
@@ -224,6 +314,7 @@ class ReplayChatModel(BaseChatModel):
|
||||
f"replay miss: no recorded output for input hash {key} in {self._fixture_path!r}. "
|
||||
"The replayed run diverged from the recording (graph changed, a non-deterministic tool result "
|
||||
"altered a downstream input, or a volatile field slipped past normalization). "
|
||||
f"Caller: {caller!r}. "
|
||||
f"Known hashes: {sorted(self._table)}. "
|
||||
f"Normalized input (first 800 chars): {preview[:800]!r}"
|
||||
)
|
||||
@@ -236,7 +327,7 @@ class ReplayChatModel(BaseChatModel):
|
||||
run_manager: CallbackManagerForLLMRun | None = None,
|
||||
**kwargs: Any,
|
||||
) -> ChatResult:
|
||||
return ChatResult(generations=[ChatGeneration(message=self._match(messages))])
|
||||
return ChatResult(generations=[ChatGeneration(message=self._match(messages, run_manager))])
|
||||
|
||||
def _stream(
|
||||
self,
|
||||
@@ -245,9 +336,16 @@ class ReplayChatModel(BaseChatModel):
|
||||
run_manager: CallbackManagerForLLMRun | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Iterator[ChatGenerationChunk]:
|
||||
turn = self._match(messages)
|
||||
turn = self._match(messages, run_manager)
|
||||
text = turn.content if isinstance(turn.content, str) else ""
|
||||
chunk = ChatGenerationChunk(message=AIMessageChunk(content=turn.content, tool_calls=turn.tool_calls, additional_kwargs=turn.additional_kwargs, id=turn.id))
|
||||
chunk = ChatGenerationChunk(
|
||||
message=AIMessageChunk(
|
||||
content=turn.content,
|
||||
tool_calls=turn.tool_calls,
|
||||
additional_kwargs=turn.additional_kwargs,
|
||||
id=turn.id,
|
||||
)
|
||||
)
|
||||
if run_manager is not None and text:
|
||||
run_manager.on_llm_new_token(text, chunk=chunk)
|
||||
yield chunk
|
||||
@@ -256,5 +354,31 @@ class ReplayChatModel(BaseChatModel):
|
||||
return self
|
||||
|
||||
|
||||
class _ReplayCallerCapture(BaseCallbackHandler):
|
||||
def __init__(self, run_callers: dict[str, str]) -> None:
|
||||
self._run_callers = run_callers
|
||||
|
||||
def on_chat_model_start(
|
||||
self,
|
||||
serialized: dict,
|
||||
messages: list[list[BaseMessage]],
|
||||
*,
|
||||
run_id: Any = None,
|
||||
tags: list[str] | None = None,
|
||||
name: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if run_id is not None:
|
||||
self._run_callers[str(run_id)] = caller_identity(name=name, tags=tags)
|
||||
|
||||
|
||||
# Re-export so the recorder shares the exact hashing logic.
|
||||
__all__ = ["ReplayChatModel", "hash_messages", "replay_misses", "reset_replay_misses"]
|
||||
__all__ = [
|
||||
"ReplayChatModel",
|
||||
"caller_identity",
|
||||
"hash_input_key",
|
||||
"hash_messages",
|
||||
"hash_replay_input",
|
||||
"replay_misses",
|
||||
"reset_replay_misses",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
"""Connection binding tests for browser-connectable IM channels beyond Telegram/Slack/Discord."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from app.channels.message_bus import InboundMessage, MessageBus
|
||||
|
||||
|
||||
async def _make_repo(tmp_path, name: str):
|
||||
from deerflow.persistence.channel_connections import ChannelConnectionRepository
|
||||
from deerflow.persistence.engine import get_session_factory, init_engine
|
||||
|
||||
await init_engine("sqlite", url=f"sqlite+aiosqlite:///{tmp_path / f'{name}.db'}", sqlite_dir=str(tmp_path))
|
||||
return ChannelConnectionRepository(get_session_factory())
|
||||
|
||||
|
||||
async def _seed_state(repo, provider: str, state: str, owner_user_id: str = "deerflow-user-1") -> None:
|
||||
await repo.create_oauth_state(
|
||||
owner_user_id=owner_user_id,
|
||||
provider=provider,
|
||||
state=state,
|
||||
expires_at=datetime.now(UTC) + timedelta(minutes=5),
|
||||
)
|
||||
|
||||
|
||||
def test_feishu_connect_command_binds_identity(tmp_path):
|
||||
import anyio
|
||||
|
||||
from app.channels.feishu import FeishuChannel
|
||||
|
||||
async def go():
|
||||
repo = await _make_repo(tmp_path, "feishu")
|
||||
state = "feishu-bind-code"
|
||||
await _seed_state(repo, "feishu", state)
|
||||
channel = FeishuChannel(
|
||||
bus=MessageBus(),
|
||||
config={"app_id": "app", "app_secret": "secret", "connection_repo": repo},
|
||||
)
|
||||
channel._reply_card = AsyncMock()
|
||||
|
||||
handled = await channel._bind_connection_from_connect_code(
|
||||
message_id="om-message-1",
|
||||
chat_id="oc-chat-1",
|
||||
user_id="ou-user-1",
|
||||
code=state,
|
||||
)
|
||||
|
||||
connections = await repo.list_connections("deerflow-user-1")
|
||||
assert handled is True
|
||||
assert len(connections) == 1
|
||||
assert connections[0]["provider"] == "feishu"
|
||||
assert connections[0]["external_account_id"] == "ou-user-1"
|
||||
assert connections[0]["workspace_id"] == "oc-chat-1"
|
||||
channel._reply_card.assert_awaited_once_with("om-message-1", "Feishu connected to DeerFlow.")
|
||||
await repo.close()
|
||||
|
||||
anyio.run(go)
|
||||
|
||||
|
||||
def test_dingtalk_connect_command_binds_identity(tmp_path):
|
||||
import anyio
|
||||
|
||||
from app.channels.dingtalk import _CONVERSATION_TYPE_GROUP, DingTalkChannel
|
||||
|
||||
async def go():
|
||||
repo = await _make_repo(tmp_path, "dingtalk")
|
||||
state = "dingtalk-bind-code"
|
||||
await _seed_state(repo, "dingtalk", state)
|
||||
channel = DingTalkChannel(
|
||||
bus=MessageBus(),
|
||||
config={"client_id": "client", "client_secret": "secret", "connection_repo": repo},
|
||||
)
|
||||
channel._send_connection_reply = AsyncMock()
|
||||
|
||||
handled = await channel._bind_connection_from_connect_code(
|
||||
conversation_type=_CONVERSATION_TYPE_GROUP,
|
||||
sender_staff_id="staff-user-1",
|
||||
sender_nick="Alice",
|
||||
conversation_id="cid-group-1",
|
||||
code=state,
|
||||
)
|
||||
|
||||
connections = await repo.list_connections("deerflow-user-1")
|
||||
assert handled is True
|
||||
assert len(connections) == 1
|
||||
assert connections[0]["provider"] == "dingtalk"
|
||||
assert connections[0]["external_account_id"] == "staff-user-1"
|
||||
assert connections[0]["external_account_name"] == "Alice"
|
||||
assert connections[0]["workspace_id"] == "cid-group-1"
|
||||
channel._send_connection_reply.assert_awaited_once()
|
||||
await repo.close()
|
||||
|
||||
anyio.run(go)
|
||||
|
||||
|
||||
def test_wechat_connect_command_binds_identity(tmp_path):
|
||||
import anyio
|
||||
|
||||
from app.channels.wechat import WechatChannel
|
||||
|
||||
async def go():
|
||||
repo = await _make_repo(tmp_path, "wechat")
|
||||
state = "wechat-bind-code"
|
||||
await _seed_state(repo, "wechat", state)
|
||||
channel = WechatChannel(
|
||||
bus=MessageBus(),
|
||||
config={"bot_token": "token", "connection_repo": repo},
|
||||
)
|
||||
channel._send_connection_reply = AsyncMock()
|
||||
|
||||
handled = await channel._bind_connection_from_connect_code(
|
||||
chat_id="wx-user-1",
|
||||
context_token="ctx-1",
|
||||
code=state,
|
||||
)
|
||||
|
||||
connections = await repo.list_connections("deerflow-user-1")
|
||||
assert handled is True
|
||||
assert len(connections) == 1
|
||||
assert connections[0]["provider"] == "wechat"
|
||||
assert connections[0]["external_account_id"] == "wx-user-1"
|
||||
assert connections[0]["workspace_id"] == "wx-user-1"
|
||||
channel._send_connection_reply.assert_awaited_once_with("wx-user-1", "ctx-1", "WeChat connected to DeerFlow.")
|
||||
await repo.close()
|
||||
|
||||
anyio.run(go)
|
||||
|
||||
|
||||
def test_wecom_connect_command_binds_identity(tmp_path):
|
||||
import anyio
|
||||
|
||||
from app.channels.wecom import WeComChannel
|
||||
|
||||
async def go():
|
||||
repo = await _make_repo(tmp_path, "wecom")
|
||||
state = "wecom-bind-code"
|
||||
await _seed_state(repo, "wecom", state)
|
||||
channel = WeComChannel(
|
||||
bus=MessageBus(),
|
||||
config={"bot_id": "bot", "bot_secret": "secret", "connection_repo": repo},
|
||||
)
|
||||
channel._ws_client = MagicMock()
|
||||
channel._ws_client.reply = AsyncMock()
|
||||
frame = {"body": {"aibotid": "bot-1", "chattype": "single"}}
|
||||
|
||||
handled = await channel._bind_connection_from_connect_code(
|
||||
frame=frame,
|
||||
user_id="wecom-user-1",
|
||||
code=state,
|
||||
)
|
||||
|
||||
connections = await repo.list_connections("deerflow-user-1")
|
||||
assert handled is True
|
||||
assert len(connections) == 1
|
||||
assert connections[0]["provider"] == "wecom"
|
||||
assert connections[0]["external_account_id"] == "wecom-user-1"
|
||||
assert connections[0]["workspace_id"] == "bot-1"
|
||||
channel._ws_client.reply.assert_awaited_once_with(frame, {"msgtype": "text", "text": {"content": "WeCom connected to DeerFlow."}})
|
||||
await repo.close()
|
||||
|
||||
anyio.run(go)
|
||||
|
||||
|
||||
def test_additional_channels_attach_owner_identity(tmp_path):
|
||||
import anyio
|
||||
|
||||
from app.channels.dingtalk import _CONVERSATION_TYPE_GROUP, DingTalkChannel
|
||||
from app.channels.feishu import FeishuChannel
|
||||
from app.channels.wechat import WechatChannel
|
||||
from app.channels.wecom import WeComChannel
|
||||
|
||||
async def go():
|
||||
repo = await _make_repo(tmp_path, "additional-identity")
|
||||
await repo.upsert_connection(
|
||||
owner_user_id="deerflow-user-1",
|
||||
provider="feishu",
|
||||
external_account_id="ou-user-1",
|
||||
workspace_id="oc-chat-1",
|
||||
)
|
||||
await repo.upsert_connection(
|
||||
owner_user_id="deerflow-user-1",
|
||||
provider="dingtalk",
|
||||
external_account_id="staff-user-1",
|
||||
workspace_id="cid-group-1",
|
||||
)
|
||||
await repo.upsert_connection(
|
||||
owner_user_id="deerflow-user-1",
|
||||
provider="wechat",
|
||||
external_account_id="wx-user-1",
|
||||
workspace_id="wx-user-1",
|
||||
)
|
||||
await repo.upsert_connection(
|
||||
owner_user_id="deerflow-user-1",
|
||||
provider="wecom",
|
||||
external_account_id="wecom-user-1",
|
||||
workspace_id="bot-1",
|
||||
)
|
||||
|
||||
cases = [
|
||||
(
|
||||
FeishuChannel(bus=MessageBus(), config={"connection_repo": repo}),
|
||||
InboundMessage(channel_name="feishu", chat_id="oc-chat-1", user_id="ou-user-1", text="hello"),
|
||||
),
|
||||
(
|
||||
DingTalkChannel(bus=MessageBus(), config={"connection_repo": repo}),
|
||||
InboundMessage(
|
||||
channel_name="dingtalk",
|
||||
chat_id="cid-group-1",
|
||||
user_id="staff-user-1",
|
||||
text="hello",
|
||||
metadata={
|
||||
"conversation_type": _CONVERSATION_TYPE_GROUP,
|
||||
"conversation_id": "cid-group-1",
|
||||
},
|
||||
),
|
||||
),
|
||||
(
|
||||
WechatChannel(bus=MessageBus(), config={"connection_repo": repo}),
|
||||
InboundMessage(channel_name="wechat", chat_id="wx-user-1", user_id="wx-user-1", text="hello"),
|
||||
),
|
||||
(
|
||||
WeComChannel(bus=MessageBus(), config={"connection_repo": repo}),
|
||||
InboundMessage(
|
||||
channel_name="wecom",
|
||||
chat_id="wecom-user-1",
|
||||
user_id="wecom-user-1",
|
||||
text="hello",
|
||||
metadata={"aibotid": "bot-1"},
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
for channel, inbound in cases:
|
||||
attached = await channel._attach_connection_identity(inbound)
|
||||
assert attached.owner_user_id == "deerflow-user-1"
|
||||
assert attached.connection_id
|
||||
assert (
|
||||
attached.workspace_id
|
||||
== {
|
||||
"feishu": "oc-chat-1",
|
||||
"dingtalk": "cid-group-1",
|
||||
"wechat": "wx-user-1",
|
||||
"wecom": "bot-1",
|
||||
}[channel.name]
|
||||
)
|
||||
|
||||
await repo.close()
|
||||
|
||||
anyio.run(go)
|
||||
@@ -140,6 +140,57 @@ def test_app_config_defaults_empty_database_to_sqlite(tmp_path, monkeypatch):
|
||||
assert config.database.sqlite_dir == ".deer-flow/data"
|
||||
|
||||
|
||||
def test_app_config_coerces_commented_out_list_sections(tmp_path, monkeypatch):
|
||||
"""Commenting out every entry under a list key makes PyYAML parse it as None.
|
||||
|
||||
Regression for the documented ``cp config.example.yaml config.yaml`` flow
|
||||
(issue #1444): such a config must load with empty lists instead of raising
|
||||
``Input should be a valid list``.
|
||||
"""
|
||||
config_path = tmp_path / "config.yaml"
|
||||
extensions_path = tmp_path / "extensions_config.json"
|
||||
_write_extensions_config(extensions_path)
|
||||
config_path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
"models": None,
|
||||
"tools": None,
|
||||
"tool_groups": None,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path))
|
||||
|
||||
config = AppConfig.from_file(str(config_path))
|
||||
|
||||
assert config.models == []
|
||||
assert config.tools == []
|
||||
assert config.tool_groups == []
|
||||
|
||||
|
||||
def test_app_config_warns_when_no_models_configured(tmp_path, monkeypatch, caplog):
|
||||
config_path = tmp_path / "config.yaml"
|
||||
extensions_path = tmp_path / "extensions_config.json"
|
||||
_write_extensions_config(extensions_path)
|
||||
config_path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
"models": None,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path))
|
||||
|
||||
with caplog.at_level("WARNING", logger="deerflow.config.app_config"):
|
||||
AppConfig.from_file(str(config_path))
|
||||
|
||||
assert "No models are configured" in caplog.text
|
||||
|
||||
|
||||
def test_get_app_config_reloads_when_file_changes(tmp_path, monkeypatch):
|
||||
config_path = tmp_path / "config.yaml"
|
||||
extensions_path = tmp_path / "extensions_config.json"
|
||||
|
||||
@@ -4,6 +4,7 @@ import pytest
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from app.gateway.auth_middleware import AuthMiddleware, _is_public
|
||||
from app.gateway.csrf_middleware import CSRFMiddleware
|
||||
|
||||
# ── _is_public unit tests ─────────────────────────────────────────────────
|
||||
|
||||
@@ -38,6 +39,8 @@ def test_public_paths(path: str):
|
||||
"/api/threads/123/uploads",
|
||||
"/api/agents",
|
||||
"/api/channels",
|
||||
"/api/channels/providers",
|
||||
"/api/channels/slack/connect",
|
||||
"/api/runs/stream",
|
||||
"/api/threads/123/runs",
|
||||
"/api/v1/auth/me",
|
||||
@@ -88,7 +91,9 @@ def test_unknown_api_path_is_protected():
|
||||
|
||||
def _make_app():
|
||||
"""Create a minimal FastAPI app with AuthMiddleware for testing."""
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, Request
|
||||
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
app = FastAPI()
|
||||
app.add_middleware(AuthMiddleware)
|
||||
@@ -98,8 +103,16 @@ def _make_app():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/api/v1/auth/me")
|
||||
async def auth_me():
|
||||
return {"id": "1", "email": "test@test.com"}
|
||||
async def auth_me(request: Request):
|
||||
from app.gateway.deps import get_current_user_from_request
|
||||
|
||||
user = await get_current_user_from_request(request)
|
||||
return {
|
||||
"id": str(user.id),
|
||||
"email": user.email,
|
||||
"system_role": user.system_role,
|
||||
"needs_setup": user.needs_setup,
|
||||
}
|
||||
|
||||
@app.get("/api/v1/auth/setup-status")
|
||||
async def setup_status():
|
||||
@@ -109,6 +122,29 @@ def _make_app():
|
||||
async def models_get():
|
||||
return {"models": []}
|
||||
|
||||
@app.get("/api/whoami")
|
||||
async def whoami(request: Request):
|
||||
user = request.state.user
|
||||
return {
|
||||
"id": str(user.id),
|
||||
"email": getattr(user, "email", None),
|
||||
"system_role": getattr(user, "system_role", None),
|
||||
"context_user_id": get_effective_user_id(),
|
||||
}
|
||||
|
||||
@app.get("/api/current-user-from-dep")
|
||||
async def current_user_from_dep(request: Request):
|
||||
from app.gateway.deps import get_current_user_from_request
|
||||
|
||||
user = await get_current_user_from_request(request)
|
||||
state_user = request.state.user
|
||||
return {
|
||||
"id": str(user.id),
|
||||
"state_id": str(state_user.id),
|
||||
"auth_source": request.state.auth_source,
|
||||
"context_user_id": get_effective_user_id(),
|
||||
}
|
||||
|
||||
@app.put("/api/mcp/config")
|
||||
async def mcp_put():
|
||||
return {"ok": True}
|
||||
@@ -132,8 +168,24 @@ def _make_app():
|
||||
return app
|
||||
|
||||
|
||||
def _make_auth_csrf_app():
|
||||
"""Create a minimal app with production middleware ordering."""
|
||||
from fastapi import FastAPI
|
||||
|
||||
app = FastAPI()
|
||||
app.add_middleware(AuthMiddleware)
|
||||
app.add_middleware(CSRFMiddleware)
|
||||
|
||||
@app.post("/api/threads/abc/runs/stream")
|
||||
async def protected_mutation():
|
||||
return {"ok": True}
|
||||
|
||||
return app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
def client(monkeypatch):
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "")
|
||||
return TestClient(_make_app())
|
||||
|
||||
|
||||
@@ -161,11 +213,145 @@ def test_protected_path_no_cookie_returns_401(client):
|
||||
assert body["detail"]["code"] == "not_authenticated"
|
||||
|
||||
|
||||
def test_auth_disabled_allows_protected_path_without_cookie(monkeypatch):
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
client = TestClient(_make_app())
|
||||
|
||||
res = client.get("/api/models")
|
||||
|
||||
assert res.status_code == 200
|
||||
assert res.json() == {"models": []}
|
||||
|
||||
|
||||
def test_auth_disabled_stamps_default_admin_user_without_cookie(monkeypatch):
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
client = TestClient(_make_app())
|
||||
|
||||
res = client.get("/api/whoami")
|
||||
|
||||
assert res.status_code == 200
|
||||
assert res.json() == {
|
||||
"id": "default",
|
||||
"email": "default@test.local",
|
||||
"system_role": "admin",
|
||||
"context_user_id": "default",
|
||||
}
|
||||
|
||||
|
||||
def test_auth_disabled_auth_me_reuses_middleware_user_without_cookie(monkeypatch):
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
client = TestClient(_make_app())
|
||||
|
||||
res = client.get("/api/v1/auth/me")
|
||||
|
||||
assert res.status_code == 200
|
||||
assert res.json() == {
|
||||
"id": "default",
|
||||
"email": "default@test.local",
|
||||
"system_role": "admin",
|
||||
"needs_setup": False,
|
||||
}
|
||||
|
||||
|
||||
def test_auth_disabled_does_not_clobber_valid_session_cookie(monkeypatch):
|
||||
from types import SimpleNamespace
|
||||
|
||||
async def fake_current_user(request):
|
||||
return SimpleNamespace(
|
||||
id="session-user",
|
||||
email="session@test.local",
|
||||
system_role="user",
|
||||
needs_setup=False,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
monkeypatch.setattr("app.gateway.deps.get_current_user_from_request", fake_current_user)
|
||||
client = TestClient(_make_app())
|
||||
|
||||
res = client.get("/api/whoami", cookies={"access_token": "valid-session"})
|
||||
|
||||
assert res.status_code == 200
|
||||
assert res.json() == {
|
||||
"id": "session-user",
|
||||
"email": "session@test.local",
|
||||
"system_role": "user",
|
||||
"context_user_id": "session-user",
|
||||
}
|
||||
|
||||
|
||||
def test_auth_disabled_does_not_clobber_internal_auth_identity(monkeypatch):
|
||||
from app.gateway.internal_auth import create_internal_auth_headers
|
||||
from deerflow.runtime.user_context import DEFAULT_USER_ID
|
||||
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
client = TestClient(_make_app())
|
||||
|
||||
res = client.get(
|
||||
"/api/current-user-from-dep",
|
||||
headers=create_internal_auth_headers(),
|
||||
)
|
||||
|
||||
assert res.status_code == 200
|
||||
assert res.json() == {
|
||||
"id": DEFAULT_USER_ID,
|
||||
"state_id": DEFAULT_USER_ID,
|
||||
"auth_source": "internal",
|
||||
"context_user_id": DEFAULT_USER_ID,
|
||||
}
|
||||
|
||||
|
||||
def test_auth_disabled_skips_csrf_for_state_changing_requests(monkeypatch):
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
client = TestClient(_make_auth_csrf_app())
|
||||
|
||||
res = client.post("/api/threads/abc/runs/stream")
|
||||
|
||||
assert res.status_code == 200
|
||||
assert res.json() == {"ok": True}
|
||||
|
||||
|
||||
def test_auth_disabled_is_ignored_in_explicit_production_env(monkeypatch):
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
monkeypatch.setenv("DEER_FLOW_ENV", "production")
|
||||
client = TestClient(_make_app())
|
||||
|
||||
res = client.get("/api/models")
|
||||
|
||||
assert res.status_code == 401
|
||||
|
||||
|
||||
def test_auth_disabled_startup_warning_when_effective(monkeypatch, caplog):
|
||||
from app.gateway.auth_disabled import warn_if_auth_disabled_enabled
|
||||
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
monkeypatch.delenv("DEER_FLOW_ENV", raising=False)
|
||||
monkeypatch.delenv("ENVIRONMENT", raising=False)
|
||||
|
||||
with caplog.at_level("WARNING", logger="app.gateway.auth_disabled"):
|
||||
warn_if_auth_disabled_enabled()
|
||||
|
||||
assert "authentication is bypassed" in caplog.text
|
||||
assert "default" in caplog.text
|
||||
|
||||
|
||||
def test_auth_disabled_startup_warning_suppressed_in_explicit_production_env(monkeypatch, caplog):
|
||||
from app.gateway.auth_disabled import warn_if_auth_disabled_enabled
|
||||
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
monkeypatch.setenv("ENVIRONMENT", "production")
|
||||
|
||||
with caplog.at_level("WARNING", logger="app.gateway.auth_disabled"):
|
||||
warn_if_auth_disabled_enabled()
|
||||
|
||||
assert "authentication is bypassed" not in caplog.text
|
||||
|
||||
|
||||
def test_protected_path_with_junk_cookie_rejected(client):
|
||||
"""Junk cookie → 401. Middleware strictly validates the JWT now
|
||||
(AUTH_TEST_PLAN test 7.5.8); it no longer silently passes bad
|
||||
tokens through to the route handler."""
|
||||
res = client.get("/api/models", cookies={"access_token": "some-token"})
|
||||
client.cookies.set("access_token", "some-token")
|
||||
res = client.get("/api/models")
|
||||
assert res.status_code == 401
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Tests for user-facing IM channel connection configuration."""
|
||||
|
||||
from deerflow.config.channel_connections_config import ChannelConnectionsConfig
|
||||
|
||||
|
||||
def test_channel_connections_disabled_by_default():
|
||||
config = ChannelConnectionsConfig()
|
||||
|
||||
assert config.enabled is False
|
||||
assert config.slack.enabled is False
|
||||
assert config.telegram.enabled is False
|
||||
assert config.discord.enabled is False
|
||||
assert config.feishu.enabled is False
|
||||
assert config.dingtalk.enabled is False
|
||||
assert config.wechat.enabled is False
|
||||
assert config.wecom.enabled is False
|
||||
|
||||
|
||||
def test_enabled_channel_connections_do_not_require_public_url_or_encryption_key():
|
||||
config = ChannelConnectionsConfig.model_validate(
|
||||
{
|
||||
"enabled": True,
|
||||
"telegram": {
|
||||
"enabled": True,
|
||||
"bot_username": "deerflow_bot",
|
||||
},
|
||||
"slack": {"enabled": True},
|
||||
"discord": {"enabled": True},
|
||||
"feishu": {"enabled": True},
|
||||
"dingtalk": {"enabled": True},
|
||||
"wechat": {"enabled": True},
|
||||
"wecom": {"enabled": True},
|
||||
}
|
||||
)
|
||||
|
||||
assert config.enabled is True
|
||||
assert config.provider_status("telegram") == {"enabled": True, "configured": True}
|
||||
assert config.provider_status("slack") == {"enabled": True, "configured": True}
|
||||
assert config.provider_status("discord") == {"enabled": True, "configured": True}
|
||||
assert config.provider_status("feishu") == {"enabled": True, "configured": True}
|
||||
assert config.provider_status("dingtalk") == {"enabled": True, "configured": True}
|
||||
assert config.provider_status("wechat") == {"enabled": True, "configured": True}
|
||||
assert config.provider_status("wecom") == {"enabled": True, "configured": True}
|
||||
|
||||
|
||||
def test_provider_status_reports_disabled_and_unknown_providers():
|
||||
config = ChannelConnectionsConfig.model_validate({"enabled": True})
|
||||
|
||||
assert config.provider_status("slack") == {"enabled": False, "configured": False}
|
||||
assert config.provider_status("telegram") == {"enabled": False, "configured": False}
|
||||
assert config.provider_status("discord") == {"enabled": False, "configured": False}
|
||||
assert config.provider_status("feishu") == {"enabled": False, "configured": False}
|
||||
assert config.provider_status("dingtalk") == {"enabled": False, "configured": False}
|
||||
assert config.provider_status("wechat") == {"enabled": False, "configured": False}
|
||||
assert config.provider_status("wecom") == {"enabled": False, "configured": False}
|
||||
assert config.provider_status("unknown") == {"enabled": False, "configured": False}
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Tests for per-user IM channel connection persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from deerflow.persistence.channel_connections import (
|
||||
ChannelConnectionRepository,
|
||||
ChannelConnectionRow,
|
||||
ChannelCredentialCipher,
|
||||
ChannelCredentialRow,
|
||||
ChannelOAuthStateRow,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def repo(tmp_path):
|
||||
from deerflow.persistence.engine import close_engine, get_session_factory, init_engine
|
||||
|
||||
url = f"sqlite+aiosqlite:///{tmp_path / 'channels.db'}"
|
||||
await init_engine("sqlite", url=url, sqlite_dir=str(tmp_path))
|
||||
try:
|
||||
yield ChannelConnectionRepository(
|
||||
get_session_factory(),
|
||||
cipher=ChannelCredentialCipher.from_key("test-encryption-key"),
|
||||
)
|
||||
finally:
|
||||
await close_engine()
|
||||
|
||||
|
||||
class TestChannelConnectionRepository:
|
||||
@pytest.mark.anyio
|
||||
async def test_connections_are_listed_per_owner(self, repo):
|
||||
alice = await repo.upsert_connection(
|
||||
owner_user_id="alice",
|
||||
provider="slack",
|
||||
external_account_id="U-alice",
|
||||
external_account_name="Alice",
|
||||
workspace_id="T1",
|
||||
workspace_name="Team One",
|
||||
scopes=["chat:write"],
|
||||
)
|
||||
await repo.upsert_connection(
|
||||
owner_user_id="bob",
|
||||
provider="slack",
|
||||
external_account_id="U-bob",
|
||||
external_account_name="Bob",
|
||||
workspace_id="T1",
|
||||
workspace_name="Team One",
|
||||
scopes=["chat:write"],
|
||||
)
|
||||
|
||||
results = await repo.list_connections("alice")
|
||||
|
||||
assert [item["id"] for item in results] == [alice["id"]]
|
||||
assert results[0]["owner_user_id"] == "alice"
|
||||
assert results[0]["provider"] == "slack"
|
||||
assert results[0]["scopes"] == ["chat:write"]
|
||||
assert "encrypted_access_token" not in results[0]
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_upsert_connection_updates_existing_provider_identity(self, repo):
|
||||
first = await repo.upsert_connection(
|
||||
owner_user_id="alice",
|
||||
provider="telegram",
|
||||
external_account_id="42",
|
||||
external_account_name="Alice",
|
||||
workspace_id=None,
|
||||
workspace_name=None,
|
||||
status="pending",
|
||||
)
|
||||
second = await repo.upsert_connection(
|
||||
owner_user_id="alice",
|
||||
provider="telegram",
|
||||
external_account_id="42",
|
||||
external_account_name="Alice Telegram",
|
||||
workspace_id=None,
|
||||
workspace_name=None,
|
||||
status="connected",
|
||||
)
|
||||
|
||||
assert second["id"] == first["id"]
|
||||
assert second["status"] == "connected"
|
||||
assert second["external_account_name"] == "Alice Telegram"
|
||||
assert len(await repo.list_connections("alice")) == 1
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_credentials_are_encrypted_at_rest_and_decrypted_by_repository(self, repo):
|
||||
connection = await repo.upsert_connection(
|
||||
owner_user_id="alice",
|
||||
provider="slack",
|
||||
external_account_id="U-alice",
|
||||
workspace_id="T1",
|
||||
)
|
||||
expires_at = datetime.now(UTC) + timedelta(hours=1)
|
||||
|
||||
await repo.store_credentials(
|
||||
connection["id"],
|
||||
access_token="xoxb-secret-access-token",
|
||||
refresh_token="secret-refresh-token",
|
||||
token_type="Bearer",
|
||||
expires_at=expires_at,
|
||||
extra={"bot_user_id": "B123"},
|
||||
)
|
||||
|
||||
async with repo.session_factory() as session:
|
||||
row = (await session.execute(select(ChannelCredentialRow))).scalar_one()
|
||||
assert row.encrypted_access_token is not None
|
||||
assert "xoxb-secret-access-token" not in row.encrypted_access_token
|
||||
assert "secret-refresh-token" not in (row.encrypted_refresh_token or "")
|
||||
assert "B123" not in (row.encrypted_extra_json or "")
|
||||
|
||||
credentials = await repo.get_credentials(connection["id"])
|
||||
|
||||
assert credentials is not None
|
||||
assert credentials["access_token"] == "xoxb-secret-access-token"
|
||||
assert credentials["refresh_token"] == "secret-refresh-token"
|
||||
assert credentials["token_type"] == "Bearer"
|
||||
assert credentials["expires_at"] == expires_at
|
||||
assert credentials["extra"] == {"bot_user_id": "B123"}
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_conversations_are_scoped_by_connection(self, repo):
|
||||
alice = await repo.upsert_connection(
|
||||
owner_user_id="alice",
|
||||
provider="slack",
|
||||
external_account_id="U-alice",
|
||||
workspace_id="T1",
|
||||
)
|
||||
bob = await repo.upsert_connection(
|
||||
owner_user_id="bob",
|
||||
provider="slack",
|
||||
external_account_id="U-bob",
|
||||
workspace_id="T1",
|
||||
)
|
||||
|
||||
await repo.set_thread_id(
|
||||
connection_id=alice["id"],
|
||||
owner_user_id="alice",
|
||||
provider="slack",
|
||||
external_conversation_id="C-shared",
|
||||
external_topic_id="1710000000.000100",
|
||||
thread_id="thread-alice",
|
||||
)
|
||||
await repo.set_thread_id(
|
||||
connection_id=bob["id"],
|
||||
owner_user_id="bob",
|
||||
provider="slack",
|
||||
external_conversation_id="C-shared",
|
||||
external_topic_id="1710000000.000100",
|
||||
thread_id="thread-bob",
|
||||
)
|
||||
|
||||
assert await repo.get_thread_id(alice["id"], "C-shared", "1710000000.000100") == "thread-alice"
|
||||
assert await repo.get_thread_id(bob["id"], "C-shared", "1710000000.000100") == "thread-bob"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_disconnect_connection_revokes_owner_connection_and_removes_credentials(self, repo):
|
||||
connection = await repo.upsert_connection(
|
||||
owner_user_id="alice",
|
||||
provider="telegram",
|
||||
external_account_id="42",
|
||||
)
|
||||
await repo.store_credentials(connection["id"], access_token="secret-token")
|
||||
|
||||
disconnected = await repo.disconnect_connection(
|
||||
connection_id=connection["id"],
|
||||
owner_user_id="alice",
|
||||
)
|
||||
|
||||
assert disconnected is True
|
||||
async with repo.session_factory() as session:
|
||||
connection_row = await session.get(ChannelConnectionRow, connection["id"])
|
||||
credential_row = await session.get(ChannelCredentialRow, connection["id"])
|
||||
assert connection_row is not None
|
||||
assert connection_row.status == "revoked"
|
||||
assert credential_row is None
|
||||
assert (
|
||||
await repo.find_connection_by_external_identity(
|
||||
provider="telegram",
|
||||
external_account_id="42",
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_disconnect_connection_is_owner_scoped(self, repo):
|
||||
connection = await repo.upsert_connection(
|
||||
owner_user_id="alice",
|
||||
provider="telegram",
|
||||
external_account_id="42",
|
||||
)
|
||||
|
||||
disconnected = await repo.disconnect_connection(
|
||||
connection_id=connection["id"],
|
||||
owner_user_id="bob",
|
||||
)
|
||||
|
||||
assert disconnected is False
|
||||
assert (await repo.list_connections("alice"))[0]["status"] == "connected"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_consume_oauth_state_deletes_expired_states(self, repo):
|
||||
now = datetime.now(UTC)
|
||||
await repo.create_oauth_state(
|
||||
owner_user_id="alice",
|
||||
provider="slack",
|
||||
state="expired-state",
|
||||
expires_at=now - timedelta(minutes=1),
|
||||
)
|
||||
await repo.create_oauth_state(
|
||||
owner_user_id="alice",
|
||||
provider="slack",
|
||||
state="active-state",
|
||||
expires_at=now + timedelta(minutes=5),
|
||||
)
|
||||
|
||||
consumed = await repo.consume_oauth_state(provider="slack", state="expired-state", now=now)
|
||||
|
||||
assert consumed is None
|
||||
async with repo.session_factory() as session:
|
||||
states = (await session.execute(select(ChannelOAuthStateRow))).scalars().all()
|
||||
assert [state.state_hash for state in states] == [repo.hash_state("active-state")]
|
||||
@@ -0,0 +1,707 @@
|
||||
"""Router tests for browser-connectable IM channels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID
|
||||
|
||||
from _router_auth_helpers import make_authed_test_app
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.channels.runtime_config_store import ChannelRuntimeConfigStore
|
||||
from app.gateway.auth.models import User
|
||||
from app.gateway.routers import channel_connections
|
||||
from deerflow.config.channel_connections_config import ChannelConnectionsConfig
|
||||
|
||||
|
||||
def _user() -> User:
|
||||
return User(
|
||||
id=UUID("11111111-2222-3333-4444-555555555555"),
|
||||
email="alice@example.com",
|
||||
password_hash="x",
|
||||
system_role="user",
|
||||
)
|
||||
|
||||
|
||||
async def _make_repo(tmp_path):
|
||||
from deerflow.persistence.channel_connections import ChannelConnectionRepository
|
||||
from deerflow.persistence.engine import get_session_factory, init_engine
|
||||
|
||||
await init_engine("sqlite", url=f"sqlite+aiosqlite:///{tmp_path / 'router.db'}", sqlite_dir=str(tmp_path))
|
||||
return ChannelConnectionRepository(get_session_factory())
|
||||
|
||||
|
||||
def _make_app(
|
||||
config: ChannelConnectionsConfig,
|
||||
repo,
|
||||
channels_config: dict | None = None,
|
||||
*,
|
||||
runtime_config_store: ChannelRuntimeConfigStore | None = None,
|
||||
set_channels_config_state: bool = True,
|
||||
):
|
||||
app = make_authed_test_app(user_factory=_user)
|
||||
app.state.channel_connections_config = config
|
||||
app.state.channel_connection_repo = repo
|
||||
if set_channels_config_state:
|
||||
app.state.channels_config = channels_config or {}
|
||||
if runtime_config_store is not None:
|
||||
app.state.channel_runtime_config_store = runtime_config_store
|
||||
app.include_router(channel_connections.router)
|
||||
return app
|
||||
|
||||
|
||||
def _enabled_connections_config() -> ChannelConnectionsConfig:
|
||||
return ChannelConnectionsConfig.model_validate(
|
||||
{
|
||||
"enabled": True,
|
||||
"telegram": {"enabled": True, "bot_username": "deerflow_bot"},
|
||||
"slack": {"enabled": True},
|
||||
"discord": {"enabled": True},
|
||||
"feishu": {"enabled": True},
|
||||
"dingtalk": {"enabled": True},
|
||||
"wechat": {"enabled": True},
|
||||
"wecom": {"enabled": True},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _channels_config() -> dict:
|
||||
return {
|
||||
"telegram": {"enabled": True, "bot_token": "telegram-token"},
|
||||
"slack": {"enabled": True, "bot_token": "xoxb-operator", "app_token": "xapp-operator"},
|
||||
"discord": {"enabled": True, "bot_token": "discord-bot"},
|
||||
"feishu": {"enabled": True, "app_id": "feishu-app", "app_secret": "feishu-secret"},
|
||||
"dingtalk": {"enabled": True, "client_id": "dingtalk-client", "client_secret": "dingtalk-secret"},
|
||||
"wechat": {"enabled": True, "bot_token": "wechat-token"},
|
||||
"wecom": {"enabled": True, "bot_id": "wecom-bot", "bot_secret": "wecom-secret"},
|
||||
}
|
||||
|
||||
|
||||
def test_get_providers_only_returns_enabled_channels_and_setup_fields(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
config = ChannelConnectionsConfig.model_validate(
|
||||
{
|
||||
"enabled": True,
|
||||
"slack": {"enabled": True},
|
||||
"discord": {"enabled": False},
|
||||
}
|
||||
)
|
||||
app = _make_app(config, repo, {})
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/api/channels/providers")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["enabled"] is True
|
||||
assert [provider["provider"] for provider in body["providers"]] == ["slack"]
|
||||
assert body["providers"][0]["configured"] is False
|
||||
assert body["providers"][0]["connectable"] is False
|
||||
assert body["providers"][0]["credential_fields"] == [
|
||||
{
|
||||
"name": "bot_token",
|
||||
"label": "Bot token",
|
||||
"type": "password",
|
||||
"required": True,
|
||||
},
|
||||
{
|
||||
"name": "app_token",
|
||||
"label": "App token",
|
||||
"type": "password",
|
||||
"required": True,
|
||||
},
|
||||
]
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_get_providers_uses_existing_channels_config(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
app = _make_app(_enabled_connections_config(), repo, _channels_config())
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/api/channels/providers")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["enabled"] is True
|
||||
by_provider = {item["provider"]: item for item in body["providers"]}
|
||||
assert set(by_provider) == {"telegram", "slack", "discord", "feishu", "dingtalk", "wechat", "wecom"}
|
||||
assert by_provider["telegram"]["configured"] is True
|
||||
assert by_provider["telegram"]["auth_mode"] == "deep_link"
|
||||
assert by_provider["telegram"]["credential_values"] == {
|
||||
"bot_token": "********",
|
||||
"bot_username": "deerflow_bot",
|
||||
}
|
||||
assert by_provider["slack"]["configured"] is True
|
||||
assert by_provider["slack"]["auth_mode"] == "binding_code"
|
||||
assert by_provider["slack"]["connection_status"] == "connected"
|
||||
assert by_provider["slack"]["credential_values"] == {
|
||||
"bot_token": "********",
|
||||
"app_token": "********",
|
||||
}
|
||||
assert by_provider["discord"]["configured"] is True
|
||||
assert by_provider["discord"]["auth_mode"] == "binding_code"
|
||||
assert by_provider["discord"]["credential_values"] == {"bot_token": "********"}
|
||||
assert by_provider["feishu"]["configured"] is True
|
||||
assert by_provider["feishu"]["auth_mode"] == "binding_code"
|
||||
assert by_provider["feishu"]["connection_status"] == "connected"
|
||||
assert by_provider["feishu"]["credential_values"] == {
|
||||
"app_id": "feishu-app",
|
||||
"app_secret": "********",
|
||||
}
|
||||
assert by_provider["dingtalk"]["configured"] is True
|
||||
assert by_provider["dingtalk"]["auth_mode"] == "binding_code"
|
||||
assert by_provider["dingtalk"]["credential_values"] == {
|
||||
"client_id": "dingtalk-client",
|
||||
"client_secret": "********",
|
||||
}
|
||||
assert by_provider["wechat"]["configured"] is True
|
||||
assert by_provider["wechat"]["auth_mode"] == "binding_code"
|
||||
assert by_provider["wechat"]["credential_values"] == {"bot_token": "********"}
|
||||
assert by_provider["wecom"]["configured"] is True
|
||||
assert by_provider["wecom"]["auth_mode"] == "binding_code"
|
||||
assert by_provider["wecom"]["credential_values"] == {
|
||||
"bot_id": "wecom-bot",
|
||||
"bot_secret": "********",
|
||||
}
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_get_providers_reports_unconfigured_when_runtime_channel_is_missing(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
app = _make_app(_enabled_connections_config(), repo, {"telegram": {"enabled": True, "bot_token": "telegram-token"}})
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/api/channels/providers")
|
||||
|
||||
assert response.status_code == 200
|
||||
by_provider = {item["provider"]: item for item in response.json()["providers"]}
|
||||
assert by_provider["telegram"]["configured"] is True
|
||||
assert by_provider["slack"]["configured"] is False
|
||||
assert by_provider["slack"]["connectable"] is False
|
||||
assert "Slack credentials" in by_provider["slack"]["unavailable_reason"]
|
||||
assert by_provider["discord"]["configured"] is False
|
||||
assert "Discord credentials" in by_provider["discord"]["unavailable_reason"]
|
||||
assert by_provider["feishu"]["configured"] is False
|
||||
assert "Feishu credentials" in by_provider["feishu"]["unavailable_reason"]
|
||||
assert by_provider["dingtalk"]["configured"] is False
|
||||
assert "DingTalk credentials" in by_provider["dingtalk"]["unavailable_reason"]
|
||||
assert by_provider["wechat"]["configured"] is False
|
||||
assert "WeChat credentials" in by_provider["wechat"]["unavailable_reason"]
|
||||
assert by_provider["wecom"]["configured"] is False
|
||||
assert "WeCom credentials" in by_provider["wecom"]["unavailable_reason"]
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_get_providers_reports_configured_channel_not_running(tmp_path, monkeypatch):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
app = _make_app(_enabled_connections_config(), repo, _channels_config())
|
||||
service = SimpleNamespace(
|
||||
get_status=lambda: {
|
||||
"service_running": True,
|
||||
"channels": {
|
||||
"feishu": {
|
||||
"enabled": True,
|
||||
"running": False,
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr("app.channels.service.get_channel_service", lambda: service)
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/api/channels/providers")
|
||||
|
||||
assert response.status_code == 200
|
||||
by_provider = {item["provider"]: item for item in response.json()["providers"]}
|
||||
assert by_provider["feishu"]["configured"] is True
|
||||
assert by_provider["feishu"]["connectable"] is False
|
||||
assert by_provider["feishu"]["connection_status"] == "not_connected"
|
||||
assert "configured but is not running" in by_provider["feishu"]["unavailable_reason"]
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_get_providers_uses_newest_connection_status_per_provider(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
|
||||
async def seed_connections():
|
||||
await repo.upsert_connection(
|
||||
owner_user_id=str(_user().id),
|
||||
provider="slack",
|
||||
external_account_id="U-old",
|
||||
workspace_id="T-old",
|
||||
status="revoked",
|
||||
)
|
||||
await anyio.sleep(0.01)
|
||||
await repo.upsert_connection(
|
||||
owner_user_id=str(_user().id),
|
||||
provider="slack",
|
||||
external_account_id="U-new",
|
||||
workspace_id="T-new",
|
||||
status="connected",
|
||||
)
|
||||
|
||||
anyio.run(seed_connections)
|
||||
app = _make_app(_enabled_connections_config(), repo, _channels_config())
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/api/channels/providers")
|
||||
|
||||
assert response.status_code == 200
|
||||
by_provider = {item["provider"]: item for item in response.json()["providers"]}
|
||||
assert by_provider["slack"]["connection_status"] == "connected"
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_get_connections_returns_current_user_connections_only(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
|
||||
async def seed_connections():
|
||||
await repo.upsert_connection(
|
||||
owner_user_id=str(_user().id),
|
||||
provider="telegram",
|
||||
external_account_id="42",
|
||||
external_account_name="Alice",
|
||||
status="connected",
|
||||
)
|
||||
await repo.upsert_connection(
|
||||
owner_user_id="other-user",
|
||||
provider="telegram",
|
||||
external_account_id="99",
|
||||
external_account_name="Bob",
|
||||
status="connected",
|
||||
)
|
||||
|
||||
anyio.run(seed_connections)
|
||||
app = _make_app(_enabled_connections_config(), repo, _channels_config())
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/api/channels/connections")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert len(body["connections"]) == 1
|
||||
assert body["connections"][0]["provider"] == "telegram"
|
||||
assert body["connections"][0]["external_account_id"] == "42"
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_connect_telegram_returns_deep_link_and_persists_state(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
app = _make_app(_enabled_connections_config(), repo, _channels_config())
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/api/channels/telegram/connect")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["provider"] == "telegram"
|
||||
assert body["mode"] == "deep_link"
|
||||
assert body["url"].startswith("https://t.me/deerflow_bot?start=")
|
||||
assert body["code"]
|
||||
assert "/start" in body["instruction"]
|
||||
|
||||
async def count_states():
|
||||
return await repo.count_oauth_states(owner_user_id=str(_user().id), provider="telegram")
|
||||
|
||||
assert anyio.run(count_states) == 1
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_connect_slack_returns_binding_command_and_persists_state(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
app = _make_app(_enabled_connections_config(), repo, _channels_config())
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/api/channels/slack/connect")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["provider"] == "slack"
|
||||
assert body["mode"] == "binding_code"
|
||||
assert body["url"] is None
|
||||
assert len(body["code"]) >= 22
|
||||
assert body["instruction"] == f"Send /connect {body['code']} to the DeerFlow Slack bot."
|
||||
|
||||
async def count_states():
|
||||
return await repo.count_oauth_states(owner_user_id=str(_user().id), provider="slack")
|
||||
|
||||
assert anyio.run(count_states) == 1
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_connect_discord_returns_binding_command_and_persists_state(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
app = _make_app(_enabled_connections_config(), repo, _channels_config())
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/api/channels/discord/connect")
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["provider"] == "discord"
|
||||
assert body["mode"] == "binding_code"
|
||||
assert body["url"] is None
|
||||
assert body["code"]
|
||||
assert body["instruction"] == f"Send /connect {body['code']} to the DeerFlow Discord bot."
|
||||
|
||||
async def count_states():
|
||||
return await repo.count_oauth_states(owner_user_id=str(_user().id), provider="discord")
|
||||
|
||||
assert anyio.run(count_states) == 1
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_connect_existing_binding_code_channels_return_command_and_persist_state(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
app = _make_app(_enabled_connections_config(), repo, _channels_config())
|
||||
|
||||
providers = ["feishu", "dingtalk", "wechat", "wecom"]
|
||||
with TestClient(app) as client:
|
||||
responses = {provider: client.post(f"/api/channels/{provider}/connect") for provider in providers}
|
||||
|
||||
for provider, response in responses.items():
|
||||
expected_display_name = {
|
||||
"feishu": "Feishu",
|
||||
"dingtalk": "DingTalk",
|
||||
"wechat": "WeChat",
|
||||
"wecom": "WeCom",
|
||||
}[provider]
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["provider"] == provider
|
||||
assert body["mode"] == "binding_code"
|
||||
assert body["url"] is None
|
||||
assert len(body["code"]) >= 22
|
||||
assert body["instruction"] == f"Send /connect {body['code']} to the DeerFlow {expected_display_name} bot."
|
||||
|
||||
async def count_states(provider=provider):
|
||||
return await repo.count_oauth_states(owner_user_id=str(_user().id), provider=provider)
|
||||
|
||||
assert anyio.run(count_states) == 1
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_connect_unconfigured_runtime_channel_returns_400(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
app = _make_app(_enabled_connections_config(), repo, {})
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/api/channels/slack/connect")
|
||||
|
||||
assert response.status_code == 400
|
||||
assert "Slack credentials" in response.json()["detail"]
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_configure_provider_runtime_credentials_enables_connect_without_file_edits(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
config = ChannelConnectionsConfig.model_validate(
|
||||
{
|
||||
"enabled": True,
|
||||
"slack": {"enabled": True},
|
||||
}
|
||||
)
|
||||
app = _make_app(config, repo, {})
|
||||
|
||||
with TestClient(app) as client:
|
||||
configure_response = client.post(
|
||||
"/api/channels/slack/runtime-config",
|
||||
json={"values": {"bot_token": "xoxb-ui", "app_token": "xapp-ui"}},
|
||||
)
|
||||
connect_response = client.post("/api/channels/slack/connect")
|
||||
|
||||
assert configure_response.status_code == 200
|
||||
configured = configure_response.json()
|
||||
assert configured["provider"] == "slack"
|
||||
assert configured["configured"] is True
|
||||
assert configured["connectable"] is True
|
||||
assert configured["connection_status"] == "connected"
|
||||
assert app.state.channels_config["slack"] == {
|
||||
"enabled": True,
|
||||
"bot_token": "xoxb-ui",
|
||||
"app_token": "xapp-ui",
|
||||
}
|
||||
assert connect_response.status_code == 200
|
||||
assert connect_response.json()["provider"] == "slack"
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_configure_provider_runtime_credentials_survive_local_restart(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
config = ChannelConnectionsConfig.model_validate(
|
||||
{
|
||||
"enabled": True,
|
||||
"slack": {"enabled": True},
|
||||
}
|
||||
)
|
||||
runtime_config_path = tmp_path / "channels" / "runtime-config.json"
|
||||
first_app = _make_app(
|
||||
config,
|
||||
repo,
|
||||
{},
|
||||
runtime_config_store=ChannelRuntimeConfigStore(runtime_config_path),
|
||||
)
|
||||
|
||||
with TestClient(first_app) as client:
|
||||
configure_response = client.post(
|
||||
"/api/channels/slack/runtime-config",
|
||||
json={"values": {"bot_token": "xoxb-ui", "app_token": "xapp-ui"}},
|
||||
)
|
||||
|
||||
assert configure_response.status_code == 200
|
||||
|
||||
restarted_app = _make_app(
|
||||
config,
|
||||
repo,
|
||||
runtime_config_store=ChannelRuntimeConfigStore(runtime_config_path),
|
||||
set_channels_config_state=False,
|
||||
)
|
||||
|
||||
with TestClient(restarted_app) as client:
|
||||
response = client.get("/api/channels/providers")
|
||||
|
||||
assert response.status_code == 200
|
||||
by_provider = {item["provider"]: item for item in response.json()["providers"]}
|
||||
assert by_provider["slack"]["configured"] is True
|
||||
assert by_provider["slack"]["connectable"] is True
|
||||
assert by_provider["slack"]["connection_status"] == "connected"
|
||||
assert restarted_app.state.channels_config["slack"] == {
|
||||
"enabled": True,
|
||||
"bot_token": "xoxb-ui",
|
||||
"app_token": "xapp-ui",
|
||||
}
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_configure_provider_runtime_credentials_preserves_masked_secrets(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
config = ChannelConnectionsConfig.model_validate(
|
||||
{
|
||||
"enabled": True,
|
||||
"feishu": {"enabled": True},
|
||||
}
|
||||
)
|
||||
runtime_config_store = ChannelRuntimeConfigStore(tmp_path / "channels" / "runtime-config.json")
|
||||
app = _make_app(
|
||||
config,
|
||||
repo,
|
||||
{
|
||||
"feishu": {
|
||||
"enabled": True,
|
||||
"app_id": "old-app-id",
|
||||
"app_secret": "old-secret",
|
||||
}
|
||||
},
|
||||
runtime_config_store=runtime_config_store,
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
configure_response = client.post(
|
||||
"/api/channels/feishu/runtime-config",
|
||||
json={
|
||||
"values": {
|
||||
"app_id": "new-app-id",
|
||||
"app_secret": "********",
|
||||
}
|
||||
},
|
||||
)
|
||||
providers_response = client.get("/api/channels/providers")
|
||||
|
||||
assert configure_response.status_code == 200
|
||||
assert app.state.channels_config["feishu"] == {
|
||||
"enabled": True,
|
||||
"app_id": "new-app-id",
|
||||
"app_secret": "old-secret",
|
||||
}
|
||||
assert runtime_config_store.get_provider_config("feishu") == {
|
||||
"enabled": True,
|
||||
"app_id": "new-app-id",
|
||||
"app_secret": "old-secret",
|
||||
}
|
||||
by_provider = {item["provider"]: item for item in providers_response.json()["providers"]}
|
||||
assert by_provider["feishu"]["credential_values"] == {
|
||||
"app_id": "new-app-id",
|
||||
"app_secret": "********",
|
||||
}
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_disconnect_provider_runtime_config_clears_connected_state(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
config = ChannelConnectionsConfig.model_validate(
|
||||
{
|
||||
"enabled": True,
|
||||
"slack": {"enabled": True},
|
||||
}
|
||||
)
|
||||
runtime_config_store = ChannelRuntimeConfigStore(tmp_path / "channels" / "runtime-config.json")
|
||||
app = _make_app(config, repo, {}, runtime_config_store=runtime_config_store)
|
||||
|
||||
with TestClient(app) as client:
|
||||
configure_response = client.post(
|
||||
"/api/channels/slack/runtime-config",
|
||||
json={"values": {"bot_token": "xoxb-ui", "app_token": "xapp-ui"}},
|
||||
)
|
||||
disconnect_response = client.delete("/api/channels/slack/runtime-config")
|
||||
providers_response = client.get("/api/channels/providers")
|
||||
|
||||
assert configure_response.status_code == 200
|
||||
assert disconnect_response.status_code == 200
|
||||
disconnected = disconnect_response.json()
|
||||
assert disconnected["provider"] == "slack"
|
||||
assert disconnected["configured"] is False
|
||||
assert disconnected["connectable"] is False
|
||||
assert disconnected["connection_status"] == "not_connected"
|
||||
assert runtime_config_store.get_provider_config("slack") is None
|
||||
|
||||
assert providers_response.status_code == 200
|
||||
by_provider = {item["provider"]: item for item in providers_response.json()["providers"]}
|
||||
assert by_provider["slack"]["connection_status"] == "not_connected"
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_disconnect_provider_runtime_config_revokes_current_user_provider_connections(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
|
||||
async def seed_connection():
|
||||
await repo.upsert_connection(
|
||||
owner_user_id=str(_user().id),
|
||||
provider="slack",
|
||||
external_account_id="U123",
|
||||
status="connected",
|
||||
)
|
||||
|
||||
anyio.run(seed_connection)
|
||||
config = ChannelConnectionsConfig.model_validate(
|
||||
{
|
||||
"enabled": True,
|
||||
"slack": {"enabled": True},
|
||||
}
|
||||
)
|
||||
runtime_config_store = ChannelRuntimeConfigStore(tmp_path / "channels" / "runtime-config.json")
|
||||
app = _make_app(config, repo, {}, runtime_config_store=runtime_config_store)
|
||||
|
||||
with TestClient(app) as client:
|
||||
configure_response = client.post(
|
||||
"/api/channels/slack/runtime-config",
|
||||
json={"values": {"bot_token": "xoxb-ui", "app_token": "xapp-ui"}},
|
||||
)
|
||||
disconnect_response = client.delete("/api/channels/slack/runtime-config")
|
||||
|
||||
assert configure_response.status_code == 200
|
||||
assert disconnect_response.status_code == 200
|
||||
|
||||
async def get_connection_status():
|
||||
return (await repo.list_connections(str(_user().id)))[0]["status"]
|
||||
|
||||
assert anyio.run(get_connection_status) == "revoked"
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_disconnect_connection_revokes_current_user_connection(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
|
||||
async def seed_connection():
|
||||
connection = await repo.upsert_connection(
|
||||
owner_user_id=str(_user().id),
|
||||
provider="telegram",
|
||||
external_account_id="42",
|
||||
status="connected",
|
||||
)
|
||||
return connection["id"]
|
||||
|
||||
connection_id = anyio.run(seed_connection)
|
||||
app = _make_app(_enabled_connections_config(), repo, _channels_config())
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.delete(f"/api/channels/connections/{connection_id}")
|
||||
|
||||
assert response.status_code == 204
|
||||
|
||||
async def get_connection_status():
|
||||
return (await repo.list_connections(str(_user().id)))[0]["status"]
|
||||
|
||||
assert anyio.run(get_connection_status) == "revoked"
|
||||
|
||||
anyio.run(repo.close)
|
||||
|
||||
|
||||
def test_disconnect_connection_is_current_user_scoped(tmp_path):
|
||||
import anyio
|
||||
|
||||
repo = anyio.run(_make_repo, tmp_path)
|
||||
|
||||
async def seed_connection():
|
||||
connection = await repo.upsert_connection(
|
||||
owner_user_id="other-user",
|
||||
provider="telegram",
|
||||
external_account_id="42",
|
||||
status="connected",
|
||||
)
|
||||
return connection["id"]
|
||||
|
||||
connection_id = anyio.run(seed_connection)
|
||||
app = _make_app(_enabled_connections_config(), repo, _channels_config())
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.delete(f"/api/channels/connections/{connection_id}")
|
||||
|
||||
assert response.status_code == 404
|
||||
|
||||
async def get_connection_status():
|
||||
return (await repo.list_connections("other-user"))[0]["status"]
|
||||
|
||||
assert anyio.run(get_connection_status) == "connected"
|
||||
|
||||
anyio.run(repo.close)
|
||||
+1256
-7
File diff suppressed because it is too large
Load Diff
@@ -747,7 +747,7 @@ class TestClientCheckpointerFallback:
|
||||
patch("deerflow.client.get_app_config", return_value=config_mock),
|
||||
patch("deerflow.client.create_agent", side_effect=fake_create_agent),
|
||||
patch("deerflow.client.create_chat_model", return_value=MagicMock()),
|
||||
patch("deerflow.client._build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.apply_prompt_template", return_value=""),
|
||||
patch("deerflow.client.DeerFlowClient._get_tools", return_value=[]),
|
||||
):
|
||||
@@ -781,7 +781,7 @@ class TestClientCheckpointerFallback:
|
||||
patch("deerflow.client.get_app_config", return_value=config_mock),
|
||||
patch("deerflow.client.create_agent", side_effect=fake_create_agent),
|
||||
patch("deerflow.client.create_chat_model", return_value=MagicMock()),
|
||||
patch("deerflow.client._build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.apply_prompt_template", return_value=""),
|
||||
patch("deerflow.client.DeerFlowClient._get_tools", return_value=[]),
|
||||
):
|
||||
|
||||
@@ -910,7 +910,7 @@ class TestEnsureAgent:
|
||||
with (
|
||||
patch("deerflow.client.create_chat_model"),
|
||||
patch("deerflow.client.create_agent", return_value=mock_agent),
|
||||
patch("deerflow.client._build_middlewares", return_value=[]) as mock_build_middlewares,
|
||||
patch("deerflow.client.build_middlewares", return_value=[]) as mock_build_middlewares,
|
||||
patch("deerflow.client.apply_prompt_template", return_value="prompt") as mock_apply_prompt,
|
||||
patch.object(client, "_get_tools", return_value=[]),
|
||||
patch("deerflow.runtime.checkpointer.get_checkpointer", return_value=MagicMock()),
|
||||
@@ -935,7 +935,7 @@ class TestEnsureAgent:
|
||||
with (
|
||||
patch("deerflow.client.create_chat_model"),
|
||||
patch("deerflow.client.create_agent", return_value=mock_agent) as mock_create_agent,
|
||||
patch("deerflow.client._build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.apply_prompt_template", return_value="prompt"),
|
||||
patch.object(client, "_get_tools", return_value=[]),
|
||||
patch("deerflow.runtime.checkpointer.get_checkpointer", return_value=mock_checkpointer),
|
||||
@@ -960,7 +960,7 @@ class TestEnsureAgent:
|
||||
with (
|
||||
patch("deerflow.client.create_chat_model"),
|
||||
patch("deerflow.client.create_agent", return_value=mock_agent) as mock_create_agent,
|
||||
patch("deerflow.client._build_middlewares", side_effect=fake_build_middlewares),
|
||||
patch("deerflow.client.build_middlewares", side_effect=fake_build_middlewares),
|
||||
patch("deerflow.client.apply_prompt_template", return_value="prompt"),
|
||||
patch.object(client, "_get_tools", return_value=[]),
|
||||
patch("deerflow.runtime.checkpointer.get_checkpointer", return_value=MagicMock()),
|
||||
@@ -979,7 +979,7 @@ class TestEnsureAgent:
|
||||
with (
|
||||
patch("deerflow.client.create_chat_model"),
|
||||
patch("deerflow.client.create_agent", return_value=mock_agent) as mock_create_agent,
|
||||
patch("deerflow.client._build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.apply_prompt_template", return_value="prompt"),
|
||||
patch.object(client, "_get_tools", return_value=[]),
|
||||
patch("deerflow.runtime.checkpointer.get_checkpointer", return_value=None),
|
||||
@@ -1957,7 +1957,7 @@ class TestScenarioAgentRecreation:
|
||||
with (
|
||||
patch("deerflow.client.create_chat_model"),
|
||||
patch("deerflow.client.create_agent", side_effect=fake_create_agent),
|
||||
patch("deerflow.client._build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.apply_prompt_template", return_value="prompt"),
|
||||
patch.object(client, "_get_tools", return_value=[]),
|
||||
patch("deerflow.runtime.checkpointer.get_checkpointer", return_value=MagicMock()),
|
||||
@@ -1985,7 +1985,7 @@ class TestScenarioAgentRecreation:
|
||||
with (
|
||||
patch("deerflow.client.create_chat_model"),
|
||||
patch("deerflow.client.create_agent", side_effect=fake_create_agent),
|
||||
patch("deerflow.client._build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.apply_prompt_template", return_value="prompt"),
|
||||
patch.object(client, "_get_tools", return_value=[]),
|
||||
patch("deerflow.runtime.checkpointer.get_checkpointer", return_value=MagicMock()),
|
||||
@@ -2010,7 +2010,7 @@ class TestScenarioAgentRecreation:
|
||||
with (
|
||||
patch("deerflow.client.create_chat_model"),
|
||||
patch("deerflow.client.create_agent", side_effect=fake_create_agent),
|
||||
patch("deerflow.client._build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.build_middlewares", return_value=[]),
|
||||
patch("deerflow.client.apply_prompt_template", return_value="prompt"),
|
||||
patch.object(client, "_get_tools", return_value=[]),
|
||||
patch("deerflow.runtime.checkpointer.get_checkpointer", return_value=MagicMock()),
|
||||
@@ -2472,6 +2472,7 @@ class TestGatewayConformance:
|
||||
mem_cfg.fact_confidence_threshold = 0.7
|
||||
mem_cfg.injection_enabled = True
|
||||
mem_cfg.max_injection_tokens = 2000
|
||||
mem_cfg.token_counting = "tiktoken"
|
||||
|
||||
with patch("deerflow.config.memory_config.get_memory_config", return_value=mem_cfg):
|
||||
result = client.get_memory_config()
|
||||
@@ -2479,6 +2480,7 @@ class TestGatewayConformance:
|
||||
parsed = MemoryConfigResponse(**result)
|
||||
assert parsed.enabled is True
|
||||
assert parsed.max_facts == 100
|
||||
assert parsed.token_counting == "tiktoken"
|
||||
|
||||
def test_get_memory_status(self, client):
|
||||
mem_cfg = MagicMock()
|
||||
@@ -2489,6 +2491,7 @@ class TestGatewayConformance:
|
||||
mem_cfg.fact_confidence_threshold = 0.7
|
||||
mem_cfg.injection_enabled = True
|
||||
mem_cfg.max_injection_tokens = 2000
|
||||
mem_cfg.token_counting = "tiktoken"
|
||||
|
||||
memory_data = {
|
||||
"version": "1.0",
|
||||
@@ -2514,6 +2517,7 @@ class TestGatewayConformance:
|
||||
|
||||
parsed = MemoryStatusResponse(**result)
|
||||
assert parsed.config.enabled is True
|
||||
assert parsed.config.token_counting == "tiktoken"
|
||||
assert parsed.data.version == "1.0"
|
||||
|
||||
|
||||
|
||||
@@ -144,14 +144,14 @@ def e2e_env(tmp_path, monkeypatch):
|
||||
# non-determinism and cost to E2E tests (title generation is already
|
||||
# disabled via TitleConfig above, but the middleware still participates
|
||||
# in the chain and can interfere with event ordering).
|
||||
from deerflow.agents.lead_agent.agent import _build_middlewares as _original_build_middlewares
|
||||
from deerflow.agents.lead_agent.agent import build_middlewares as _original_build_middlewares
|
||||
from deerflow.agents.middlewares.title_middleware import TitleMiddleware
|
||||
|
||||
def _sync_safe_build_middlewares(*args, **kwargs):
|
||||
mws = _original_build_middlewares(*args, **kwargs)
|
||||
return [m for m in mws if not isinstance(m, TitleMiddleware)]
|
||||
|
||||
monkeypatch.setattr("deerflow.client._build_middlewares", _sync_safe_build_middlewares)
|
||||
monkeypatch.setattr("deerflow.client.build_middlewares", _sync_safe_build_middlewares)
|
||||
|
||||
return {"tmp_path": tmp_path}
|
||||
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Regression test for the Docker Compose default Gateway worker count.
|
||||
|
||||
The Gateway holds run state (RunManager and the stream bridge) in process, so
|
||||
the default deployment must run a single Uvicorn worker. Running more than one
|
||||
worker without a shared cross-worker stream bridge breaks run cancellation, SSE
|
||||
reconnects, request de-duplication, and IM channels (nginx has no sticky
|
||||
sessions, so requests scatter across workers that each keep their own run
|
||||
state). This test pins the safe default so it cannot silently regress to a
|
||||
multi-worker default, while still allowing operators to override it once a
|
||||
shared stream bridge exists.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
COMPOSE_PATH = REPO_ROOT / "docker" / "docker-compose.yaml"
|
||||
|
||||
|
||||
def _gateway_command() -> str:
|
||||
"""Return the gateway service command as a single string."""
|
||||
compose = yaml.safe_load(COMPOSE_PATH.read_text(encoding="utf-8"))
|
||||
command = compose["services"]["gateway"]["command"]
|
||||
# ``command`` may load as a scalar string or a list depending on YAML style.
|
||||
if isinstance(command, list):
|
||||
command = " ".join(str(part) for part in command)
|
||||
return command
|
||||
|
||||
|
||||
def test_gateway_defaults_to_single_worker():
|
||||
"""With GATEWAY_WORKERS unset, the worker count must default to 1."""
|
||||
command = _gateway_command()
|
||||
match = re.search(r"GATEWAY_WORKERS:-(\d+)", command)
|
||||
assert match is not None, f"gateway command must set a GATEWAY_WORKERS default; got: {command}"
|
||||
assert match.group(1) == "1", f"default Gateway worker count must be 1, got {match.group(1)}"
|
||||
|
||||
|
||||
def test_gateway_worker_count_remains_overridable():
|
||||
"""The worker count must stay configurable, not hard-coded to 1."""
|
||||
command = _gateway_command()
|
||||
assert "${GATEWAY_WORKERS:-1}" in command, f"worker count must use ${{GATEWAY_WORKERS:-1}} so operators can override it; got: {command}"
|
||||
@@ -233,3 +233,15 @@ def test_non_auth_mutation_rejects_mismatched_double_submit_token():
|
||||
|
||||
assert response.status_code == 403
|
||||
assert response.json()["detail"] == "CSRF token mismatch."
|
||||
|
||||
|
||||
def test_channel_posts_require_double_submit_csrf():
|
||||
client = TestClient(_make_app(), base_url="https://deerflow.example")
|
||||
|
||||
response = client.post(
|
||||
"/api/channels/slack/connect",
|
||||
headers={"Origin": "https://deerflow.example"},
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert response.json()["detail"] == "CSRF token missing. Include X-CSRF-Token header."
|
||||
|
||||
@@ -203,6 +203,79 @@ class TestLoadAgentConfig:
|
||||
assert cfg.name == "legacy-agent"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 3b. resolve_agent_dir — memory-only directory fallback (#3390)
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestResolveAgentDirMemoryOnlyFallback:
|
||||
"""Regression tests for #3390.
|
||||
|
||||
When memory is enabled, the first conversation creates a user-isolated
|
||||
agent directory containing only ``memory.json`` (no ``config.yaml``).
|
||||
On the next turn ``resolve_agent_dir`` must fall through to the legacy
|
||||
shared layout instead of returning the incomplete user directory.
|
||||
"""
|
||||
|
||||
def test_user_dir_with_only_memory_falls_back_to_legacy(self, tmp_path):
|
||||
"""User dir has memory.json but no config.yaml → use legacy dir."""
|
||||
from deerflow.config.agents_config import resolve_agent_dir
|
||||
|
||||
# Legacy agent with full config
|
||||
legacy_dir = tmp_path / "agents" / "my-agent"
|
||||
legacy_dir.mkdir(parents=True)
|
||||
(legacy_dir / "config.yaml").write_text("name: my-agent\n", encoding="utf-8")
|
||||
(legacy_dir / "SOUL.md").write_text("legacy soul", encoding="utf-8")
|
||||
|
||||
# User dir created by memory write — no config.yaml
|
||||
user_dir = tmp_path / "users" / "u1" / "agents" / "my-agent"
|
||||
user_dir.mkdir(parents=True)
|
||||
(user_dir / "memory.json").write_text("{}", encoding="utf-8")
|
||||
|
||||
with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)), patch("deerflow.config.agents_config.get_effective_user_id", return_value="u1"):
|
||||
result = resolve_agent_dir("my-agent", user_id="u1")
|
||||
|
||||
assert result == legacy_dir
|
||||
|
||||
def test_user_dir_with_config_takes_priority(self, tmp_path):
|
||||
"""User dir with config.yaml should still win over legacy."""
|
||||
from deerflow.config.agents_config import resolve_agent_dir
|
||||
|
||||
# Legacy
|
||||
legacy_dir = tmp_path / "agents" / "my-agent"
|
||||
legacy_dir.mkdir(parents=True)
|
||||
(legacy_dir / "config.yaml").write_text("name: my-agent\n", encoding="utf-8")
|
||||
|
||||
# User dir with full config (migrated)
|
||||
user_dir = tmp_path / "users" / "u1" / "agents" / "my-agent"
|
||||
user_dir.mkdir(parents=True)
|
||||
(user_dir / "config.yaml").write_text("name: my-agent\nmodel: gpt-4\n", encoding="utf-8")
|
||||
(user_dir / "memory.json").write_text("{}", encoding="utf-8")
|
||||
|
||||
with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)), patch("deerflow.config.agents_config.get_effective_user_id", return_value="u1"):
|
||||
result = resolve_agent_dir("my-agent", user_id="u1")
|
||||
|
||||
assert result == user_dir
|
||||
|
||||
def test_load_config_falls_back_when_user_dir_is_memory_only(self, tmp_path):
|
||||
"""End-to-end: load_agent_config works when user dir only has memory.json."""
|
||||
config_dict = {"name": "my-agent", "description": "Legacy agent", "model": "deepseek-v3"}
|
||||
_write_agent(tmp_path, "my-agent", config_dict)
|
||||
|
||||
# Simulate memory write creating user dir without config
|
||||
user_dir = tmp_path / "users" / "u1" / "agents" / "my-agent"
|
||||
user_dir.mkdir(parents=True)
|
||||
(user_dir / "memory.json").write_text("{}", encoding="utf-8")
|
||||
|
||||
with patch("deerflow.config.agents_config.get_paths", return_value=_make_paths(tmp_path)), patch("deerflow.config.agents_config.get_effective_user_id", return_value="u1"):
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
|
||||
cfg = load_agent_config("my-agent", user_id="u1")
|
||||
|
||||
assert cfg.name == "my-agent"
|
||||
assert cfg.model == "deepseek-v3"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 4. load_agent_soul
|
||||
# ===========================================================================
|
||||
|
||||
@@ -44,7 +44,8 @@ def test_entrypoint_excludes_runtime_state_from_uvicorn_reload():
|
||||
content = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
|
||||
assert ': "${DEER_FLOW_HOME:=/app/backend/.deer-flow}"' in content
|
||||
assert 'mkdir -p "$DEER_FLOW_HOME" /app/backend/.deer-flow' in content
|
||||
# sandbox must be created too, not just .deer-flow (#3459 / #3454).
|
||||
assert 'mkdir -p "$DEER_FLOW_HOME" /app/backend/.deer-flow /app/backend/sandbox' in content
|
||||
assert "--reload-include='*.yaml .env'" not in content
|
||||
assert "--reload-include='*.yaml'" in content
|
||||
assert "--reload-include='.env'" in content
|
||||
|
||||
@@ -2,9 +2,13 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.channels.discord import DiscordChannel
|
||||
from app.channels.manager import CHANNEL_CAPABILITIES
|
||||
from app.channels.message_bus import MessageBus
|
||||
from app.channels.message_bus import InboundMessageType, MessageBus
|
||||
from app.channels.service import _CHANNEL_REGISTRY
|
||||
|
||||
|
||||
@@ -21,3 +25,64 @@ def test_discord_channel_init() -> None:
|
||||
channel = DiscordChannel(bus=bus, config={"bot_token": "token"})
|
||||
|
||||
assert channel.name == "discord"
|
||||
|
||||
|
||||
def _make_discord_message(text: str):
|
||||
return SimpleNamespace(
|
||||
id=111,
|
||||
content=text,
|
||||
author=SimpleNamespace(id=123, bot=False, display_name="alice"),
|
||||
guild=SimpleNamespace(id=321),
|
||||
channel=SimpleNamespace(id=456),
|
||||
add_reaction=lambda _emoji: None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_bot_mention_slash_skill_routes_as_chat() -> None:
|
||||
bus = MessageBus()
|
||||
channel = DiscordChannel(bus=bus, config={"bot_token": "token"})
|
||||
captured = []
|
||||
channel._running = True
|
||||
channel._client = SimpleNamespace(user=SimpleNamespace(id=999, mention="<@999>"))
|
||||
channel._discord_module = SimpleNamespace(Thread=type("FakeThread", (), {}))
|
||||
channel._publish = captured.append
|
||||
|
||||
async def noop(*_args, **_kwargs):
|
||||
return None
|
||||
|
||||
channel._start_typing = noop
|
||||
channel._add_reaction = noop
|
||||
|
||||
await channel._on_message(_make_discord_message("<@999> /data-analysis analyze uploads/foo.csv"))
|
||||
|
||||
assert len(captured) == 1
|
||||
inbound = captured[0]
|
||||
assert inbound.text == "/data-analysis analyze uploads/foo.csv"
|
||||
assert inbound.msg_type == InboundMessageType.CHAT
|
||||
assert inbound.topic_id == "456"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_bot_mention_known_command_routes_as_command() -> None:
|
||||
bus = MessageBus()
|
||||
channel = DiscordChannel(bus=bus, config={"bot_token": "token"})
|
||||
captured = []
|
||||
channel._running = True
|
||||
channel._client = SimpleNamespace(user=SimpleNamespace(id=999, mention="<@999>"))
|
||||
channel._discord_module = SimpleNamespace(Thread=type("FakeThread", (), {}))
|
||||
channel._publish = captured.append
|
||||
|
||||
async def noop(*_args, **_kwargs):
|
||||
return None
|
||||
|
||||
channel._start_typing = noop
|
||||
channel._add_reaction = noop
|
||||
|
||||
await channel._on_message(_make_discord_message("<@999> /help"))
|
||||
|
||||
assert len(captured) == 1
|
||||
inbound = captured[0]
|
||||
assert inbound.text == "/help"
|
||||
assert inbound.msg_type == InboundMessageType.COMMAND
|
||||
assert inbound.topic_id == "456"
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Discord connection routing tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.channels.discord import DiscordChannel
|
||||
from app.channels.message_bus import InboundMessage, MessageBus
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def repo(tmp_path):
|
||||
from deerflow.persistence.channel_connections import ChannelConnectionRepository, ChannelCredentialCipher
|
||||
from deerflow.persistence.engine import close_engine, get_session_factory, init_engine
|
||||
|
||||
await init_engine("sqlite", url=f"sqlite+aiosqlite:///{tmp_path / 'discord.db'}", sqlite_dir=str(tmp_path))
|
||||
try:
|
||||
yield ChannelConnectionRepository(
|
||||
get_session_factory(),
|
||||
cipher=ChannelCredentialCipher.from_key("discord-secret"),
|
||||
)
|
||||
finally:
|
||||
await close_engine()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_discord_inbound_attaches_owner_identity_from_user_level_connection(repo):
|
||||
connection = await repo.upsert_connection(
|
||||
owner_user_id="alice",
|
||||
provider="discord",
|
||||
external_account_id="987",
|
||||
external_account_name="Alice",
|
||||
status="connected",
|
||||
)
|
||||
channel = DiscordChannel(
|
||||
bus=MessageBus(),
|
||||
config={"bot_token": "discord-bot", "connection_repo": repo},
|
||||
)
|
||||
inbound = InboundMessage(
|
||||
channel_name="discord",
|
||||
chat_id="C123",
|
||||
user_id="987",
|
||||
text="hello",
|
||||
)
|
||||
|
||||
attached = await channel._attach_connection_identity(inbound, guild_id="G123")
|
||||
|
||||
assert attached.connection_id == connection["id"]
|
||||
assert attached.owner_user_id == "alice"
|
||||
assert attached.workspace_id is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_discord_connect_command_binds_gateway_identity(repo):
|
||||
state = "discord-bind-code"
|
||||
await repo.create_oauth_state(
|
||||
owner_user_id="deerflow-user-1",
|
||||
provider="discord",
|
||||
state=state,
|
||||
expires_at=datetime.now(UTC) + timedelta(minutes=5),
|
||||
)
|
||||
channel = DiscordChannel(
|
||||
bus=MessageBus(),
|
||||
config={"bot_token": "discord-bot", "connection_repo": repo},
|
||||
)
|
||||
message = MagicMock()
|
||||
message.author.id = 987
|
||||
message.author.display_name = "Alice"
|
||||
message.guild.id = 123
|
||||
message.guild.name = "Deer Guild"
|
||||
message.channel.id = 456
|
||||
message.channel.send = AsyncMock()
|
||||
|
||||
handled = await channel._bind_connection_from_connect_code(message, state)
|
||||
|
||||
connections = await repo.list_connections("deerflow-user-1")
|
||||
assert handled is True
|
||||
assert len(connections) == 1
|
||||
assert connections[0]["provider"] == "discord"
|
||||
assert connections[0]["external_account_id"] == "987"
|
||||
assert connections[0]["external_account_name"] == "Alice"
|
||||
assert connections[0]["workspace_id"] == "123"
|
||||
assert connections[0]["workspace_name"] == "Deer Guild"
|
||||
assert connections[0]["metadata"]["channel_id"] == "456"
|
||||
message.channel.send.assert_awaited_once()
|
||||
@@ -73,6 +73,31 @@ def test_feishu_on_message_plain_text():
|
||||
assert mock_make_inbound.call_args[1]["text"] == "Hello world"
|
||||
|
||||
|
||||
def test_feishu_is_not_running_when_ws_thread_exits():
|
||||
bus = MessageBus()
|
||||
channel = FeishuChannel(bus, {"app_id": "test", "app_secret": "test"})
|
||||
channel._running = True
|
||||
channel._thread = MagicMock()
|
||||
channel._thread.is_alive.return_value = False
|
||||
|
||||
assert channel.is_running is False
|
||||
|
||||
|
||||
def test_feishu_event_handler_ignores_non_content_message_events():
|
||||
import lark_oapi as lark
|
||||
|
||||
bus = MessageBus()
|
||||
channel = FeishuChannel(bus, {"app_id": "test", "app_secret": "test"})
|
||||
|
||||
event_handler = channel._build_event_handler(lark)
|
||||
|
||||
assert "p2.im.message.receive_v1" in event_handler._processorMap
|
||||
assert "p2.im.message.message_read_v1" in event_handler._processorMap
|
||||
assert "p2.im.message.reaction.created_v1" in event_handler._processorMap
|
||||
assert "p2.im.message.reaction.deleted_v1" in event_handler._processorMap
|
||||
assert "p2.im.message.recalled_v1" in event_handler._processorMap
|
||||
|
||||
|
||||
def test_feishu_on_message_rich_text():
|
||||
bus = MessageBus()
|
||||
config = {"app_id": "test", "app_secret": "test"}
|
||||
|
||||
@@ -49,7 +49,9 @@ def test_local_dev_gateway_reload_excludes_runtime_state_with_absolute_dirs():
|
||||
assert 'export DEER_FLOW_PROJECT_ROOT="$REPO_ROOT"' in serve_sh
|
||||
assert 'BACKEND_RUNTIME_HOME="$REPO_ROOT/backend/.deer-flow"' in serve_sh
|
||||
assert 'export DEER_FLOW_HOME="$BACKEND_RUNTIME_HOME"' in serve_sh
|
||||
assert 'mkdir -p "$DEER_FLOW_HOME" "$BACKEND_RUNTIME_HOME"' in serve_sh
|
||||
# Every absolute reload-exclude must be pre-created, including backend/sandbox
|
||||
# (#3459 / #3454) — see test_uvicorn_reload_exclude.py for the mechanism.
|
||||
assert 'mkdir -p "$DEER_FLOW_HOME" "$BACKEND_RUNTIME_HOME" "$REPO_ROOT/backend/sandbox"' in serve_sh
|
||||
assert "--reload-exclude='$DEER_FLOW_HOME'" in serve_sh
|
||||
assert "--reload-exclude='$BACKEND_RUNTIME_HOME'" in serve_sh
|
||||
assert "--reload-exclude='sandbox/'" not in serve_sh
|
||||
|
||||
@@ -474,6 +474,83 @@ def test_inject_authenticated_user_context_skips_internal_role():
|
||||
assert config["context"]["user_id"] == "channel-user-7"
|
||||
|
||||
|
||||
def test_start_run_uses_internal_owner_header_for_persistence():
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.store.memory import InMemoryStore
|
||||
|
||||
from app.gateway.internal_auth import INTERNAL_OWNER_USER_ID_HEADER_NAME, INTERNAL_SYSTEM_ROLE
|
||||
from app.gateway.services import start_run
|
||||
from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore
|
||||
from deerflow.runtime import RunManager
|
||||
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
async def _scenario():
|
||||
run_store = MemoryRunStore()
|
||||
thread_store = MemoryThreadMetaStore(InMemoryStore())
|
||||
await thread_store.create("channel-thread", user_id="default", metadata={"legacy": True})
|
||||
run_manager = RunManager(store=run_store)
|
||||
state = SimpleNamespace(
|
||||
stream_bridge=SimpleNamespace(),
|
||||
run_manager=run_manager,
|
||||
checkpointer=InMemorySaver(),
|
||||
store=InMemoryStore(),
|
||||
run_event_store=SimpleNamespace(),
|
||||
run_events_config=None,
|
||||
thread_store=thread_store,
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
headers={INTERNAL_OWNER_USER_ID_HEADER_NAME: "owner-1"},
|
||||
state=SimpleNamespace(user=SimpleNamespace(id="default", system_role=INTERNAL_SYSTEM_ROLE)),
|
||||
app=SimpleNamespace(state=state),
|
||||
)
|
||||
body = SimpleNamespace(
|
||||
assistant_id="lead_agent",
|
||||
input={"messages": [{"role": "human", "content": "hi"}]},
|
||||
metadata={},
|
||||
config=None,
|
||||
context=None,
|
||||
on_disconnect="cancel",
|
||||
multitask_strategy="reject",
|
||||
stream_mode=None,
|
||||
stream_subgraphs=False,
|
||||
interrupt_before=None,
|
||||
interrupt_after=None,
|
||||
)
|
||||
task_context: dict[str, str] = {}
|
||||
|
||||
async def fake_run_agent(*args, **kwargs):
|
||||
task_context["user_id"] = get_effective_user_id()
|
||||
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.services.run_agent", side_effect=fake_run_agent),
|
||||
):
|
||||
record = await start_run(body, "channel-thread", request)
|
||||
await record.task
|
||||
|
||||
owner_run = await run_store.get(record.run_id, user_id="owner-1")
|
||||
default_run = await run_store.get(record.run_id, user_id="default")
|
||||
owner_thread = await thread_store.get("channel-thread", user_id="owner-1")
|
||||
default_thread = await thread_store.get("channel-thread", user_id="default")
|
||||
return owner_run, default_run, owner_thread, default_thread, task_context
|
||||
|
||||
owner_run, default_run, owner_thread, default_thread, task_context = asyncio.run(_scenario())
|
||||
|
||||
assert owner_run is not None
|
||||
assert owner_run["user_id"] == "owner-1"
|
||||
assert default_run is None
|
||||
assert owner_thread is not None
|
||||
assert owner_thread["user_id"] == "owner-1"
|
||||
assert owner_thread["metadata"] == {"legacy": True}
|
||||
assert default_thread is None
|
||||
assert task_context["user_id"] == "owner-1"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_run_config — context / configurable precedence (LangGraph >= 0.6.0)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -33,3 +33,18 @@ def test_internal_auth_generates_process_local_fallback(monkeypatch):
|
||||
assert reloaded.is_valid_internal_auth_token(token) is True
|
||||
finally:
|
||||
importlib.reload(reloaded)
|
||||
|
||||
|
||||
def test_internal_auth_headers_can_carry_owner_user_id(monkeypatch):
|
||||
import app.gateway.internal_auth as internal_auth
|
||||
|
||||
monkeypatch.setenv("DEER_FLOW_INTERNAL_AUTH_TOKEN", "shared-token")
|
||||
reloaded = importlib.reload(internal_auth)
|
||||
try:
|
||||
headers = reloaded.create_internal_auth_headers(owner_user_id="owner-1")
|
||||
|
||||
assert headers[reloaded.INTERNAL_AUTH_HEADER_NAME] == "shared-token"
|
||||
assert headers[reloaded.INTERNAL_OWNER_USER_ID_HEADER_NAME] == "owner-1"
|
||||
finally:
|
||||
monkeypatch.delenv("DEER_FLOW_INTERNAL_AUTH_TOKEN", raising=False)
|
||||
importlib.reload(reloaded)
|
||||
|
||||
@@ -8,7 +8,12 @@ import pytest
|
||||
|
||||
import deerflow.community.jina_ai.jina_client as jina_client_module
|
||||
from deerflow.community.jina_ai.jina_client import JinaClient
|
||||
from deerflow.community.jina_ai.tools import web_fetch_tool
|
||||
from deerflow.community.jina_ai.tools import (
|
||||
_coerce_bool,
|
||||
_coerce_proxy,
|
||||
_coerce_timeout,
|
||||
web_fetch_tool,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -117,6 +122,59 @@ async def test_crawl_passes_headers(jina_client, monkeypatch):
|
||||
assert captured_headers["X-Timeout"] == "30"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_crawl_passes_proxy_to_httpx_client(jina_client, monkeypatch):
|
||||
"""Explicit proxy config should be passed to httpx.AsyncClient."""
|
||||
captured_client_kwargs = {}
|
||||
|
||||
class MockAsyncClient:
|
||||
def __init__(self, **kwargs):
|
||||
captured_client_kwargs.update(kwargs)
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
async def post(self, url, **kwargs):
|
||||
return httpx.Response(200, text="ok", request=httpx.Request("POST", url))
|
||||
|
||||
monkeypatch.setattr(httpx, "AsyncClient", MockAsyncClient)
|
||||
|
||||
result = await jina_client.crawl("https://example.com", proxy="http://127.0.0.1:7890")
|
||||
|
||||
assert result == "ok"
|
||||
assert captured_client_kwargs["proxy"] == "http://127.0.0.1:7890"
|
||||
assert captured_client_kwargs["trust_env"] is True
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_crawl_can_disable_trust_env(jina_client, monkeypatch):
|
||||
"""Callers can disable environment proxy lookup for deterministic networking."""
|
||||
captured_client_kwargs = {}
|
||||
|
||||
class MockAsyncClient:
|
||||
def __init__(self, **kwargs):
|
||||
captured_client_kwargs.update(kwargs)
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
async def post(self, url, **kwargs):
|
||||
return httpx.Response(200, text="ok", request=httpx.Request("POST", url))
|
||||
|
||||
monkeypatch.setattr(httpx, "AsyncClient", MockAsyncClient)
|
||||
|
||||
result = await jina_client.crawl("https://example.com", trust_env=False)
|
||||
|
||||
assert result == "ok"
|
||||
assert captured_client_kwargs == {"trust_env": False}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_crawl_includes_api_key_when_set(jina_client, monkeypatch):
|
||||
"""Test that Authorization header is set when JINA_API_KEY is available."""
|
||||
@@ -199,6 +257,60 @@ async def test_web_fetch_tool_returns_markdown_on_success(monkeypatch):
|
||||
assert not result.startswith("Error:")
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_web_fetch_tool_forwards_proxy_and_trust_env(monkeypatch):
|
||||
"""web_fetch tool config should be forwarded to JinaClient.crawl."""
|
||||
captured_crawl_kwargs = {}
|
||||
|
||||
async def mock_crawl(self, url, **kwargs):
|
||||
captured_crawl_kwargs.update(kwargs)
|
||||
return "<html><body><p>Hello world</p></body></html>"
|
||||
|
||||
mock_config = MagicMock()
|
||||
mock_tool_config = MagicMock()
|
||||
mock_tool_config.model_extra = {
|
||||
"timeout": "20",
|
||||
"proxy": "http://host.docker.internal:7890",
|
||||
"trust_env": "false",
|
||||
}
|
||||
mock_config.get_tool_config.return_value = mock_tool_config
|
||||
monkeypatch.setattr("deerflow.community.jina_ai.tools.get_app_config", lambda: mock_config)
|
||||
monkeypatch.setattr(JinaClient, "crawl", mock_crawl)
|
||||
|
||||
result = await web_fetch_tool.ainvoke("https://example.com")
|
||||
|
||||
assert "Hello world" in result
|
||||
assert captured_crawl_kwargs == {
|
||||
"return_format": "html",
|
||||
"timeout": 20,
|
||||
"proxy": "http://host.docker.internal:7890",
|
||||
"trust_env": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_web_fetch_tool_ignores_empty_proxy(monkeypatch):
|
||||
"""Empty proxy values from unresolved env vars should not be passed to httpx."""
|
||||
captured_crawl_kwargs = {}
|
||||
|
||||
async def mock_crawl(self, url, **kwargs):
|
||||
captured_crawl_kwargs.update(kwargs)
|
||||
return "<html><body><p>Hello world</p></body></html>"
|
||||
|
||||
mock_config = MagicMock()
|
||||
mock_tool_config = MagicMock()
|
||||
mock_tool_config.model_extra = {"proxy": " ", "trust_env": True}
|
||||
mock_config.get_tool_config.return_value = mock_tool_config
|
||||
monkeypatch.setattr("deerflow.community.jina_ai.tools.get_app_config", lambda: mock_config)
|
||||
monkeypatch.setattr(JinaClient, "crawl", mock_crawl)
|
||||
|
||||
result = await web_fetch_tool.ainvoke("https://example.com")
|
||||
|
||||
assert "Hello world" in result
|
||||
assert captured_crawl_kwargs["proxy"] is None
|
||||
assert captured_crawl_kwargs["trust_env"] is True
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_web_fetch_tool_offloads_extraction_to_thread(monkeypatch):
|
||||
"""Test that readability extraction is offloaded via asyncio.to_thread to avoid blocking the event loop."""
|
||||
@@ -224,3 +336,60 @@ async def test_web_fetch_tool_offloads_extraction_to_thread(monkeypatch):
|
||||
result = await web_fetch_tool.ainvoke("https://example.com")
|
||||
assert to_thread_called, "extract_article must be called via asyncio.to_thread to avoid blocking the event loop"
|
||||
assert "threaded" in result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "default", "expected"),
|
||||
[
|
||||
(True, False, True),
|
||||
(False, True, False),
|
||||
("true", False, True),
|
||||
("YES", False, True),
|
||||
(" on ", False, True),
|
||||
("1", False, True),
|
||||
("false", True, False),
|
||||
("No", True, False),
|
||||
("off", True, False),
|
||||
("0", True, False),
|
||||
("maybe", True, True),
|
||||
("maybe", False, False),
|
||||
(None, True, True),
|
||||
(123, False, False),
|
||||
],
|
||||
)
|
||||
def test_coerce_bool(value, default, expected):
|
||||
"""_coerce_bool normalizes booleans, known strings, and falls back to the default."""
|
||||
assert _coerce_bool(value, default) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "default", "expected"),
|
||||
[
|
||||
(30, 10, 30),
|
||||
("45", 10, 45),
|
||||
("not-a-number", 10, 10),
|
||||
(True, 10, 10),
|
||||
(False, 10, 10),
|
||||
(None, 10, 10),
|
||||
(1.5, 10, 10),
|
||||
],
|
||||
)
|
||||
def test_coerce_timeout(value, default, expected):
|
||||
"""_coerce_timeout accepts ints and numeric strings, rejecting bools and junk."""
|
||||
assert _coerce_timeout(value, default) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
("http://127.0.0.1:7890", "http://127.0.0.1:7890"),
|
||||
(" http://proxy:8080 ", "http://proxy:8080"),
|
||||
("", None),
|
||||
(" ", None),
|
||||
(None, None),
|
||||
(123, None),
|
||||
],
|
||||
)
|
||||
def test_coerce_proxy(value, expected):
|
||||
"""_coerce_proxy trims strings and treats empty/non-string values as None."""
|
||||
assert _coerce_proxy(value) == expected
|
||||
|
||||
@@ -21,6 +21,7 @@ from langgraph_sdk import Auth
|
||||
from app.gateway.auth.config import AuthConfig, set_auth_config
|
||||
from app.gateway.auth.jwt import create_access_token, decode_token
|
||||
from app.gateway.auth.models import User
|
||||
from app.gateway.auth_disabled import AUTH_DISABLED_USER_ID
|
||||
from app.gateway.langgraph_auth import add_owner_filter, authenticate
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────
|
||||
@@ -59,6 +60,14 @@ def test_no_cookie_raises_401():
|
||||
assert "Not authenticated" in str(exc.value.detail)
|
||||
|
||||
|
||||
def test_auth_disabled_skips_csrf_and_authenticates_e2e_user(monkeypatch):
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
|
||||
identity = asyncio.run(authenticate(_req(method="POST")))
|
||||
|
||||
assert identity == AUTH_DISABLED_USER_ID
|
||||
|
||||
|
||||
def test_invalid_jwt_raises_401():
|
||||
with pytest.raises(Auth.exceptions.HTTPException) as exc:
|
||||
asyncio.run(authenticate(_req({"access_token": "garbage"})))
|
||||
|
||||
@@ -56,7 +56,7 @@ def test_make_lead_agent_attaches_tracing_callbacks_at_graph_root(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(tools_module, "get_available_tools", lambda **kwargs: [])
|
||||
monkeypatch.setattr(lead_agent_module, "_build_middlewares", lambda config, model_name, agent_name=None, **kwargs: [])
|
||||
monkeypatch.setattr(lead_agent_module, "build_middlewares", lambda config, model_name, agent_name=None, **kwargs: [])
|
||||
|
||||
sentinel_handler = object()
|
||||
monkeypatch.setattr(lead_agent_module, "build_tracing_callbacks", lambda: [sentinel_handler])
|
||||
@@ -94,7 +94,7 @@ def test_internal_make_lead_agent_uses_explicit_app_config(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(lead_agent_module, "get_app_config", _raise_get_app_config)
|
||||
monkeypatch.setattr(tools_module, "get_available_tools", lambda **kwargs: [])
|
||||
monkeypatch.setattr(lead_agent_module, "_build_middlewares", lambda config, model_name, agent_name=None, **kwargs: [])
|
||||
monkeypatch.setattr(lead_agent_module, "build_middlewares", lambda config, model_name, agent_name=None, **kwargs: [])
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
@@ -128,7 +128,7 @@ def test_make_lead_agent_uses_runtime_app_config_from_context_without_global_rea
|
||||
|
||||
monkeypatch.setattr(lead_agent_module, "get_app_config", _raise_get_app_config)
|
||||
monkeypatch.setattr(tools_module, "get_available_tools", lambda **kwargs: [])
|
||||
monkeypatch.setattr(lead_agent_module, "_build_middlewares", lambda config, model_name, agent_name=None, **kwargs: [])
|
||||
monkeypatch.setattr(lead_agent_module, "build_middlewares", lambda config, model_name, agent_name=None, **kwargs: [])
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
@@ -207,7 +207,7 @@ def test_make_lead_agent_disables_thinking_when_model_does_not_support_it(monkey
|
||||
|
||||
monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(tools_module, "get_available_tools", lambda **kwargs: [])
|
||||
monkeypatch.setattr(lead_agent_module, "_build_middlewares", lambda config, model_name, agent_name=None, **kwargs: [])
|
||||
monkeypatch.setattr(lead_agent_module, "build_middlewares", lambda config, model_name, agent_name=None, **kwargs: [])
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
@@ -251,7 +251,7 @@ def test_make_lead_agent_reads_runtime_options_from_context(monkeypatch):
|
||||
get_available_tools = MagicMock(return_value=[])
|
||||
monkeypatch.setattr(lead_agent_module, "get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(tools_module, "get_available_tools", get_available_tools)
|
||||
monkeypatch.setattr(lead_agent_module, "_build_middlewares", lambda config, model_name, agent_name=None, **kwargs: [])
|
||||
monkeypatch.setattr(lead_agent_module, "build_middlewares", lambda config, model_name, agent_name=None, **kwargs: [])
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
@@ -328,7 +328,7 @@ def test_build_middlewares_uses_resolved_model_name_for_vision(monkeypatch):
|
||||
monkeypatch.setattr(lead_agent_module, "_create_summarization_middleware", lambda **kwargs: None)
|
||||
monkeypatch.setattr(lead_agent_module, "_create_todo_list_middleware", lambda is_plan_mode: None)
|
||||
|
||||
middlewares = lead_agent_module._build_middlewares(
|
||||
middlewares = lead_agent_module.build_middlewares(
|
||||
{"configurable": {"model_name": "stale-model", "is_plan_mode": False, "subagent_enabled": False}},
|
||||
model_name="vision-model",
|
||||
custom_middlewares=[MagicMock()],
|
||||
@@ -374,7 +374,7 @@ def test_build_middlewares_passes_explicit_app_config_to_shared_factory(monkeypa
|
||||
lambda agent_name=None, *, memory_config: captured.setdefault("memory_config", memory_config) or "memory-middleware",
|
||||
)
|
||||
|
||||
middlewares = lead_agent_module._build_middlewares(
|
||||
middlewares = lead_agent_module.build_middlewares(
|
||||
{"configurable": {"is_plan_mode": False, "subagent_enabled": False}},
|
||||
model_name="safe-model",
|
||||
app_config=app_config,
|
||||
@@ -407,7 +407,7 @@ def test_build_middlewares_uses_loop_detection_config(monkeypatch):
|
||||
monkeypatch.setattr(lead_agent_module, "_create_summarization_middleware", lambda *, app_config=None: None)
|
||||
monkeypatch.setattr(lead_agent_module, "_create_todo_list_middleware", lambda is_plan_mode: None)
|
||||
|
||||
middlewares = lead_agent_module._build_middlewares(
|
||||
middlewares = lead_agent_module.build_middlewares(
|
||||
{"configurable": {"is_plan_mode": False, "subagent_enabled": False}},
|
||||
model_name="safe-model",
|
||||
app_config=app_config,
|
||||
@@ -433,7 +433,7 @@ def test_build_middlewares_omits_loop_detection_when_disabled(monkeypatch):
|
||||
monkeypatch.setattr(lead_agent_module, "_create_summarization_middleware", lambda *, app_config=None: None)
|
||||
monkeypatch.setattr(lead_agent_module, "_create_todo_list_middleware", lambda is_plan_mode: None)
|
||||
|
||||
middlewares = lead_agent_module._build_middlewares(
|
||||
middlewares = lead_agent_module.build_middlewares(
|
||||
{"configurable": {"is_plan_mode": False, "subagent_enabled": False}},
|
||||
model_name="safe-model",
|
||||
app_config=app_config,
|
||||
|
||||
@@ -192,7 +192,7 @@ def test_build_acp_section_uses_explicit_app_config_without_global_config(monkey
|
||||
|
||||
def test_get_memory_context_uses_explicit_app_config_without_global_config(monkeypatch):
|
||||
explicit_config = SimpleNamespace(
|
||||
memory=SimpleNamespace(enabled=True, injection_enabled=True, max_injection_tokens=1234),
|
||||
memory=SimpleNamespace(enabled=True, injection_enabled=True, max_injection_tokens=1234, token_counting="tiktoken"),
|
||||
)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
@@ -204,9 +204,10 @@ def test_get_memory_context_uses_explicit_app_config_without_global_config(monke
|
||||
captured["user_id"] = user_id
|
||||
return {"facts": []}
|
||||
|
||||
def fake_format_memory_for_injection(memory_data, *, max_tokens):
|
||||
def fake_format_memory_for_injection(memory_data, *, max_tokens, use_tiktoken=True):
|
||||
captured["memory_data"] = memory_data
|
||||
captured["max_tokens"] = max_tokens
|
||||
captured["use_tiktoken"] = use_tiktoken
|
||||
return "remember this"
|
||||
|
||||
monkeypatch.setattr("deerflow.config.memory_config.get_memory_config", fail_get_memory_config)
|
||||
@@ -223,6 +224,7 @@ def test_get_memory_context_uses_explicit_app_config_without_global_config(monke
|
||||
"user_id": "user-1",
|
||||
"memory_data": {"facts": []},
|
||||
"max_tokens": 1234,
|
||||
"use_tiktoken": True,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user