diff --git a/.gitignore b/.gitignore index 6dba53b46..d4649770f 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ config.yaml.bak /frontend/playwright-report/ .gstack/ .worktrees + +# Monocle agent-observability trace output (local file exporter) +.monocle/ diff --git a/README.md b/README.md index 3ea9241a5..c98e23f1d 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,8 @@ DeerFlow has newly integrated the intelligent search and crawling toolset indepe - [IM Channels](#im-channels) - [LangSmith Tracing](#langsmith-tracing) - [Langfuse Tracing](#langfuse-tracing) - - [Using Both Providers](#using-both-providers) + - [Monocle Tracing](#monocle-tracing) + - [Using Multiple Providers](#using-multiple-providers) - [From Deep Research to Super Agent Harness](#from-deep-research-to-super-agent-harness) - [Core Features](#core-features) - [Skills \& Tools](#skills--tools) @@ -593,11 +594,25 @@ If you are using a self-hosted Langfuse instance, set `LANGFUSE_BASE_URL` to you These are injected into `RunnableConfig.metadata` at the graph invocation root for both the gateway path (`runtime/runs/worker.py::run_agent`) and the embedded path (`client.py::DeerFlowClient.stream`), so any LangChain-compatible callback can read them. Set `DEER_FLOW_ENV` (or `ENVIRONMENT`) to tag traces by deployment environment. -#### Using Both Providers +#### Monocle Tracing -If both LangSmith and Langfuse are enabled, DeerFlow attaches both tracing callbacks and reports the same model activity to both systems. +DeerFlow also supports [Monocle](https://github.com/monocle2ai/monocle), an OpenTelemetry-based tracer for agentic applications. It records each run end-to-end: LLM calls, agent steps, and tool and MCP invocations, with their inputs, outputs, timings, and token counts. -If a provider is explicitly enabled but missing required credentials, or if its callback fails to initialize, DeerFlow fails fast when tracing is initialized during model creation and the error message names the provider that caused the failure. +Add the following to your `.env` file: + +```bash +MONOCLE_TRACING=true +MONOCLE_EXPORTERS=file # file, console, okahu, s3, blob, gcs (default: file) +OKAHU_API_KEY=okh_xxxxxxxx # required only for the `okahu` exporter +``` + +Each run writes one trace file to `.monocle/`; open it in the [Monocle VS Code extension](https://marketplace.visualstudio.com/items?itemName=OkahuAI.monocle-apptrace) to inspect the span timeline and token counts. Connect to [Okahu](https://www.okahu.ai), an agent-observability platform, to analyze traces across runs and run trace-based and agentic evaluations (via the `okahu` exporter). + +Traces capture span inputs and outputs verbatim — prompts, tool arguments, and model responses — plus token usage and timings. The `file` exporter keeps them on local disk and never rotates or cleans them up, so prune `.monocle/` periodically; the remote exporters (`okahu`, `s3`, `blob`, `gcs`) send that same data off-box, so enable only destinations you trust. Monocle is initialized once at Gateway startup: a configuration error (unknown exporter, missing `OKAHU_API_KEY`) is logged there and tracing stays off until the Gateway restarts. + +#### Using Multiple Providers + +LangSmith and Langfuse attach as LangChain callbacks, so you can enable both and DeerFlow reports each run to both. If an enabled provider is missing required credentials or fails to initialize, DeerFlow fails fast and names it. Monocle uses a global OpenTelemetry provider rather than a callback; Langfuse shares that provider, so all three can run together. Because both span processors sit on the same shared provider, Monocle's exporters also see Langfuse's spans when both are enabled. For Docker deployments, tracing is disabled by default. Set `LANGSMITH_TRACING=true` and `LANGSMITH_API_KEY` in your `.env` to enable it. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index b078d2b5b..818376d27 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -711,6 +711,12 @@ LangSmith and Langfuse are both supported. The wiring lives in two layers: Returns `{}` when Langfuse is not in the enabled providers — LangSmith-only deployments are unaffected. Set `DEER_FLOW_ENV` (or `ENVIRONMENT`) to tag traces by deployment environment. Tests live in `tests/test_tracing_factory.py`, `tests/test_tracing_metadata.py`, `tests/test_worker_langfuse_metadata.py`, `tests/test_client_langfuse_metadata.py`, and `tests/test_subagent_executor.py::TestSubagentTracingWiring`. +**Monocle telemetry** is a third provider, structurally unlike LangSmith/Langfuse. It is **not** a LangChain callback: `tracing/monocle.py::setup_monocle_tracing_if_enabled()` calls `monocle_apptrace.setup_monocle_telemetry()` once, which installs a **process-global OTel `TracerProvider`**, patches span serialization, and auto-instruments the openai/langchain/langgraph clients. Because that is a one-time, process-global side effect (not a per-run callback), it is initialized from the **Gateway lifespan** (`app/gateway/app.py`) — never from `build_tracing_callbacks()` — and it is **off by default**. The setup call was deliberately moved out of `agents/__init__.py`, so `import deerflow.agents` must never start tracing (pinned by `tests/test_monocle_tracing.py::test_no_import_time_setup`). The Gateway lifespan is the **sole call site** (pinned by `test_gateway_lifespan_initializes_monocle`), so unlike LangSmith/Langfuse — which attach at the graph roots and cover every path — the embedded `DeerFlowClient` and the TUI are not instrumented; embedded users who want Monocle traces call `setup_monocle_tracing_if_enabled()` themselves before running the agent. + +Unlike the Langfuse metadata above, DeerFlow injects **no** per-run fields into Monocle traces — the only attribute it sets is `workflow_name="deer-flow"`; every span attribute (`span.type`, `entity.*`, token usage, span inputs/outputs, `scope.agentic.session`) is produced by Monocle's own metamodel and auto-instrumentation, so there is no DeerFlow trace-attribute layer to maintain here. + +Config is env-driven like the others — `MonocleTracingConfig`, built in `get_tracing_config()` and gated by `is_monocle_tracing_enabled()`. `MONOCLE_TRACING` enables it; `MONOCLE_EXPORTERS` selects exporters (default `file` → trace JSON in `.monocle/`; also `console`, `okahu`, `s3`, `blob`, `gcs`, where `okahu` requires `OKAHU_API_KEY`). `setup_monocle_tracing_if_enabled()` stays a thin wrapper on purpose: `monocle_apptrace` already guards duplicate setup (`instrumentor.py::check_duplicate_setup`) and never force-overrides an existing global provider, so the wrapper only gates on config. Coexistence with Langfuse (v4, also OTel-based) is **verified**: whichever library initializes second reuses the existing global `TracerProvider` and attaches its own span processor, so neither side loses spans (pinned by `test_coexists_with_langfuse`). Both processors see all spans, so Monocle's exporters also capture Langfuse's spans when both are enabled. (LangSmith is a plain callback and coexists trivially.) Tests: `tests/test_monocle_tracing.py`. + ### Config Schema **`config.yaml`** key sections: diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 42ccaca33..21ba2ac8d 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -37,6 +37,7 @@ from app.gateway.routers import ( from app.gateway.trace_middleware import TraceMiddleware, resolve_trace_enabled from deerflow.config import app_config as deerflow_app_config from deerflow.logging_config import DEFAULT_LOG_DATE_FORMAT, DEFAULT_LOG_FORMAT, configure_logging +from deerflow.tracing.monocle import setup_monocle_tracing_if_enabled from deerflow.uploads.manager import cleanup_stale_upload_staging_files AppConfig = deerflow_app_config.AppConfig @@ -189,6 +190,16 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: config = get_gateway_config() logger.info(f"Starting API Gateway on {config.host}:{config.port}") + # Agent observability (Monocle). Off by default; enabled with + # MONOCLE_TRACING. Initialized here at startup — not at import time — so a + # plain `import deerflow.agents` never installs a process-global tracer. + # Unlike LangSmith/Langfuse, whose validation failures abort the agent run, + # a bad Monocle config only logs: the Gateway keeps serving without tracing. + try: + setup_monocle_tracing_if_enabled() + except Exception: # observability must never break startup + logger.exception("Monocle tracing setup failed; continuing without it") + # Pre-warm tiktoken encoding cache so the first memory-injection request # never blocks on the BPE data download (which hits an OpenAI/Azure URL # that may be unreachable in restricted networks — see issue #3402). diff --git a/backend/packages/harness/deerflow/config/__init__.py b/backend/packages/harness/deerflow/config/__init__.py index bf74dc5e8..76751936b 100644 --- a/backend/packages/harness/deerflow/config/__init__.py +++ b/backend/packages/harness/deerflow/config/__init__.py @@ -9,6 +9,7 @@ from .tracing_config import ( get_enabled_tracing_providers, get_explicitly_enabled_tracing_providers, get_tracing_config, + is_monocle_tracing_enabled, is_tracing_enabled, validate_enabled_tracing_providers, ) @@ -27,6 +28,7 @@ __all__ = [ "get_tracing_config", "get_explicitly_enabled_tracing_providers", "get_enabled_tracing_providers", + "is_monocle_tracing_enabled", "is_tracing_enabled", "validate_enabled_tracing_providers", ] diff --git a/backend/packages/harness/deerflow/config/tracing_config.py b/backend/packages/harness/deerflow/config/tracing_config.py index 399e37424..79fb158e6 100644 --- a/backend/packages/harness/deerflow/config/tracing_config.py +++ b/backend/packages/harness/deerflow/config/tracing_config.py @@ -47,11 +47,47 @@ class LangfuseTracingConfig(BaseModel): raise ValueError(f"Langfuse tracing is enabled but required settings are missing: {', '.join(missing)}") +# Manual mirror of monocle_apptrace's supported exporters, kept local so a typo +# fails at startup with a clear message instead of an opaque upstream error. +# Update this tuple when a monocle_apptrace bump adds or renames an exporter. +_MONOCLE_EXPORTERS = ("file", "console", "okahu", "s3", "blob", "gcs") + + +class MonocleTracingConfig(BaseModel): + """Configuration for Monocle telemetry.""" + + enabled: bool = Field(...) + exporters: str = Field(...) + okahu_api_key: str | None = Field(...) + + @property + def is_enabled(self) -> bool: + # Unlike the siblings' is_configured, no credential check here: that is + # exporter-dependent and lives in validate(), run at Gateway startup. + return self.enabled + + @property + def exporter_list(self) -> list[str]: + """The configured exporters, parsed once so validation and setup agree.""" + return [e.strip() for e in self.exporters.split(",") if e.strip()] + + def validate(self) -> None: + if not self.enabled: + return + selected = self.exporter_list + unknown = [e for e in selected if e not in _MONOCLE_EXPORTERS] + if unknown: + raise ValueError(f"MONOCLE_EXPORTERS has unknown exporter(s): {', '.join(unknown)}. Allowed: {', '.join(_MONOCLE_EXPORTERS)}.") + if "okahu" in selected and not self.okahu_api_key: + raise ValueError("Monocle 'okahu' exporter is selected but OKAHU_API_KEY is not set.") + + class TracingConfig(BaseModel): """Tracing configuration for supported providers.""" langsmith: LangSmithTracingConfig = Field(...) langfuse: LangfuseTracingConfig = Field(...) + monocle: MonocleTracingConfig = Field(...) @property def is_configured(self) -> bool: @@ -125,6 +161,11 @@ def get_tracing_config() -> TracingConfig: secret_key=_first_env_value("LANGFUSE_SECRET_KEY"), host=_first_env_value("LANGFUSE_BASE_URL") or "https://cloud.langfuse.com", ), + monocle=MonocleTracingConfig( + enabled=_env_flag_preferred("MONOCLE_TRACING"), + exporters=_first_env_value("MONOCLE_EXPORTERS") or "file", + okahu_api_key=_first_env_value("OKAHU_API_KEY"), + ), ) return _tracing_config @@ -149,6 +190,16 @@ def is_tracing_enabled() -> bool: return get_tracing_config().is_configured +def is_monocle_tracing_enabled() -> bool: + """Whether Monocle OTel observability is enabled (via ``MONOCLE_TRACING``). + + Kept separate from :func:`get_enabled_tracing_providers` because Monocle is a + process-global instrumentor activated at startup, not a per-run LangChain + callback. + """ + return get_tracing_config().monocle.is_enabled + + def reset_tracing_config() -> None: """Discard the cached :class:`TracingConfig` so the next call rebuilds it. diff --git a/backend/packages/harness/deerflow/tracing/__init__.py b/backend/packages/harness/deerflow/tracing/__init__.py index 6d00e9c69..cfec1f256 100644 --- a/backend/packages/harness/deerflow/tracing/__init__.py +++ b/backend/packages/harness/deerflow/tracing/__init__.py @@ -1,8 +1,10 @@ from .factory import build_tracing_callbacks from .metadata import build_langfuse_trace_metadata, inject_langfuse_metadata +from .monocle import setup_monocle_tracing_if_enabled __all__ = [ "build_langfuse_trace_metadata", "build_tracing_callbacks", "inject_langfuse_metadata", + "setup_monocle_tracing_if_enabled", ] diff --git a/backend/packages/harness/deerflow/tracing/factory.py b/backend/packages/harness/deerflow/tracing/factory.py index a8ef85721..896f8f316 100644 --- a/backend/packages/harness/deerflow/tracing/factory.py +++ b/backend/packages/harness/deerflow/tracing/factory.py @@ -1,12 +1,17 @@ from __future__ import annotations +import logging from typing import Any from deerflow.config import ( get_enabled_tracing_providers, get_tracing_config, + is_monocle_tracing_enabled, validate_enabled_tracing_providers, ) +from deerflow.tracing.monocle import is_monocle_setup_completed + +logger = logging.getLogger(__name__) def _create_langsmith_tracer(config) -> Any: @@ -32,6 +37,12 @@ def _create_langfuse_handler(config) -> Any: def build_tracing_callbacks() -> list[Any]: """Build callbacks for all explicitly enabled tracing providers.""" validate_enabled_tracing_providers() + # Monocle is not a callback provider; this per-run path is just where an + # embedded process that skipped Gateway-lifespan setup can be told about it. + if is_monocle_tracing_enabled() and not is_monocle_setup_completed(): + logger.debug( + "MONOCLE_TRACING is set but Monocle is not initialized in this process — only the Gateway lifespan runs setup automatically; embedded/TUI callers must call deerflow.tracing.setup_monocle_tracing_if_enabled() themselves." + ) enabled_providers = get_enabled_tracing_providers() if not enabled_providers: return [] diff --git a/backend/packages/harness/deerflow/tracing/monocle.py b/backend/packages/harness/deerflow/tracing/monocle.py new file mode 100644 index 000000000..4da611263 --- /dev/null +++ b/backend/packages/harness/deerflow/tracing/monocle.py @@ -0,0 +1,69 @@ +"""Monocle telemetry: initialized once from the Gateway lifespan when ``MONOCLE_TRACING`` is set.""" + +from __future__ import annotations + +import logging + +from deerflow.config import ( + get_enabled_tracing_providers, + get_tracing_config, + is_monocle_tracing_enabled, +) + +logger = logging.getLogger(__name__) + +# Read by build_tracing_callbacks() to hint embedded/TUI processes that +# enabled MONOCLE_TRACING but never ran the Gateway-lifespan setup. +_setup_completed = False + + +def is_monocle_setup_completed() -> bool: + """Whether :func:`setup_monocle_tracing_if_enabled` ran in this process.""" + return _setup_completed + + +def setup_monocle_tracing_if_enabled() -> bool: + """Initialize Monocle telemetry when ``MONOCLE_TRACING`` is enabled; a no-op otherwise. + + ``monocle_apptrace.setup_monocle_telemetry()`` is idempotent, so this stays a thin, + config-gated wrapper. Returns ``True`` when enabled. + """ + if not is_monocle_tracing_enabled(): + return False + + monocle = get_tracing_config().monocle + # Fail fast on an unknown MONOCLE_EXPORTERS value or a missing OKAHU_API_KEY, + # with a clear message, before instrumenting. Validated here (not in the + # per-run callback path) so a config typo never breaks agent runs. + monocle.validate() + + # Coexistence with Langfuse (v4, also OTel-based) is verified: whichever + # library initializes second reuses the existing global TracerProvider and + # attaches its own span processor, so neither side loses spans (see + # test_coexists_with_langfuse). Both processors see all spans, so Monocle's + # exporters also capture Langfuse's spans when both are enabled. + exporters = monocle.exporters + + # `console` stays on local stdout, so only the remote exporters are flagged. + off_box = [e for e in monocle.exporter_list if e not in ("file", "console")] + if off_box: + # Monocle's exporters see every span on the shared global provider, so a + # co-enabled OTel provider's spans leave the box too. + langfuse_note = " Langfuse is also enabled and shares the global provider, so its spans are exported there as well." if "langfuse" in get_enabled_tracing_providers() else "" + logger.warning( + "Monocle is exporting trace data (prompts, tool inputs/outputs, completions) beyond the local .monocle/ file via: %s. Make sure that destination is trusted.%s", + ", ".join(off_box), + langfuse_note, + ) + + try: + from monocle_apptrace import setup_monocle_telemetry + except ImportError as exc: + raise RuntimeError("MONOCLE_TRACING is enabled but monocle_apptrace is not installed. Install the 'monocle' extra: `uv sync --extra monocle` in backend/, or `pip install 'deerflow-harness[monocle]'`.") from exc + + # monocle_exporters_list takes the comma-separated string as-is (monocle_apptrace's API). + setup_monocle_telemetry(workflow_name="deer-flow", monocle_exporters_list=exporters) + global _setup_completed + _setup_completed = True + logger.info("Monocle telemetry enabled (exporters=%s)", exporters) + return True diff --git a/backend/packages/harness/pyproject.toml b/backend/packages/harness/pyproject.toml index 746d5badf..d78a19e27 100644 --- a/backend/packages/harness/pyproject.toml +++ b/backend/packages/harness/pyproject.toml @@ -65,6 +65,9 @@ postgres = [ redis = ["redis>=5.0.0"] pymupdf = ["pymupdf4llm>=0.0.17"] boxlite = ["boxlite>=0.9.7"] +# Agent observability (Monocle). Optional so a default install stays free of the +# OpenTelemetry stack; only pulled in when MONOCLE_TRACING is used. +monocle = ["monocle_apptrace>=0.8.8"] [build-system] requires = ["hatchling"] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 26456b6c6..917d0c42d 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ postgres = ["deerflow-harness[postgres]"] redis = ["deerflow-harness[redis]"] discord = ["discord.py>=2.7.0"] +monocle = ["deerflow-harness[monocle]"] [dependency-groups] dev = [ @@ -37,6 +38,9 @@ dev = [ "pytest>=9.0.3", "pytest-asyncio>=1.3.0", "ruff>=0.14.11", + # Monocle tracer (also the deerflow-harness[monocle] extra); kept in the dev + # group so the tracing tests can import it without forcing it onto installs. + "monocle_apptrace>=0.8.8", # redis is an optional runtime extra (deerflow-harness[redis]); pin it in the # dev group so the stream-bridge tests can always import/exercise the redis # bridge without forcing it onto production installs. diff --git a/backend/tests/test_monocle_tracing.py b/backend/tests/test_monocle_tracing.py new file mode 100644 index 000000000..075897ab8 --- /dev/null +++ b/backend/tests/test_monocle_tracing.py @@ -0,0 +1,398 @@ +"""Tests for Monocle telemetry setup. + +Covers the config gate (``MONOCLE_TRACING`` default off / toggle on), the setup +helper's behavior (off-box exporter warning, exporter validation, idempotency, +Langfuse coexistence), the Gateway-lifespan wiring, and the regression that +importing ``deerflow.agents`` no longer sets up telemetry at import time. +""" + +from __future__ import annotations + +import asyncio +import logging +import subprocess +import sys +import textwrap +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +# monocle_apptrace is an optional extra (pinned in the dev group); skip the whole +# module in minimal installs instead of erroring at collection. +pytest.importorskip("monocle_apptrace") + +from deerflow.config import is_monocle_tracing_enabled +from deerflow.config.tracing_config import get_tracing_config, reset_tracing_config +from deerflow.tracing.monocle import setup_monocle_tracing_if_enabled + +_TRACING_ENV = ( + "MONOCLE_TRACING", + "MONOCLE_EXPORTERS", + "OKAHU_API_KEY", + "LANGFUSE_TRACING", + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", +) + + +@pytest.fixture(autouse=True) +def clear_monocle_env(monkeypatch): + for name in _TRACING_ENV: + monkeypatch.delenv(name, raising=False) + # The setup-completed flag is process-global; reset it so a test that runs + # (mocked) setup cannot change how later tests observe the embedded hint. + monkeypatch.setattr("deerflow.tracing.monocle._setup_completed", False) + reset_tracing_config() + yield + reset_tracing_config() + + +def test_disabled_by_default(): + assert is_monocle_tracing_enabled() is False + assert get_tracing_config().monocle.enabled is False + + +def test_setup_noop_when_disabled(monkeypatch): + called = False + + def _fail(*args, **kwargs): + nonlocal called + called = True + + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", _fail) + assert setup_monocle_tracing_if_enabled() is False + assert called is False + + +def test_toggles_on_and_sets_up(monkeypatch): + monkeypatch.setenv("MONOCLE_TRACING", "true") + reset_tracing_config() + + captured: dict = {} + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: captured.update(kw)) + + assert is_monocle_tracing_enabled() is True + assert setup_monocle_tracing_if_enabled() is True + assert captured == {"workflow_name": "deer-flow", "monocle_exporters_list": "file"} + + +def test_custom_exporters(monkeypatch): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "file,console") + reset_tracing_config() + + captured: dict = {} + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: captured.update(kw)) + + assert setup_monocle_tracing_if_enabled() is True + assert captured["monocle_exporters_list"] == "file,console" + + +def test_warns_on_non_file_exporter(monkeypatch, caplog): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "file,s3") + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + with caplog.at_level(logging.WARNING): + assert setup_monocle_tracing_if_enabled() is True + + warnings = [r.message for r in caplog.records if "beyond the local" in r.message] + assert warnings, "expected an off-box exporter warning" + assert "s3" in warnings[0] + assert "Langfuse" not in warnings[0] # only mentioned when Langfuse is co-enabled + + +def test_off_box_warning_mentions_langfuse_when_co_enabled(monkeypatch, caplog): + """With Langfuse sharing the global provider, its spans leave the box too — say so.""" + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "okahu") + monkeypatch.setenv("OKAHU_API_KEY", "okh_test") + monkeypatch.setenv("LANGFUSE_TRACING", "true") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + with caplog.at_level(logging.WARNING): + assert setup_monocle_tracing_if_enabled() is True + + warnings = [r.message for r in caplog.records if "beyond the local" in r.message] + assert warnings, "expected an off-box exporter warning" + assert "Langfuse" in warnings[0] + + +def test_no_off_box_warning_for_file_exporter(monkeypatch, caplog): + monkeypatch.setenv("MONOCLE_TRACING", "true") # default exporter is file + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + with caplog.at_level(logging.WARNING): + assert setup_monocle_tracing_if_enabled() is True + + assert not any("beyond the local" in r.message for r in caplog.records) + + +def test_no_off_box_warning_for_console_exporter(monkeypatch, caplog): + """``console`` writes to local stdout, so it must not trip the off-box warning.""" + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "file,console") + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + with caplog.at_level(logging.WARNING): + assert setup_monocle_tracing_if_enabled() is True + + assert not any("beyond the local" in r.message for r in caplog.records) + + +def test_coexists_with_langfuse(): + """Monocle and Langfuse (v4, OTel-based) share the global provider without span loss. + + Verified against the installed langfuse: whichever library initializes second + reuses the existing global ``TracerProvider`` and attaches its own span + processor, so both sides keep exporting. Runs the real setup (no mocks) in a + subprocess so the process-global provider never leaks into the suite. + """ + script = textwrap.dedent( + """ + import os + os.environ["MONOCLE_TRACING"] = "true" + os.environ["MONOCLE_EXPORTERS"] = "console" + os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-test" + os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-test" + os.environ["LANGFUSE_HOST"] = "http://127.0.0.1:9" # unreachable; offline test + + from opentelemetry import trace + + from deerflow.tracing.monocle import setup_monocle_tracing_if_enabled + + # Gateway order: Monocle at startup, Langfuse per-run afterwards. + assert setup_monocle_tracing_if_enabled() is True + provider = trace.get_tracer_provider() + + from langfuse import Langfuse + + Langfuse(tracing_enabled=True) + + assert trace.get_tracer_provider() is provider # provider not replaced + # NOTE: reaches into OTel SDK internals (no public API lists a + # provider's span processors). The SDK is pinned by uv.lock; if a bump + # renames these attributes, update this introspection — the coexistence + # behavior itself is unaffected. + names = [type(p).__name__ for p in provider._active_span_processor._span_processors] + assert any("Langfuse" in n for n in names), names # Langfuse attached alongside Monocle + print("COEXIST_OK") + """ + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert "COEXIST_OK" in result.stdout + + +def test_rejects_unknown_exporter(monkeypatch): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "fle") + reset_tracing_config() + with pytest.raises(ValueError, match="unknown exporter"): + setup_monocle_tracing_if_enabled() + + +def test_okahu_exporter_requires_api_key(monkeypatch): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "okahu") + reset_tracing_config() + with pytest.raises(ValueError, match="OKAHU_API_KEY"): + setup_monocle_tracing_if_enabled() + + +def test_okahu_exporter_with_api_key_ok(monkeypatch): + monkeypatch.setenv("MONOCLE_TRACING", "true") + monkeypatch.setenv("MONOCLE_EXPORTERS", "okahu") + monkeypatch.setenv("OKAHU_API_KEY", "okh_test") + reset_tracing_config() + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + + assert setup_monocle_tracing_if_enabled() is True + + +def test_embedded_hint_when_enabled_but_uninitialized(monkeypatch, caplog): + """``build_tracing_callbacks()`` hints when Monocle is enabled but setup never ran. + + The embedded ``DeerFlowClient`` and the TUI never hit the Gateway lifespan, + so this debug line is the only in-process signal explaining why no Monocle + traces appear. + """ + from deerflow.tracing import build_tracing_callbacks + + monkeypatch.setenv("MONOCLE_TRACING", "true") + reset_tracing_config() + monkeypatch.setattr("deerflow.tracing.monocle._setup_completed", False) + + # Scoped to the factory's logger: earlier tests in the suite may have run + # configure_logging(), which pins an explicit INFO level on the hierarchy + # that a root-level caplog.at_level(DEBUG) would not override. + with caplog.at_level(logging.DEBUG, logger="deerflow.tracing.factory"): + assert build_tracing_callbacks() == [] + + assert any("not initialized in this process" in r.message for r in caplog.records) + + +def test_no_embedded_hint_after_setup(monkeypatch, caplog): + from deerflow.tracing import build_tracing_callbacks + + monkeypatch.setenv("MONOCLE_TRACING", "true") + reset_tracing_config() + monkeypatch.setattr("deerflow.tracing.monocle._setup_completed", False) + monkeypatch.setattr("monocle_apptrace.setup_monocle_telemetry", lambda **kw: None) + assert setup_monocle_tracing_if_enabled() is True + + with caplog.at_level(logging.DEBUG, logger="deerflow.tracing.factory"): + build_tracing_callbacks() + + assert not any("not initialized in this process" in r.message for r in caplog.records) + + +def test_no_import_time_setup(): + """Regression: importing deerflow.agents must not install telemetry. + + The setup call used to live at module import in ``deerflow/agents/__init__``. + It now happens only via the gateway lifespan, so a plain import must neither + expose ``setup_monocle_telemetry`` nor install a global OTel + ``TracerProvider`` (which is what ``setup_monocle_telemetry`` does). Runs in + a subprocess so the import is genuinely fresh and, unlike deleting + ``sys.modules`` entries in-process, cannot corrupt module identity for the + rest of the suite. + """ + script = textwrap.dedent( + """ + from opentelemetry import trace + from opentelemetry.trace import ProxyTracerProvider + + import deerflow.agents as agents + + assert not hasattr(agents, "setup_monocle_telemetry") + # Still the SDK-less default proxy: no provider was installed on import. + assert isinstance(trace.get_tracer_provider(), ProxyTracerProvider) + print("IMPORT_CLEAN_OK") + """ + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert "IMPORT_CLEAN_OK" in result.stdout + + +def test_double_invoke_is_idempotent(): + """Calling setup twice must not double-instrument. + + Exercises upstream ``check_duplicate_setup`` with the real tracer (no mock). + Run in a subprocess so the process-global OTel provider it installs never + leaks into the rest of the suite. + """ + script = textwrap.dedent( + """ + import os + os.environ["MONOCLE_TRACING"] = "true" + os.environ["MONOCLE_EXPORTERS"] = "console" # avoid writing .monocle/ files + from deerflow.tracing.monocle import setup_monocle_tracing_if_enabled + from monocle_apptrace.instrumentation.common.instrumentor import get_monocle_instrumentor + + assert setup_monocle_tracing_if_enabled() is True + first = get_monocle_instrumentor() + assert first is not None + assert setup_monocle_tracing_if_enabled() is True + assert get_monocle_instrumentor() is first # no second provider installed + print("IDEMPOTENT_OK") + """ + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert "IDEMPOTENT_OK" in result.stdout + + +def test_gateway_lifespan_initializes_monocle(): + """The Gateway lifespan is the sole Monocle call site; pin that wiring. + + Mirrors the patching in ``test_gateway_lifespan_shutdown.py`` so the lifespan + can be driven directly, and asserts the setup helper runs during startup. + """ + from fastapi import FastAPI + + from app.gateway.app import lifespan + + @asynccontextmanager + async def _noop_langgraph_runtime(_app, _startup_config): + yield + + startup_config = SimpleNamespace(log_level="INFO", memory=SimpleNamespace(token_counting="char")) + fake_service = MagicMock() + fake_service.get_status = MagicMock(return_value={}) + + async def fake_start(_startup_config): + return fake_service + + setup_spy = MagicMock(return_value=False) + + with ( + patch("app.gateway.app.get_app_config", return_value=startup_config), + patch("app.gateway.app.get_gateway_config", return_value=MagicMock(host="x", port=0)), + patch("app.gateway.app.langgraph_runtime", _noop_langgraph_runtime), + patch("app.gateway.app.setup_monocle_tracing_if_enabled", setup_spy), + patch("app.gateway.app.auth.close_oidc_service", AsyncMock()), + patch("app.channels.service.start_channel_service", side_effect=fake_start), + patch("app.channels.service.stop_channel_service", AsyncMock()), + ): + + async def drive() -> None: + async with lifespan(FastAPI()): + pass + + asyncio.run(drive()) + + setup_spy.assert_called_once_with() + + +def test_gateway_lifespan_survives_monocle_setup_failure(caplog): + """A raising Monocle setup (e.g. bad MONOCLE_EXPORTERS) must not break startup. + + Pins the lifespan's fail-open contract: the error is logged and the Gateway + keeps serving without tracing. + """ + from fastapi import FastAPI + + from app.gateway.app import lifespan + + @asynccontextmanager + async def _noop_langgraph_runtime(_app, _startup_config): + yield + + startup_config = SimpleNamespace(log_level="INFO", memory=SimpleNamespace(token_counting="char")) + fake_service = MagicMock() + fake_service.get_status = MagicMock(return_value={}) + + async def fake_start(_startup_config): + return fake_service + + setup_spy = MagicMock(side_effect=ValueError("MONOCLE_EXPORTERS has unknown exporter(s): fle.")) + + with ( + patch("app.gateway.app.get_app_config", return_value=startup_config), + patch("app.gateway.app.get_gateway_config", return_value=MagicMock(host="x", port=0)), + patch("app.gateway.app.langgraph_runtime", _noop_langgraph_runtime), + patch("app.gateway.app.setup_monocle_tracing_if_enabled", setup_spy), + patch("app.gateway.app.auth.close_oidc_service", AsyncMock()), + patch("app.channels.service.start_channel_service", side_effect=fake_start), + patch("app.channels.service.stop_channel_service", AsyncMock()), + ): + + async def drive() -> None: + async with lifespan(FastAPI()): + pass + + with caplog.at_level(logging.ERROR, logger="app.gateway.app"): + asyncio.run(drive()) # completes despite the raising setup + + setup_spy.assert_called_once_with() + assert any("Monocle tracing setup failed" in r.message for r in caplog.records) diff --git a/backend/tests/test_tracing_config.py b/backend/tests/test_tracing_config.py index 943401c97..164d70d3a 100644 --- a/backend/tests/test_tracing_config.py +++ b/backend/tests/test_tracing_config.py @@ -28,6 +28,9 @@ def clear_tracing_env(monkeypatch): "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL", + "MONOCLE_TRACING", + "MONOCLE_EXPORTERS", + "OKAHU_API_KEY", ): monkeypatch.delenv(name, raising=False) _reset_tracing_cache() diff --git a/backend/tests/test_tracing_factory.py b/backend/tests/test_tracing_factory.py index 723e42e80..eeac84224 100644 --- a/backend/tests/test_tracing_factory.py +++ b/backend/tests/test_tracing_factory.py @@ -28,6 +28,9 @@ def clear_tracing_env(monkeypatch): "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL", + "MONOCLE_TRACING", + "MONOCLE_EXPORTERS", + "OKAHU_API_KEY", ): monkeypatch.delenv(name, raising=False) reset_tracing_config() diff --git a/backend/uv.lock b/backend/uv.lock index 398f1b8a9..7249eacca 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -799,6 +799,9 @@ dependencies = [ discord = [ { name = "discord-py" }, ] +monocle = [ + { name = "deerflow-harness", extra = ["monocle"] }, +] postgres = [ { name = "deerflow-harness", extra = ["postgres"] }, ] @@ -810,6 +813,7 @@ redis = [ dev = [ { name = "blockbuster" }, { name = "jsonschema" }, + { name = "monocle-apptrace" }, { name = "prompt-toolkit" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -822,6 +826,7 @@ dev = [ requires-dist = [ { name = "bcrypt", specifier = ">=4.0.0" }, { name = "deerflow-harness", editable = "packages/harness" }, + { name = "deerflow-harness", extras = ["monocle"], marker = "extra == 'monocle'", editable = "packages/harness" }, { name = "deerflow-harness", extras = ["postgres"], marker = "extra == 'postgres'", editable = "packages/harness" }, { name = "deerflow-harness", extras = ["redis"], marker = "extra == 'redis'", editable = "packages/harness" }, { name = "dingtalk-stream", specifier = ">=0.24.3" }, @@ -841,12 +846,13 @@ requires-dist = [ { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, { name = "wecom-aibot-python-sdk", specifier = ">=0.1.6" }, ] -provides-extras = ["postgres", "redis", "discord"] +provides-extras = ["postgres", "redis", "discord", "monocle"] [package.metadata.requires-dev] dev = [ { name = "blockbuster", specifier = ">=1.5.26,<1.6" }, { name = "jsonschema", specifier = ">=4.26.0" }, + { name = "monocle-apptrace", specifier = ">=0.8.8" }, { name = "prompt-toolkit", specifier = ">=3.0.0" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, @@ -901,6 +907,9 @@ dependencies = [ boxlite = [ { name = "boxlite" }, ] +monocle = [ + { name = "monocle-apptrace" }, +] ollama = [ { name = "langchain-ollama" }, ] @@ -955,6 +964,7 @@ requires-dist = [ { name = "langgraph-sdk", specifier = ">=0.1.51" }, { name = "markdownify", specifier = ">=1.2.2" }, { name = "markitdown", extras = ["all", "xlsx"], specifier = ">=0.0.1a2" }, + { name = "monocle-apptrace", marker = "extra == 'monocle'", specifier = ">=0.8.8" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.3.3" }, { name = "psycopg-pool", marker = "extra == 'postgres'", specifier = ">=3.3.0" }, { name = "pydantic", specifier = ">=2.12.5" }, @@ -967,7 +977,7 @@ requires-dist = [ { name = "textual", marker = "extra == 'tui'", specifier = ">=0.80" }, { name = "tiktoken", specifier = ">=0.8.0" }, ] -provides-extras = ["tui", "groundroute", "ollama", "postgres", "redis", "pymupdf", "boxlite"] +provides-extras = ["tui", "groundroute", "ollama", "postgres", "redis", "pymupdf", "boxlite", "monocle"] [[package]] name = "defusedxml" @@ -2526,6 +2536,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "monocle-apptrace" +version = "0.8.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-sdk" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "rfc3986" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/61/ec2213cc56e2d5d6b67db691a408ca8ba9d8440ed48fc821a422b6a5560e/monocle_apptrace-0.8.8.tar.gz", hash = "sha256:ebebd8b8da8dbf48cdb708e14e1a61293c48e50ec02ba24cf29c01ea776ba02b", size = 241057, upload-time = "2026-07-08T18:01:51.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/8f/3757b6ca53346a32bc271e4d5c0dd64ad920eb1b05e514b00c8ff2e02c02/monocle_apptrace-0.8.8-py3-none-any.whl", hash = "sha256:7e17f0f73e4f446f8e8722251201ffc30429fea0d93d545b023b41bfc4f83503", size = 354697, upload-time = "2026-07-08T18:01:47.407Z" }, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -2870,6 +2899,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/4d/ef07ff2fc630849f2080ae0ae73a61f67257905b7ac79066640bfa0c5739/opentelemetry_exporter_otlp_proto_http-1.41.1-py3-none-any.whl", hash = "sha256:1a21e8f49c7a946d935551e90947d6c3eb39236723c6624401da0f33d68edcb4", size = 22673, upload-time = "2026-04-24T13:15:21.313Z" }, ] +[[package]] +name = "opentelemetry-instrumentation" +version = "0.62b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "packaging" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/cb/0523b92c112a6cc70be43724343dc45225d3af134419844d7879a07755d4/opentelemetry_instrumentation-0.62b1.tar.gz", hash = "sha256:90e92a905ba4f84db06ac3aec96701df6c079b2d66e9379f8739f0a1bdcc7f45", size = 34043, upload-time = "2026-04-24T13:22:31.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/0f/45adbaea1f81b847cffdcee4f4b5f89297e42facf7fac78c7aaac4c38e75/opentelemetry_instrumentation-0.62b1-py3-none-any.whl", hash = "sha256:976fc6e640f2006599e97429c949e622c108d0c17c2059347d1e6c93c707f257", size = 34163, upload-time = "2026-04-24T13:21:31.722Z" }, +] + [[package]] name = "opentelemetry-proto" version = "1.41.1" @@ -4014,6 +4058,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, ] +[[package]] +name = "rfc3986" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c", size = 49026, upload-time = "2022-01-10T00:52:30.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", size = 31326, upload-time = "2022-01-10T00:52:29.594Z" }, +] + [[package]] name = "rich" version = "15.0.0" diff --git a/config.example.yaml b/config.example.yaml index d0ac8fdaa..aa44b6e9e 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -30,6 +30,14 @@ logging: enabled: false format: text +# ============================================================================ +# Tracing / Observability (Monocle) +# ============================================================================ +# Optional agent tracing via Monocle. Configured through environment variables +# (MONOCLE_TRACING, MONOCLE_EXPORTERS, OKAHU_API_KEY — like LangSmith/Langfuse), +# not config.yaml keys, and OFF by default. See README.md → "Monocle Tracing" +# for setup, what each exporter captures, and where the trace data goes. + # ============================================================================ # Token Usage # ============================================================================