fix(runtime): protect sync singleton init and reset (#3413)

* fix(runtime): protect sync singleton init/reset with threading.Lock

* fix(runtime): serialize sync singleton init and reset

* make format

* test(runtime): assert store reset creates new singleton

* Apply suggestions from code review

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* fix(runtime): load config outside singleton locks

* fix(runtime): share checkpointer config loading helper

---------

Co-authored-by: GODDiao <diaoshengjia@gmail.com>
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

backend/packages/harness/deerflow/config/checkpointer_config.py

@@ -41,6 +41,20 @@ def set_checkpointer_config(config: CheckpointerConfig | None) -> None:
     _checkpointer_config = config
 
 
+def ensure_config_loaded() -> None:
+    """Lazily load app config when checkpointer config has not been initialized."""
+    from deerflow.config.app_config import _app_config, get_app_config
+
+    config = get_checkpointer_config()
+    if config is not None or _app_config is not None:
+        return
+
+    try:
+        get_app_config()
+    except FileNotFoundError:
+        pass
+
+
 def load_checkpointer_config_from_dict(config_dict: dict | None) -> None:
     """Load checkpointer configuration from a dictionary."""
     global _checkpointer_config

backend/packages/harness/deerflow/runtime/checkpointer/provider.py

@@ -21,12 +21,13 @@ from __future__ import annotations
 
 import contextlib
 import logging
+import threading
 from collections.abc import Iterator
 
 from langgraph.types import Checkpointer
 
 from deerflow.config.app_config import get_app_config
-from deerflow.config.checkpointer_config import CheckpointerConfig
+from deerflow.config.checkpointer_config import CheckpointerConfig, ensure_config_loaded
 from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
 
 logger = logging.getLogger(__name__)
@@ -100,6 +101,7 @@ def _sync_checkpointer_cm(config: CheckpointerConfig) -> Iterator[Checkpointer]:
 
 _checkpointer: Checkpointer | None = None
 _checkpointer_ctx = None  # open context manager keeping the connection alive
+_checkpointer_lock = threading.Lock()
 
 
 def get_checkpointer() -> Checkpointer:
@@ -116,34 +118,29 @@ def get_checkpointer() -> Checkpointer:
     if _checkpointer is not None:
         return _checkpointer
 
-    # Ensure app config is loaded before checking checkpointer config
-    # This prevents returning InMemorySaver when config.yaml actually has a checkpointer section
-    # but hasn't been loaded yet
-    from deerflow.config.app_config import _app_config
-    from deerflow.config.checkpointer_config import get_checkpointer_config
+    # Config loading can reset both persistence singletons. Keep it outside
+    # this provider lock to avoid cross-provider lock-order inversion.
+    ensure_config_loaded()
 
-    config = get_checkpointer_config()
+    with _checkpointer_lock:
+        if _checkpointer is not None:
+            return _checkpointer
+
+        from deerflow.config.checkpointer_config import get_checkpointer_config
 
-    if config is None and _app_config is None:
-        # Only load app config lazily when neither the app config nor an explicit
-        # checkpointer config has been initialized yet. This keeps tests that
-        # intentionally set the global checkpointer config isolated from any
-        # ambient config.yaml on disk.
-        try:
-            get_app_config()
-        except FileNotFoundError:
-            # In test environments without config.yaml, this is expected.
-            pass
         config = get_checkpointer_config()
-    if config is None:
-        from langgraph.checkpoint.memory import InMemorySaver
 
-        logger.info("Checkpointer: using InMemorySaver (in-process, not persistent)")
-        _checkpointer = InMemorySaver()
-        return _checkpointer
+        if config is None:
+            from langgraph.checkpoint.memory import InMemorySaver
 
-    _checkpointer_ctx = _sync_checkpointer_cm(config)
-    _checkpointer = _checkpointer_ctx.__enter__()
+            logger.info("Checkpointer: using InMemorySaver (in-process, not persistent)")
+            _checkpointer = InMemorySaver()
+            return _checkpointer
+
+        checkpointer_ctx = _sync_checkpointer_cm(config)
+        checkpointer = checkpointer_ctx.__enter__()
+        _checkpointer_ctx = checkpointer_ctx
+        _checkpointer = checkpointer
 
     return _checkpointer
 
@@ -155,13 +152,14 @@ def reset_checkpointer() -> None:
     Useful in tests or after a configuration change.
     """
     global _checkpointer, _checkpointer_ctx
-    if _checkpointer_ctx is not None:
-        try:
-            _checkpointer_ctx.__exit__(None, None, None)
-        except Exception:
-            logger.warning("Error during checkpointer cleanup", exc_info=True)
-        _checkpointer_ctx = None
-    _checkpointer = None
+    with _checkpointer_lock:
+        if _checkpointer_ctx is not None:
+            try:
+                _checkpointer_ctx.__exit__(None, None, None)
+            except Exception:
+                logger.warning("Error during checkpointer cleanup", exc_info=True)
+            _checkpointer_ctx = None
+        _checkpointer = None
 
 
 # ---------------------------------------------------------------------------

backend/packages/harness/deerflow/runtime/store/provider.py

@@ -22,11 +22,13 @@ from __future__ import annotations
 
 import contextlib
 import logging
+import threading
 from collections.abc import Iterator
 
 from langgraph.store.base import BaseStore
 
 from deerflow.config.app_config import get_app_config
+from deerflow.config.checkpointer_config import ensure_config_loaded
 from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
 
 logger = logging.getLogger(__name__)
@@ -100,6 +102,7 @@ def _sync_store_cm(config) -> Iterator[BaseStore]:
 
 _store: BaseStore | None = None
 _store_ctx = None  # open context manager keeping the connection alive
+_store_lock = threading.Lock()
 
 
 def get_store() -> BaseStore:
@@ -117,29 +120,29 @@ def get_store() -> BaseStore:
     if _store is not None:
         return _store
 
-    # Lazily load app config, mirroring the checkpointer singleton pattern so
-    # that tests that set the global checkpointer config explicitly remain isolated.
-    from deerflow.config.app_config import _app_config
-    from deerflow.config.checkpointer_config import get_checkpointer_config
+    # Config loading can reset both persistence singletons. Keep it outside
+    # this provider lock to avoid cross-provider lock-order inversion.
+    ensure_config_loaded()
 
-    config = get_checkpointer_config()
+    with _store_lock:
+        if _store is not None:
+            return _store
+
+        from deerflow.config.checkpointer_config import get_checkpointer_config
 
-    if config is None and _app_config is None:
-        try:
-            get_app_config()
-        except FileNotFoundError:
-            pass
         config = get_checkpointer_config()
 
-    if config is None:
-        from langgraph.store.memory import InMemoryStore
+        if config is None:
+            from langgraph.store.memory import InMemoryStore
 
-        logger.warning("No 'checkpointer' section in config.yaml — using InMemoryStore for the store. Thread list will be lost on server restart. Configure a sqlite or postgres backend for persistence.")
-        _store = InMemoryStore()
-        return _store
+            logger.warning("No 'checkpointer' section in config.yaml — using InMemoryStore for the store. Thread list will be lost on server restart. Configure a sqlite or postgres backend for persistence.")
+            _store = InMemoryStore()
+            return _store
 
-    _store_ctx = _sync_store_cm(config)
-    _store = _store_ctx.__enter__()
+        store_ctx = _sync_store_cm(config)
+        store = store_ctx.__enter__()
+        _store_ctx = store_ctx
+        _store = store
     return _store
 
 
@@ -150,13 +153,14 @@ def reset_store() -> None:
     Useful in tests or after a configuration change.
     """
     global _store, _store_ctx
-    if _store_ctx is not None:
-        try:
-            _store_ctx.__exit__(None, None, None)
-        except Exception:
-            logger.warning("Error during store cleanup", exc_info=True)
-        _store_ctx = None
-    _store = None
+    with _store_lock:
+        if _store_ctx is not None:
+            try:
+                _store_ctx.__exit__(None, None, None)
+            except Exception:
+                logger.warning("Error during store cleanup", exc_info=True)
+            _store_ctx = None
+        _store = None
 
 
 # ---------------------------------------------------------------------------