fix(channels): centralize shared channel retry helpers (#3583)

backend/app/channels/base.py

@@ -2,14 +2,19 @@
 
 from __future__ import annotations
 
+import asyncio
 import logging
 from abc import ABC, abstractmethod
-from typing import Any
+from collections.abc import Awaitable, Callable
+from concurrent.futures import CancelledError as FutureCancelledError
+from typing import Any, TypeVar
 
 from app.channels.message_bus import InboundMessage, InboundMessageType, MessageBus, OutboundMessage, ResolvedAttachment
 
 logger = logging.getLogger(__name__)
 
+T = TypeVar("T")
+
 
 class Channel(ABC):
     """Base class for all IM channel implementations.
@@ -65,6 +70,53 @@ class Channel(ABC):
 
     # -- helpers -----------------------------------------------------------
 
+    async def _send_with_retry(
+        self,
+        operation: Callable[[], Awaitable[T]],
+        *,
+        max_retries: int,
+        log_prefix: str | None = None,
+        operation_name: str = "send",
+    ) -> T:
+        """Run an outbound send operation with the shared channel retry policy."""
+        prefix = log_prefix or f"[{self.name}]"
+        last_exc: Exception | None = None
+        for attempt in range(max_retries):
+            try:
+                return await operation()
+            except Exception as exc:
+                last_exc = exc
+                if attempt < max_retries - 1:
+                    delay = 2**attempt
+                    logger.warning(
+                        "%s %s failed (attempt %d/%d), retrying in %ds: %s",
+                        prefix,
+                        operation_name,
+                        attempt + 1,
+                        max_retries,
+                        delay,
+                        exc,
+                    )
+                    await asyncio.sleep(delay)
+
+        logger.error("%s %s failed after %d attempts: %s", prefix, operation_name, max_retries, last_exc)
+        if last_exc is None:
+            raise RuntimeError(f"{self.name} {operation_name} failed without an exception from any attempt")
+        raise last_exc
+
+    def _log_future_error(self, fut: Any, name: str, msg_id: Any) -> None:
+        """Callback for concurrent futures scheduled from channel worker threads."""
+        try:
+            exc = fut.exception()
+        except (asyncio.CancelledError, FutureCancelledError, asyncio.InvalidStateError):
+            return
+        except Exception:
+            logger.exception("[%s] failed to inspect future for %s (msg_id=%s)", self.name, name, msg_id)
+            return
+
+        if exc:
+            logger.error("[%s] %s failed for msg_id=%s: %s", self.name, name, msg_id, exc)
+
     def _make_inbound(
         self,
         chat_id: str,