mirror of
https://github.com/furyhawk/agent_alpha.git
synced 2026-07-21 10:35:34 +00:00
- Added `vectorstore.py` for managing vector storage operations using Milvus. - Introduced `rag_document.py` for handling RAG document lifecycle, including ingestion and status updates. - Created `rag_status.py` for streaming RAG ingestion status via Redis pub/sub. - Developed `rag_sync.py` for managing synchronization operations and logs. - Implemented `sync_source.py` for managing sync source configurations and triggering syncs. - Established a lightweight task dispatcher in `dispatcher.py` for background task execution. - Registered task functions for RAG ingestion and sync operations in `rag_tasks.py`. - Configured task dispatcher settings in `arq_settings.py`. - Added necessary database models and schemas for RAG document and sync operations.
142 lines
4.6 KiB
Python
142 lines
4.6 KiB
Python
"""File storage service for chat file uploads.
|
|
|
|
Supports local filesystem storage.
|
|
Files are organized per-user: {storage_root}/{user_id}/{uuid}_{filename}
|
|
"""
|
|
|
|
import logging
|
|
import re
|
|
import uuid
|
|
from abc import ABC, abstractmethod
|
|
from pathlib import Path
|
|
|
|
from backend.core.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
ALLOWED_MIME_TYPES = {
|
|
"image/jpeg",
|
|
"image/png",
|
|
"image/gif",
|
|
"image/webp",
|
|
"text/plain",
|
|
"text/markdown",
|
|
"text/csv",
|
|
"text/html",
|
|
"text/css",
|
|
"text/xml",
|
|
"text/x-python",
|
|
"text/javascript",
|
|
"text/x-yaml",
|
|
"application/json",
|
|
"application/pdf",
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
"application/x-yaml",
|
|
}
|
|
|
|
IMAGE_MIME_TYPES = {"image/jpeg", "image/png", "image/gif", "image/webp"}
|
|
|
|
MAX_UPLOAD_SIZE = 10 * 1024 * 1024 # 10MB
|
|
|
|
|
|
def classify_file(mime_type: str, filename: str) -> str:
|
|
"""Classify file type based on MIME type and extension."""
|
|
if mime_type in IMAGE_MIME_TYPES:
|
|
return "image"
|
|
if mime_type == "application/pdf" or filename.lower().endswith(".pdf"):
|
|
return "pdf"
|
|
ext = filename.lower().rsplit(".", 1)[-1] if "." in filename else ""
|
|
if ext == "docx" or "wordprocessingml" in mime_type:
|
|
return "docx"
|
|
return "text"
|
|
|
|
|
|
_UNSAFE_FILENAME_CHARS = re.compile(r"[^\w.\-]+")
|
|
|
|
|
|
def _sanitize_filename(filename: str) -> str:
|
|
"""Strip path separators, NULL bytes, and unsafe chars from a filename.
|
|
|
|
The result is always a single path component with no traversal segments.
|
|
Empty results fall back to ``"file"`` to preserve a non-empty name.
|
|
"""
|
|
base = Path(filename).name.replace("\x00", "")
|
|
cleaned = _UNSAFE_FILENAME_CHARS.sub("_", base).strip("._")
|
|
return cleaned or "file"
|
|
|
|
|
|
def make_storage_filename(filename: str) -> str:
|
|
"""Create a unique storage filename to prevent collisions and path traversal."""
|
|
safe = _sanitize_filename(filename)
|
|
return f"{uuid.uuid4().hex[:12]}_{safe}"
|
|
|
|
|
|
class BaseFileStorage(ABC):
|
|
"""Abstract file storage backend."""
|
|
|
|
@abstractmethod
|
|
async def save(self, user_id: str, filename: str, data: bytes) -> str:
|
|
"""Save file and return storage path/key."""
|
|
|
|
@abstractmethod
|
|
async def load(self, storage_path: str) -> bytes:
|
|
"""Load file bytes by storage path."""
|
|
|
|
@abstractmethod
|
|
async def delete(self, storage_path: str) -> None:
|
|
"""Delete file by storage path."""
|
|
|
|
def get_full_path(self, storage_path: str) -> Path | None:
|
|
"""Return absolute filesystem path if available (local storage only)."""
|
|
return None # pragma: no cover
|
|
|
|
|
|
class LocalFileStorage(BaseFileStorage):
|
|
"""Store files on local filesystem."""
|
|
|
|
def __init__(self, base_dir: str | Path = "media"):
|
|
self.base_dir = Path(base_dir)
|
|
self.base_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
def _resolve_safe_path(self, storage_path: str) -> Path:
|
|
"""Resolve a storage path under base_dir, rejecting traversal attempts."""
|
|
base = self.base_dir.resolve()
|
|
candidate = (base / storage_path).resolve()
|
|
if base != candidate and base not in candidate.parents:
|
|
raise ValueError(f"Path escapes storage root: {storage_path}")
|
|
return candidate
|
|
|
|
async def save(self, user_id: str, filename: str, data: bytes) -> str:
|
|
safe_user = _sanitize_filename(user_id)
|
|
user_dir = self.base_dir / safe_user
|
|
user_dir.mkdir(parents=True, exist_ok=True)
|
|
storage_name = make_storage_filename(filename)
|
|
file_path = user_dir / storage_name
|
|
file_path.write_bytes(data)
|
|
return f"{safe_user}/{storage_name}"
|
|
|
|
async def load(self, storage_path: str) -> bytes:
|
|
file_path = self._resolve_safe_path(storage_path)
|
|
if not file_path.exists():
|
|
raise FileNotFoundError(f"File not found: {storage_path}")
|
|
return file_path.read_bytes()
|
|
|
|
async def delete(self, storage_path: str) -> None:
|
|
file_path = self._resolve_safe_path(storage_path)
|
|
if file_path.exists():
|
|
file_path.unlink()
|
|
|
|
def get_full_path(self, storage_path: str) -> Path | None:
|
|
"""Return absolute filesystem path for local files."""
|
|
try:
|
|
file_path = self._resolve_safe_path(storage_path)
|
|
except ValueError:
|
|
return None
|
|
return file_path if file_path.exists() else None
|
|
|
|
|
|
def get_file_storage() -> BaseFileStorage:
|
|
"""Factory: create file storage backend based on settings."""
|
|
media_dir = getattr(settings, "media_dir", "media")
|
|
return LocalFileStorage(base_dir=media_dir)
|