Files
furyhawk bb4b614711 feat: Implement RAG document and sync services with vector store integration
- Added `vectorstore.py` for managing vector storage operations using Milvus.
- Introduced `rag_document.py` for handling RAG document lifecycle, including ingestion and status updates.
- Created `rag_status.py` for streaming RAG ingestion status via Redis pub/sub.
- Developed `rag_sync.py` for managing synchronization operations and logs.
- Implemented `sync_source.py` for managing sync source configurations and triggering syncs.
- Established a lightweight task dispatcher in `dispatcher.py` for background task execution.
- Registered task functions for RAG ingestion and sync operations in `rag_tasks.py`.
- Configured task dispatcher settings in `arq_settings.py`.
- Added necessary database models and schemas for RAG document and sync operations.
2026-06-13 21:52:27 +08:00

142 lines
4.6 KiB
Python

"""File storage service for chat file uploads.
Supports local filesystem storage.
Files are organized per-user: {storage_root}/{user_id}/{uuid}_{filename}
"""
import logging
import re
import uuid
from abc import ABC, abstractmethod
from pathlib import Path
from backend.core.config import settings
logger = logging.getLogger(__name__)
ALLOWED_MIME_TYPES = {
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
"text/plain",
"text/markdown",
"text/csv",
"text/html",
"text/css",
"text/xml",
"text/x-python",
"text/javascript",
"text/x-yaml",
"application/json",
"application/pdf",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/x-yaml",
}
IMAGE_MIME_TYPES = {"image/jpeg", "image/png", "image/gif", "image/webp"}
MAX_UPLOAD_SIZE = 10 * 1024 * 1024 # 10MB
def classify_file(mime_type: str, filename: str) -> str:
"""Classify file type based on MIME type and extension."""
if mime_type in IMAGE_MIME_TYPES:
return "image"
if mime_type == "application/pdf" or filename.lower().endswith(".pdf"):
return "pdf"
ext = filename.lower().rsplit(".", 1)[-1] if "." in filename else ""
if ext == "docx" or "wordprocessingml" in mime_type:
return "docx"
return "text"
_UNSAFE_FILENAME_CHARS = re.compile(r"[^\w.\-]+")
def _sanitize_filename(filename: str) -> str:
"""Strip path separators, NULL bytes, and unsafe chars from a filename.
The result is always a single path component with no traversal segments.
Empty results fall back to ``"file"`` to preserve a non-empty name.
"""
base = Path(filename).name.replace("\x00", "")
cleaned = _UNSAFE_FILENAME_CHARS.sub("_", base).strip("._")
return cleaned or "file"
def make_storage_filename(filename: str) -> str:
"""Create a unique storage filename to prevent collisions and path traversal."""
safe = _sanitize_filename(filename)
return f"{uuid.uuid4().hex[:12]}_{safe}"
class BaseFileStorage(ABC):
"""Abstract file storage backend."""
@abstractmethod
async def save(self, user_id: str, filename: str, data: bytes) -> str:
"""Save file and return storage path/key."""
@abstractmethod
async def load(self, storage_path: str) -> bytes:
"""Load file bytes by storage path."""
@abstractmethod
async def delete(self, storage_path: str) -> None:
"""Delete file by storage path."""
def get_full_path(self, storage_path: str) -> Path | None:
"""Return absolute filesystem path if available (local storage only)."""
return None # pragma: no cover
class LocalFileStorage(BaseFileStorage):
"""Store files on local filesystem."""
def __init__(self, base_dir: str | Path = "media"):
self.base_dir = Path(base_dir)
self.base_dir.mkdir(parents=True, exist_ok=True)
def _resolve_safe_path(self, storage_path: str) -> Path:
"""Resolve a storage path under base_dir, rejecting traversal attempts."""
base = self.base_dir.resolve()
candidate = (base / storage_path).resolve()
if base != candidate and base not in candidate.parents:
raise ValueError(f"Path escapes storage root: {storage_path}")
return candidate
async def save(self, user_id: str, filename: str, data: bytes) -> str:
safe_user = _sanitize_filename(user_id)
user_dir = self.base_dir / safe_user
user_dir.mkdir(parents=True, exist_ok=True)
storage_name = make_storage_filename(filename)
file_path = user_dir / storage_name
file_path.write_bytes(data)
return f"{safe_user}/{storage_name}"
async def load(self, storage_path: str) -> bytes:
file_path = self._resolve_safe_path(storage_path)
if not file_path.exists():
raise FileNotFoundError(f"File not found: {storage_path}")
return file_path.read_bytes()
async def delete(self, storage_path: str) -> None:
file_path = self._resolve_safe_path(storage_path)
if file_path.exists():
file_path.unlink()
def get_full_path(self, storage_path: str) -> Path | None:
"""Return absolute filesystem path for local files."""
try:
file_path = self._resolve_safe_path(storage_path)
except ValueError:
return None
return file_path if file_path.exists() else None
def get_file_storage() -> BaseFileStorage:
"""Factory: create file storage backend based on settings."""
media_dir = getattr(settings, "media_dir", "media")
return LocalFileStorage(base_dir=media_dir)