mirror of
https://github.com/furyhawk/agent_alpha.git
synced 2026-07-21 18:45:34 +00:00
furyhawk
bb4b614711
- Added `vectorstore.py` for managing vector storage operations using Milvus. - Introduced `rag_document.py` for handling RAG document lifecycle, including ingestion and status updates. - Created `rag_status.py` for streaming RAG ingestion status via Redis pub/sub. - Developed `rag_sync.py` for managing synchronization operations and logs. - Implemented `sync_source.py` for managing sync source configurations and triggering syncs. - Established a lightweight task dispatcher in `dispatcher.py` for background task execution. - Registered task functions for RAG ingestion and sync operations in `rag_tasks.py`. - Configured task dispatcher settings in `arq_settings.py`. - Added necessary database models and schemas for RAG document and sync operations.
51 lines
1.7 KiB
Python
51 lines
1.7 KiB
Python
"""RAG document ORM model."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import Boolean, DateTime, Integer, String, Text
|
|
from sqlalchemy.dialects.postgresql import UUID
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from backend.core.models import Base
|
|
|
|
|
|
class RAGDocument(Base):
|
|
"""Tracks a document ingested into the RAG pipeline."""
|
|
|
|
__tablename__ = "rag_documents"
|
|
|
|
id: Mapped[uuid.UUID] = mapped_column(
|
|
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
|
)
|
|
collection_name: Mapped[str] = mapped_column(
|
|
String(255), nullable=False, default="documents"
|
|
)
|
|
filename: Mapped[str] = mapped_column(String(512), nullable=False)
|
|
filesize: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
filetype: Mapped[str] = mapped_column(String(50), nullable=False, default="")
|
|
status: Mapped[str] = mapped_column(
|
|
String(20), nullable=False, default="processing"
|
|
)
|
|
error_message: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
|
vector_document_id: Mapped[str | None] = mapped_column(
|
|
String(255), nullable=True, default=None
|
|
)
|
|
chunk_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
storage_path: Mapped[str] = mapped_column(
|
|
String(1024), nullable=False, default=""
|
|
)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True),
|
|
nullable=False,
|
|
default=lambda: datetime.now(timezone.utc),
|
|
)
|
|
completed_at: Mapped[datetime | None] = mapped_column(
|
|
DateTime(timezone=True), nullable=True, default=None
|
|
)
|
|
|
|
def __repr__(self) -> str:
|
|
return f"<RAGDocument {self.filename!r} status={self.status!r}>"
|