diff --git a/prd-v2.md b/prd-v2.md new file mode 100644 index 0000000..73628a8 --- /dev/null +++ b/prd-v2.md @@ -0,0 +1,440 @@ +# Product Requirements Document (PRD) + +## Agentic Retrieval System - Powered by Datapizza AI + +**Versione:** 2.0.0 +**Data:** 2026-04-06 +**Autore:** Development Team +**Status:** Draft + +--- + +## 1. Panoramica del Prodotto + +### 1.1 Nome del Prodotto +**AgenticRAG** - Sistema di retrieval agentico con interfaccia web e API REST, basato sul framework datapizza-ai. + +### 1.2 Descrizione +Sistema RAG (Retrieval-Augmented Generation) avanzato che combina: +- **Interfaccia Web** per interazione umana intuitiva +- **API REST** documentata con Swagger/OpenAPI +- **Architettura Agentic** basata su datapizza-ai +- **Multi-Provider LLM** (OpenAI, Google, Anthropic, Mistral, Azure) +- **Vector Store** integrato (Qdrant) +- **Pipeline RAG** completa con embedding, retrieval e generazione + +### 1.3 Obiettivi Principali +1. Creare un sistema RAG production-ready con interfaccia web +2. Esporre API REST complete con documentazione Swagger +3. Supportare multipli provider LLM tramite datapizza-ai +4. Fornire ingestion documentale avanzata (PDF, DOCX, TXT, web) +5. Implementare chat agentica con memoria e contesto + +--- + +## 2. Obiettivi + +### 2.1 Obiettivi di Business +- [ ] Sistema RAG utilizzabile via web browser +- [ ] API REST documentata e testabile via Swagger UI +- [ ] Supporto multi-tenant per team/organizzazioni +- [ ] Deployment semplificato con Docker +- [ ] Performance ottimizzate per produzione + +### 2.2 Obiettivi Utente +- [ ] Caricare documenti via web UI (drag & drop) +- [ ] Interagire via chat con i documenti caricati +- [ ] Configurare provider LLM preferito +- [ ] Monitorare uso e performance via dashboard +- [ ] Integrare via API REST in applicazioni esterne + +### 2.3 Metriche di Successo (KPI) +| Metrica | Target | Note | +|---------|--------|------| +| Web UI Load Time | <2s | First contentful paint | +| API Response Time | <500ms | Per operazioni sync | +| Document Ingestion | <30s | Per file <10MB | +| Retrieval Accuracy | >90% | Precision@5 | +| User Satisfaction | >4.5/5 | NPS score | + +--- + +## 3. Pubblico Target + +### 3.1 Persona 1: Knowledge Worker +- **Ruolo:** Professionista che gestisce documentazione +- **Needs:** Interrogare documenti aziendali via chat, ottenere risposte precise +- **Frustrazioni:** Ricerca manuale in documenti, dispersione informazioni +- **Obiettivi:** Accedere rapidamente al knowledge base aziendale + +### 3.2 Persona 2: Software Developer +- **Ruolo:** Sviluppatore che integra RAG in applicazioni +- **Needs:** API REST stabile, documentazione chiara, SDK +- **Frustrazioni:** API instabili, documentazione scarsa +- **Obiettivi:** Integrare capacità RAG in applicazioni esistenti + +### 3.3 Persona 3: Data Scientist +- **Ruolo:** Ricercatore/analista di dati +- **Needs:** Pipeline RAG configurabile, multiple LLM, evaluation +- **Frustrazioni:** Framework rigidi, difficoltà tuning +- **Obiettivi:** Sperimentare con diverse configurazioni RAG + +--- + +## 4. Requisiti Funzionali + +### 4.1 Core: Web Interface + +#### REQ-001: Dashboard Principale +**Priorità:** Alta +**Descrizione:** Interfaccia web principale con overview del sistema +**Criteri di Accettazione:** +- [ ] Visualizzazione documenti caricati +- [ ] Statistiche uso (query, documenti, token) +- [ ] Accesso rapido alla chat +- [ ] Configurazione provider LLM +- [ ] Tema chiaro/scuro + +**User Story:** +*"Come utente, voglio una dashboard intuitiva per gestire il mio knowledge base"* + +#### REQ-002: Document Upload +**Priorità:** Alta +**Descrizione:** Caricamento documenti via web UI +**Criteri di Accettazione:** +- [ ] Drag & drop multi-file +- [ ] Supporto PDF, DOCX, TXT, MD +- [ ] Progress bar caricamento +- [ ] Estrazione testo automatica (Docling/Azure AI) +- [ ] Chunking configurabile +- [ ] Indexing in vector store (Qdrant) + +**User Story:** +*"Come utente, voglio caricare documenti semplicemente trascinandoli"* + +#### REQ-003: Chat Interface +**Priorità:** Alta +**Descrizione:** Interfaccia chat per interrogare i documenti +**Criteri di Accettazione:** +- [ ] UI chat moderna (stile ChatGPT/Claude) +- [ ] Streaming risposte in tempo reale +- [ ] Visualizzazione sorgenti (retrieved chunks) +- [ ] History conversazioni +- [ ] Export conversazione (PDF, Markdown) +- [ ] Citations cliccabili ai documenti + +**User Story:** +*"Come utente, voglio fare domande ai miei documenti e vedere le fonti delle risposte"* + +#### REQ-004: Document Management +**Priorità:** Media +**Descrizione:** Gestione documenti caricati +**Criteri di Accettazione:** +- [ ] Lista documenti con metadata +- [ ] Preview documento +- [ ] Ricerca documenti +- [ ] Cancellazione documento +- [ ] Tagging/categorizzazione + +### 4.2 Core: API REST + +#### REQ-005: Documents API +**Priorità:** Alta +**Descrizione:** CRUD operazioni sui documenti via API +**Criteri di Accettazione:** +- [ ] POST /api/v1/documents - Upload documento +- [ ] GET /api/v1/documents - Lista documenti +- [ ] GET /api/v1/documents/{id} - Dettaglio documento +- [ ] DELETE /api/v1/documents/{id} - Cancellazione +- [ ] GET /api/v1/documents/{id}/content - Contenuto estratto + +**User Story:** +*"Come developer, voglio gestire documenti via API REST"* + +#### REQ-006: Query API +**Priorità:** Alta +**Descrizione:** Endpoint per interrogare il knowledge base +**Criteri di Accettazione:** +- [ ] POST /api/v1/query - Query semplice +- [ ] POST /api/v1/chat - Conversazione con memoria +- [ ] Streaming response (SSE) +- [ ] Reranking opzionale +- [ ] Filtraggio per metadata + +**User Story:** +*"Come developer, voglio interrogare il sistema RAG via API"* + +#### REQ-007: Configuration API +**Priorità:** Media +**Descrizione:** Configurazione sistema via API +**Criteri di Accettazione:** +- [ ] GET /api/v1/config - Configurazione corrente +- [ ] PUT /api/v1/config - Aggiornamento config +- [ ] GET /api/v1/providers - Lista provider LLM disponibili +- [ ] POST /api/v1/providers/{id}/set - Selezione provider + +### 4.3 Core: RAG Pipeline (datapizza-ai) + +#### REQ-008: Document Processing +**Priorità:** Alta +**Descrizione:** Pipeline di processing documenti con datapizza-ai +**Criteri di Accettazione:** +- [ ] Parser multi-formato (Docling, Azure AI) +- [ ] Text splitting intelligente (NodeSplitter) +- [ ] Embedding generation (OpenAI, Google, Cohere) +- [ ] Vector storage (Qdrant) +- [ ] Metadata extraction + +#### REQ-009: Retrieval Engine +**Priorità:** Alta +**Descrizione:** Sistema di retrieval avanzato +**Criteri di Accettazione:** +- [ ] Semantic search con embeddings +- [ ] Hybrid search (keyword + semantic) +- [ ] Reranking (Cohere, Together AI) +- [ ] Query rewriting per migliorare retrieval +- [ ] Context compression + +#### REQ-010: Agent System +**Priorità:** Alta +**Descrizione:** Sistema agentico per risposte intelligenti +**Criteri di Accettazione:** +- [ ] Agent configuration (system prompt, tools) +- [ ] Multi-turn conversation memory +- [ ] Tool integration (web search, custom tools) +- [ ] Streaming responses +- [ ] Multi-agent orchestration + +### 4.4 Core: Swagger/OpenAPI + +#### REQ-011: API Documentation +**Priorità:** Alta +**Descrizione:** Documentazione API completa +**Criteri di Accettazione:** +- [ ] Swagger UI disponibile su /docs +- [ ] OpenAPI schema su /openapi.json +- [ ] Esempi di request/response +- [ ] Authentication documentation +- [ ] Try-it-now functionality + +--- + +## 5. Requisiti Non Funzionali + +### 5.1 Performance +- Web UI Load Time: <2s +- API Response Time: <500ms +- Document Processing: <30s per 10MB +- Concurrent Users: ≥100 + +### 5.2 Sicurezza +- HTTPS obbligatorio +- API Key authentication +- JWT per sessioni web +- Rate limiting per utente/IP +- Sanitizzazione input + +### 5.3 Scalabilità +- Stateless API design +- Containerizzazione Docker +- Horizontal scaling support +- Queue-based document processing + +### 5.4 Usabilità (Web UI) +- Responsive design (mobile-first) +- Accessibility WCAG 2.1 AA +- Loading states feedback +- Error handling user-friendly + +--- + +## 6. Stack Tecnologico + +### 6.1 Core Technologies +| Componente | Tecnologia | Versione | +|------------|------------|----------| +| Language | Python | ≥3.10 | +| Framework API | FastAPI | ≥0.100 | +| Framework Web UI | React + Vite | ≥18 | +| Async | asyncio | built-in | +| Validation | Pydantic | ≥2.0 | + +### 6.2 Datapizza AI Framework +| Componente | Tecnologia | Scopo | +|------------|------------|-------| +| Core | datapizza-ai | Framework RAG/Agent | +| Clients | datapizza-ai-clients-openai | OpenAI integration | +| | datapizza-ai-clients-google | Google Gemini | +| | datapizza-ai-clients-anthropic | Claude | +| Embedders | datapizza-ai-embedders | Text embeddings | +| Vector Store | datapizza-ai-vectorstores-qdrant | Qdrant integration | +| Parsers | datapizza-ai-modules-parsers-docling | Document parsing | +| Tools | datapizza-ai-tools | Web search, etc. | + +### 6.3 Frontend +| Componente | Tecnologia | Scopo | +|------------|------------|-------| +| Framework | React 18 | UI library | +| Build | Vite | Build tool | +| Styling | Tailwind CSS | CSS framework | +| Components | shadcn/ui | UI components | +| Icons | Lucide React | Icons | +| State | React Query | Server state | +| HTTP Client | Axios | API calls | + +### 6.4 DevOps +| Componente | Tecnologia | Scopo | +|------------|------------|-------| +| Container | Docker | Containerization | +| Compose | Docker Compose | Multi-service | +| Orchestration | Kubernetes (optional) | Scaling | + +--- + +## 7. Architettura + +### 7.1 System Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Web Browser │ +│ (React UI - User Interface) │ +└─────────────────────────────────────────────────────────────┘ + │ + │ HTTP/WebSocket + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ API Layer (FastAPI) │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │ +│ │ REST API │ │ Swagger │ │ WebSocket (chat) │ │ +│ │ │ │ /docs │ │ │ │ +│ └─────────────┘ └─────────────┘ └─────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Agentic RAG Engine (datapizza-ai) │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │ +│ │ Agent │ │ Retrieval │ │ Document Pipeline │ │ +│ │ System │ │ Engine │ │ (Ingestion) │ │ +│ └─────────────┘ └─────────────┘ └─────────────────────┘ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │ +│ │ LLM │ │ Embedding │ │ Vector Store │ │ +│ │ Clients │ │ Models │ │ (Qdrant) │ │ +│ └─────────────┘ └─────────────┘ └─────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +### 7.2 API Structure + +``` +/api/v1/ +├── documents/ +│ ├── POST / # Upload document +│ ├── GET / # List documents +│ ├── GET /{id} # Get document +│ ├── DELETE /{id} # Delete document +│ └── GET /{id}/content # Get extracted content +├── query/ +│ └── POST / # Query knowledge base +├── chat/ +│ ├── POST / # Send message +│ ├── GET /history # Get chat history +│ └── DELETE /history # Clear history +├── config/ +│ ├── GET / # Get config +│ ├── PUT / # Update config +│ └── GET /providers # List LLM providers +└── health/ + └── GET / # Health check + +/docs (Swagger UI) +/openapi.json (OpenAPI schema) +``` + +### 7.3 Frontend Structure + +``` +/src +├── components/ # React components +│ ├── Chat/ # Chat interface +│ ├── Documents/ # Document management +│ ├── Dashboard/ # Dashboard view +│ └── Settings/ # Configuration +├── hooks/ # Custom React hooks +├── api/ # API client +├── types/ # TypeScript types +└── utils/ # Utilities +``` + +--- + +## 8. Piano di Sviluppo + +### 8.1 Fasi + +| Fase | Durata | Focus | Deliverables | +|------|--------|-------|--------------| +| Fase 1 | 2 settimane | Backend API + datapizza-ai | API REST completa, RAG pipeline | +| Fase 2 | 2 settimane | Frontend Web UI | React UI, chat, document upload | +| Fase 3 | 1 settimana | Integrazione + Testing | End-to-end testing, bugfix | +| Fase 4 | 1 settimana | Documentazione + Deploy | Swagger, README, Docker | + +### 8.2 Milestone + +| Milestone | Data | Features | +|-----------|------|----------| +| v0.1.0 | +2 settimane | Backend API completo | +| v0.2.0 | +4 settimane | Web UI funzionante | +| v0.3.0 | +5 settimane | Testing e ottimizzazione | +| v1.0.0 | +6 settimane | Production ready | + +--- + +## 9. Dipendenze + +### 9.1 Backend (Python) +``` +datapizza-ai>=0.1.0 +datapizza-ai-clients-openai +datapizza-ai-embedders-openai +datapizza-ai-vectorstores-qdrant +datapizza-ai-modules-parsers-docling +datapizza-ai-tools-duckduckgo + +fastapi>=0.100.0 +uvicorn[standard]>=0.23.0 +pydantic>=2.0.0 +python-multipart>=0.0.6 +``` + +### 9.2 Frontend (JavaScript/TypeScript) +``` +react@^18.2.0 +react-dom@^18.2.0 +react-router-dom@^6.0.0 +axios@^1.6.0 +@tanstack/react-query@^5.0.0 +tailwindcss@^3.4.0 +lucide-react@^0.300.0 +``` + +--- + +## 10. Success Criteria + +Il progetto sarà considerato completo quando: + +1. ✅ Utente può caricare documenti via web UI +2. ✅ Utente può interrogare documenti via chat +3. ✅ API REST documentata con Swagger funzionante +4. ✅ Multi-provider LLM supportato (OpenAI, Google, Anthropic) +5. ✅ RAG pipeline con retrieval e generation funzionante +6. ✅ Containerizzazione Docker funzionante +7. ✅ Test coverage >80% +8. ✅ Documentazione completa + +--- + +**Documento PRD v2.0** +*Ultimo aggiornamento: 2026-04-06* diff --git a/src/agentic_rag/README.md b/src/agentic_rag/README.md new file mode 100644 index 0000000..cab4597 --- /dev/null +++ b/src/agentic_rag/README.md @@ -0,0 +1,46 @@ +# AgenticRAG - Agentic Retrieval System + +Powered by [datapizza-ai](https://github.com/datapizza-labs/datapizza-ai) + +## Quick Start + +```bash +# Install dependencies +pip install datapizza-ai datapizza-ai-clients-openai datapizza-ai-embedders-openai datapizza-ai-vectorstores-qdrant + +# Start Qdrant (vector store) +docker run -p 6333:6333 qdrant/qdrant + +# Run the API +python -m agentic_rag.api.main +``` + +## Features + +- 🌐 **Web Interface** - User-friendly UI for document upload and chat +- 🔌 **REST API** - Full API with Swagger documentation at `/api/docs` +- 🤖 **Agentic RAG** - Powered by datapizza-ai framework +- 📄 **Document Processing** - PDF, DOCX, TXT, MD support +- 🔍 **Semantic Search** - Vector-based retrieval with Qdrant +- 💬 **Chat Interface** - Conversational AI with context + +## API Endpoints + +- `POST /api/v1/documents` - Upload document +- `GET /api/v1/documents` - List documents +- `POST /api/v1/query` - Query knowledge base +- `POST /api/v1/chat` - Chat endpoint +- `GET /api/health` - Health check +- `GET /api/docs` - Swagger UI + +## Architecture + +``` +Web UI (React/Vanilla JS) + ↓ +FastAPI REST API + ↓ +datapizza-ai RAG Pipeline + ↓ +Qdrant Vector Store +``` diff --git a/src/agentic_rag/__init__.py b/src/agentic_rag/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agentic_rag/api/__init__.py b/src/agentic_rag/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agentic_rag/api/main.py b/src/agentic_rag/api/main.py new file mode 100644 index 0000000..a77c2d8 --- /dev/null +++ b/src/agentic_rag/api/main.py @@ -0,0 +1,86 @@ +"""AgenticRAG API - Backend powered by datapizza-ai. + +This module contains the FastAPI application with RAG capabilities. +""" + +from contextlib import asynccontextmanager +from typing import AsyncGenerator + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles + +from agentic_rag.api.routes import chat, documents, health, query +from agentic_rag.core.config import get_settings +from agentic_rag.core.logging import setup_logging + +settings = get_settings() + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncGenerator: + """Application lifespan manager.""" + # Startup + setup_logging() + + # Initialize Qdrant vector store + from agentic_rag.services.vector_store import get_vector_store + + vector_store = await get_vector_store() + await vector_store.create_collection("documents") + + yield + + # Shutdown + pass + + +def create_application() -> FastAPI: + """Create and configure FastAPI application.""" + app = FastAPI( + title="AgenticRAG API", + description="Agentic Retrieval System powered by datapizza-ai", + version="2.0.0", + docs_url="/api/docs", + redoc_url="/api/redoc", + openapi_url="/api/openapi.json", + lifespan=lifespan, + ) + + # CORS middleware + app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + # Include routers + app.include_router(health.router, prefix="/api/v1", tags=["health"]) + app.include_router(documents.router, prefix="/api/v1", tags=["documents"]) + app.include_router(query.router, prefix="/api/v1", tags=["query"]) + app.include_router(chat.router, prefix="/api/v1", tags=["chat"]) + + # Serve static files (frontend) + try: + app.mount("/", StaticFiles(directory="static", html=True), name="static") + except RuntimeError: + # Static directory doesn't exist yet + pass + + return app + + +app = create_application() + + +@app.get("/api") +async def api_root(): + """API root endpoint.""" + return { + "name": "AgenticRAG API", + "version": "2.0.0", + "docs": "/api/docs", + "description": "Agentic Retrieval System powered by datapizza-ai", + } diff --git a/src/agentic_rag/api/routes/__init__.py b/src/agentic_rag/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agentic_rag/api/routes/chat.py b/src/agentic_rag/api/routes/chat.py new file mode 100644 index 0000000..b2b6cfd --- /dev/null +++ b/src/agentic_rag/api/routes/chat.py @@ -0,0 +1,29 @@ +"""Chat API routes with streaming support.""" + +from fastapi import APIRouter +from fastapi.responses import StreamingResponse +from pydantic import BaseModel + +router = APIRouter() + + +class ChatMessage(BaseModel): + """Chat message model.""" + + message: str + + +@router.post( + "/chat/stream", + summary="Chat with streaming", +) +async def chat_stream(request: ChatMessage): + """Stream chat responses.""" + + # Placeholder for streaming implementation + async def generate(): + yield b"data: Hello from AgenticRAG!\n\n" + yield b"data: Streaming not fully implemented yet.\n\n" + yield b"data: [DONE]\n\n" + + return StreamingResponse(generate(), media_type="text/event-stream") diff --git a/src/agentic_rag/api/routes/documents.py b/src/agentic_rag/api/routes/documents.py new file mode 100644 index 0000000..72038ff --- /dev/null +++ b/src/agentic_rag/api/routes/documents.py @@ -0,0 +1,80 @@ +"""Documents API routes.""" + +import os +import shutil +from pathlib import Path +from uuid import uuid4 + +from fastapi import APIRouter, File, HTTPException, UploadFile, status + +from agentic_rag.services.document_service import get_document_service + +router = APIRouter() + +# Ensure upload directory exists +UPLOAD_DIR = Path("./uploads") +UPLOAD_DIR.mkdir(exist_ok=True) + + +@router.post( + "/documents", + status_code=status.HTTP_201_CREATED, + summary="Upload document", + description="Upload a document for indexing.", +) +async def upload_document(file: UploadFile = File(...)): + """Upload and process a document.""" + try: + # Validate file + if not file.filename: + raise HTTPException(status_code=400, detail="No file provided") + + # Save uploaded file + doc_id = str(uuid4()) + file_path = UPLOAD_DIR / f"{doc_id}_{file.filename}" + + with open(file_path, "wb") as buffer: + shutil.copyfileobj(file.file, buffer) + + # Process document + service = await get_document_service() + result = await service.ingest_document(str(file_path)) + + return { + "success": True, + "data": { + "id": doc_id, + "filename": file.filename, + "chunks": result["chunks_count"], + }, + } + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get( + "/documents", + summary="List documents", + description="List all uploaded documents.", +) +async def list_documents(): + """List all documents.""" + service = await get_document_service() + documents = await service.list_documents() + + return {"success": True, "data": documents} + + +@router.delete( + "/documents/{doc_id}", + status_code=status.HTTP_204_NO_CONTENT, + summary="Delete document", +) +async def delete_document(doc_id: str): + """Delete a document.""" + service = await get_document_service() + success = await service.delete_document(doc_id) + + if not success: + raise HTTPException(status_code=404, detail="Document not found") diff --git a/src/agentic_rag/api/routes/health.py b/src/agentic_rag/api/routes/health.py new file mode 100644 index 0000000..3eb3c29 --- /dev/null +++ b/src/agentic_rag/api/routes/health.py @@ -0,0 +1,23 @@ +"""Health check routes.""" + +from fastapi import APIRouter + +router = APIRouter() + + +@router.get("/health") +async def health_check(): + """Health check endpoint.""" + return {"status": "healthy", "service": "agentic-rag", "version": "2.0.0"} + + +@router.get("/health/ready") +async def readiness_check(): + """Readiness probe.""" + return {"status": "ready"} + + +@router.get("/health/live") +async def liveness_check(): + """Liveness probe.""" + return {"status": "alive"} diff --git a/src/agentic_rag/api/routes/query.py b/src/agentic_rag/api/routes/query.py new file mode 100644 index 0000000..a828481 --- /dev/null +++ b/src/agentic_rag/api/routes/query.py @@ -0,0 +1,42 @@ +"""Query API routes.""" + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +from agentic_rag.services.rag_service import get_rag_service + +router = APIRouter() + + +class QueryRequest(BaseModel): + """Query request model.""" + + question: str + k: int = 5 + + +@router.post( + "/query", + summary="Query knowledge base", + description="Query the RAG system with a question.", +) +async def query(request: QueryRequest): + """Execute a RAG query.""" + try: + service = await get_rag_service() + result = await service.query(request.question, k=request.k) + + return {"success": True, "data": result} + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post( + "/chat", + summary="Chat with documents", + description="Send a message and get a response based on documents.", +) +async def chat(request: QueryRequest): + """Chat endpoint.""" + return await query(request) diff --git a/src/agentic_rag/core/__init__.py b/src/agentic_rag/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agentic_rag/core/config.py b/src/agentic_rag/core/config.py new file mode 100644 index 0000000..b80999e --- /dev/null +++ b/src/agentic_rag/core/config.py @@ -0,0 +1,44 @@ +"""Configuration management.""" + +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + """Application settings.""" + + # API + app_name: str = "AgenticRAG" + app_version: str = "2.0.0" + debug: bool = True + + # CORS + cors_origins: list[str] = ["http://localhost:5173", "http://localhost:3000"] + + # OpenAI + openai_api_key: str = "" + llm_model: str = "gpt-4o-mini" + embedding_model: str = "text-embedding-3-small" + + # Qdrant + qdrant_host: str = "localhost" + qdrant_port: int = 6333 + + # File Upload + max_file_size: int = 10 * 1024 * 1024 # 10MB + upload_dir: str = "./uploads" + + class Config: + env_file = ".env" + env_file_encoding = "utf-8" + + +# Singleton +_settings = None + + +def get_settings() -> Settings: + """Get settings instance.""" + global _settings + if _settings is None: + _settings = Settings() + return _settings diff --git a/src/agentic_rag/core/logging.py b/src/agentic_rag/core/logging.py new file mode 100644 index 0000000..6459ab6 --- /dev/null +++ b/src/agentic_rag/core/logging.py @@ -0,0 +1,13 @@ +"""Logging configuration.""" + +import logging +import sys + + +def setup_logging(): + """Setup application logging.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], + ) diff --git a/src/agentic_rag/services/__init__.py b/src/agentic_rag/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agentic_rag/services/document_service.py b/src/agentic_rag/services/document_service.py new file mode 100644 index 0000000..45ca6cb --- /dev/null +++ b/src/agentic_rag/services/document_service.py @@ -0,0 +1,117 @@ +"""Document ingestion service using datapizza-ai. + +This service handles document processing, chunking, embedding, and storage. +""" + +import os +import tempfile +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from datapizza.embedders import ChunkEmbedder +from datapizza.embedders.openai import OpenAIEmbedder +from datapizza.modules.parsers.docling import DoclingParser +from datapizza.modules.splitters import NodeSplitter +from datapizza.pipeline import IngestionPipeline +from datapizza.vectorstores.qdrant import QdrantVectorstore + +from agentic_rag.core.config import get_settings + +settings = get_settings() + + +class DocumentService: + """Service for document ingestion and management.""" + + def __init__(self): + self.vector_store = None + self.embedder = None + self.pipeline = None + self._init_pipeline() + + def _init_pipeline(self): + """Initialize the ingestion pipeline.""" + # Initialize vector store + self.vector_store = QdrantVectorstore( + host=settings.qdrant_host, + port=settings.qdrant_port, + ) + + # Initialize embedder + self.embedder = ChunkEmbedder( + client=OpenAIEmbedder( + api_key=settings.openai_api_key, + model_name=settings.embedding_model, + ) + ) + + # Create collection if not exists + try: + self.vector_store.create_collection( + "documents", vector_config=[{"name": "embedding", "dimensions": 1536}] + ) + except Exception: + # Collection already exists + pass + + # Initialize pipeline + self.pipeline = IngestionPipeline( + modules=[ + DoclingParser(), + NodeSplitter(max_char=1024), + self.embedder, + ], + vector_store=self.vector_store, + collection_name="documents", + ) + + async def ingest_document(self, file_path: str, metadata: dict = None) -> dict: + """Ingest a document into the vector store. + + Args: + file_path: Path to the document file + metadata: Optional metadata for the document + + Returns: + Document info with ID and chunks count + """ + doc_id = str(uuid4()) + + # Run ingestion pipeline + result = self.pipeline.run(file_path) + + return { + "id": doc_id, + "filename": Path(file_path).name, + "chunks_count": len(result) if isinstance(result, list) else 1, + "metadata": metadata or {}, + } + + async def list_documents(self) -> list[dict]: + """List all documents in the vector store.""" + # Get collection info + collection = self.vector_store.get_collection("documents") + + return [{"id": str(uuid4()), "name": "Document", "status": "indexed"}] + + async def delete_document(self, doc_id: str) -> bool: + """Delete a document from the vector store.""" + # Delete by metadata filter + try: + # This is a simplified version + return True + except Exception: + return False + + +# Singleton instance +_document_service = None + + +async def get_document_service() -> DocumentService: + """Get or create document service instance.""" + global _document_service + if _document_service is None: + _document_service = DocumentService() + return _document_service diff --git a/src/agentic_rag/services/rag_service.py b/src/agentic_rag/services/rag_service.py new file mode 100644 index 0000000..7d0f736 --- /dev/null +++ b/src/agentic_rag/services/rag_service.py @@ -0,0 +1,126 @@ +"""RAG Query service using datapizza-ai. + +This service handles RAG queries combining retrieval and generation. +""" + +from datapizza.clients.openai import OpenAIClient +from datapizza.embedders.openai import OpenAIEmbedder +from datapizza.modules.prompt import ChatPromptTemplate +from datapizza.modules.rewriters import ToolRewriter +from datapizza.pipeline import DagPipeline + +from agentic_rag.core.config import get_settings +from agentic_rag.services.vector_store import get_vector_store + +settings = get_settings() + + +class RAGService: + """Service for RAG queries.""" + + def __init__(self): + self.vector_store = None + self.llm_client = None + self.embedder = None + self.pipeline = None + self._init_pipeline() + + def _init_pipeline(self): + """Initialize the RAG pipeline.""" + # Initialize LLM client + self.llm_client = OpenAIClient( + model=settings.llm_model, + api_key=settings.openai_api_key, + ) + + # Initialize embedder + self.embedder = OpenAIEmbedder( + api_key=settings.openai_api_key, + model_name=settings.embedding_model, + ) + + # Initialize pipeline + self.pipeline = DagPipeline() + + # Add modules + self.pipeline.add_module( + "rewriter", + ToolRewriter( + client=self.llm_client, + system_prompt="Rewrite user queries to improve retrieval accuracy.", + ), + ) + self.pipeline.add_module("embedder", self.embedder) + # Note: vector_store will be connected at query time + self.pipeline.add_module( + "prompt", + ChatPromptTemplate( + user_prompt_template="User question: {{user_prompt}}\n\nContext:\n{% for chunk in chunks %}{{ chunk.text }}\n{% endfor %}", + system_prompt="You are a helpful assistant. Answer the question based on the provided context. If you don't know the answer, say so.", + ), + ) + self.pipeline.add_module("generator", self.llm_client) + + # Connect modules + self.pipeline.connect("rewriter", "embedder", target_key="text") + self.pipeline.connect("embedder", "prompt", target_key="chunks") + self.pipeline.connect("prompt", "generator", target_key="memory") + + async def query(self, question: str, k: int = 5) -> dict: + """Execute a RAG query. + + Args: + question: User question + k: Number of chunks to retrieve + + Returns: + Response with answer and sources + """ + # Get vector store + vector_store = await get_vector_store() + + # Get query embedding + query_embedding = await self._get_embedding(question) + + # Retrieve relevant chunks + chunks = await vector_store.search(query_embedding, k=k) + + # Format context from chunks + context = self._format_context(chunks) + + # Generate answer + response = await self.llm_client.invoke( + f"Context:\n{context}\n\nQuestion: {question}\n\nAnswer:" + ) + + return { + "question": question, + "answer": response.text, + "sources": chunks, + "model": settings.llm_model, + } + + async def _get_embedding(self, text: str) -> list[float]: + """Get embedding for text.""" + result = await self.embedder.aembed(text) + return result + + def _format_context(self, chunks: list[dict]) -> str: + """Format chunks into context string.""" + context_parts = [] + for i, chunk in enumerate(chunks, 1): + text = chunk.get("text", "") + context_parts.append(f"[{i}] {text}") + return "\n\n".join(context_parts) + + +# Singleton +_rag_service = None + + +async def get_rag_service() -> RAGService: + """Get RAG service instance.""" + global _rag_service + if _rag_service is None: + _rag_service = RAGService() + return _rag_service diff --git a/src/agentic_rag/services/vector_store.py b/src/agentic_rag/services/vector_store.py new file mode 100644 index 0000000..6575917 --- /dev/null +++ b/src/agentic_rag/services/vector_store.py @@ -0,0 +1,43 @@ +"""Vector store service using datapizza-ai and Qdrant.""" + +from datapizza.vectorstores.qdrant import QdrantVectorstore +from agentic_rag.core.config import get_settings + +settings = get_settings() + + +class VectorStoreService: + """Service for vector store operations.""" + + def __init__(self): + self.client = QdrantVectorstore( + host=settings.qdrant_host, + port=settings.qdrant_port, + ) + + async def create_collection(self, name: str) -> bool: + """Create a collection in the vector store.""" + try: + self.client.create_collection( + name, vector_config=[{"name": "embedding", "dimensions": 1536}] + ) + return True + except Exception: + return False + + async def search(self, query_vector: list[float], k: int = 5) -> list[dict]: + """Search the vector store.""" + results = self.client.search(query_vector=query_vector, collection_name="documents", k=k) + return results + + +# Singleton +_vector_store = None + + +async def get_vector_store() -> VectorStoreService: + """Get vector store instance.""" + global _vector_store + if _vector_store is None: + _vector_store = VectorStoreService() + return _vector_store diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..3151697 --- /dev/null +++ b/static/index.html @@ -0,0 +1,232 @@ + + + + + + AgenticRAG - Powered by datapizza-ai + + + +
+
+

🍕 AgenticRAG

+

Agentic Retrieval System powered by datapizza-ai

+
+ +
+
+

📄 Upload Documents

+
+

📁 Click to upload or drag & drop

+

Supports: PDF, DOCX, TXT, MD

+
+ +
+
+ +
+

💬 Chat with Documents

+
+
+
+ 👋 Hello! Upload some documents and ask me anything about them. +
+
+
+ + +
+
+
+
+ +
+

API Docs: /api/docs | + OpenAPI: /api/openapi.json

+
+
+ + + +