diff --git a/api/nginx.conf b/api/nginx.conf index 57c9de7..26bb43c 100644 --- a/api/nginx.conf +++ b/api/nginx.conf @@ -61,7 +61,7 @@ server { add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type' always; - proxy_pass http://anonymization-service:8094; + proxy_pass http://anonymization-service:8094/api/v1/anonymization/; client_max_body_size 5m; proxy_http_version 1.1; proxy_set_header Host $host; @@ -91,6 +91,48 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } + # ─── Document/Conversation routes for GenAI Service ───────────────────── + location ^~ /documents/ { + if ($request_method = OPTIONS) { + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type' always; + return 204; + } + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type' always; + + proxy_pass http://genai-service:8000; + client_max_body_size 5m; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location ^~ /conversation/ { + if ($request_method = OPTIONS) { + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type' always; + return 204; + } + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, PUT, DELETE' always; + add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type' always; + + proxy_pass http://genai-service:8000; + # TODO: For local development, replace with http://localhost:8000 + client_max_body_size 1m; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + # ─── All other traffic → SPA ────────────────────────────────────────────── location / { proxy_pass http://client:80; diff --git a/client/src/api/chatApi.ts b/client/src/api/chatApi.ts new file mode 100644 index 0000000..eb93e76 --- /dev/null +++ b/client/src/api/chatApi.ts @@ -0,0 +1,11 @@ +import axios from 'axios'; + +const chatApi = axios.create({ + // TODO: For local development testing, replace VITE_API_URL with 'http://localhost:8000' + baseURL: import.meta.env.VITE_API_URL, + headers: { + 'Content-Type': 'application/json', + }, +}); + +export default chatApi; diff --git a/client/src/components/ChatWidget.tsx b/client/src/components/ChatWidget.tsx new file mode 100644 index 0000000..d67bc4b --- /dev/null +++ b/client/src/components/ChatWidget.tsx @@ -0,0 +1,262 @@ +import React, { useState, useRef } from 'react'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { + MessageCircle, + Minimize2, + Maximize2, + X, + Send, + Loader2, + Trash2, +} from 'lucide-react'; +import { useChat } from '@/hooks/useChat'; +import { ChatMessage } from '@/types/chat'; + +interface ChatWidgetProps { + documentContent?: string; + documentTitle?: string; + documentStatus?: 'original' | 'anonymized'; + isVisible: boolean; +} + +const ChatWidget: React.FC = ({ + documentContent, + documentTitle, + documentStatus = 'original', + isVisible, +}) => { + const [isOpen, setIsOpen] = useState(false); + const [isMinimized, setIsMinimized] = useState(false); + const [inputValue, setInputValue] = useState(''); + const inputRef = useRef(null); + + const { + messages, + isLoading, + sendMessage, + clearChat, + messagesEndRef, + isDocumentUploaded, + uploadDocument, + } = useChat(documentContent, documentTitle, documentStatus); + + const handleSendMessage = async (e: React.FormEvent) => { + e.preventDefault(); + if (!inputValue.trim() || isLoading) return; + + await sendMessage(inputValue); + setInputValue(''); + inputRef.current?.focus(); + }; + + const handleToggle = () => { + if (!isOpen) { + setIsOpen(true); + setIsMinimized(false); + // Upload document when chat is first opened + if (!isDocumentUploaded && documentContent) { + uploadDocument(); + } + } else { + setIsOpen(false); + } + }; + + const formatTimestamp = (timestamp?: Date) => { + if (!timestamp) return ''; + return new Intl.DateTimeFormat('en-US', { + hour: '2-digit', + minute: '2-digit', + }).format(timestamp); + }; + + const renderMessage = (message: ChatMessage, index: number) => ( +
+
+
{message.content}
+ {message.timestamp && ( +
+ {formatTimestamp(message.timestamp)} +
+ )} +
+
+ ); + + if (!isVisible) { + return null; + } + + return ( +
+ {/* Chat Toggle Button */} + {!isOpen && ( + + )} + + {/* Chat Window */} + {isOpen && ( + + + + Chat with Document + {!isDocumentUploaded && documentContent && ( + + Setting up document... + + )} + {isDocumentUploaded && ( + + {documentStatus === 'anonymized' + ? '🔒 Anonymized' + : '📄 Original'}{' '} + content + + )} + +
+ + + +
+ + + {!isMinimized && ( + + {/* Messages Area */} + + {messages.length === 0 ? ( +
+ +

+ Start chatting about your document! +

+ {documentContent && ( +

+ Document: {documentTitle || 'Untitled'} ( + {documentStatus}) +

+ )} +
+ ) : ( +
+ {messages.map((message, index) => + renderMessage(message, index) + )} + {isLoading && ( +
+
+ + + Thinking... + +
+
+ )} +
+
+ )} + + + {/* Input Area */} +
+
+ setInputValue(e.target.value)} + placeholder={ + isDocumentUploaded + ? 'Ask about your document...' + : 'Setting up document...' + } + disabled={isLoading || !isDocumentUploaded} + className="flex-1 text-sm" + /> + +
+
+ + )} + + )} +
+ ); +}; + +export default ChatWidget; diff --git a/client/src/components/DocumentArchive.tsx b/client/src/components/DocumentArchive.tsx index 8065157..7411258 100644 --- a/client/src/components/DocumentArchive.tsx +++ b/client/src/components/DocumentArchive.tsx @@ -1,39 +1,155 @@ -import React, { useState } from 'react'; +// ABOUTME: Document archive component that displays user's documents from backend API +// ABOUTME: Includes filtering, loading states, and proper error handling for document management +import { useState, useEffect } from 'react'; import { Link } from 'react-router-dom'; import { Button } from '@/components/ui/button'; import DocumentStatusFilter from '@/components/DocumentStatusFilter'; +import { fetchDocuments } from '@/services/documentService'; +import { fetchAnonymizations } from '@/services/anonymizeService'; +import { Document } from '@/types/document'; +import { ArchiveDocument } from '@/types/archiveDocument'; -// Mock data for demonstration -const mockDocuments = [ - { - id: 'doc1', - name: 'Financial Report 2024', - date: '2024-05-10', - status: 'Anonymized', - }, - { - id: 'doc2', - name: 'Employee Records', - date: '2024-05-08', - status: 'Original', - }, - { - id: 'doc3', - name: 'Contract Agreement', - date: '2024-05-05', - status: 'Summarized', - }, -]; - -type DocumentStatus = 'All' | 'Anonymized' | 'Original' | 'Summarized'; +import { useToast } from '@/hooks/use-toast'; + +type DocumentStatus = 'All' | 'Anonymized' | 'Original'; const DocumentArchive = () => { const [selectedStatus, setSelectedStatus] = useState('All'); + const [documents, setDocuments] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const { toast } = useToast(); + + useEffect(() => { + const loadDocuments = async () => { + try { + setLoading(true); + setError(null); + + // Fetch both original documents and anonymizations in parallel + const [originalDocs, anonymizations] = await Promise.all([ + fetchDocuments(), + fetchAnonymizations().catch((err) => { + console.warn('Failed to fetch anonymizations:', err); + // Show a warning toast but don't fail the entire operation + toast({ + title: 'Warning', + description: + 'Could not load anonymized documents. Showing original documents only.', + variant: 'default', + }); + return []; + }), + ]); + + // Convert original documents to archive format + const archiveOriginals: ArchiveDocument[] = originalDocs.map((doc) => ({ + id: doc.id, + fileName: doc.fileName, + uploadDate: doc.uploadDate, + status: 'Original' as const, + documentType: 'original' as const, + documentText: doc.documentText, + })); + + // Convert anonymizations to archive format + const archiveAnonymized: ArchiveDocument[] = anonymizations.map( + (anon) => ({ + id: anon.id, + fileName: `${getOriginalFileName( + originalDocs, + anon.documentId + )} (Anonymized)`, + uploadDate: anon.created, + status: 'Anonymized' as const, + documentType: 'anonymized' as const, + originalDocumentId: anon.documentId, + documentText: anon.anonymizedText, + }) + ); + + // Combine and sort by upload date (newest first) + const allDocuments = [...archiveOriginals, ...archiveAnonymized].sort( + (a, b) => + new Date(b.uploadDate).getTime() - new Date(a.uploadDate).getTime() + ); + + setDocuments(allDocuments); + } catch (err) { + const errorMessage = + err instanceof Error ? err.message : 'Failed to load documents'; + setError(errorMessage); + toast({ + title: 'Error', + description: errorMessage, + variant: 'destructive', + }); + } finally { + setLoading(false); + } + }; + + loadDocuments(); + }, [toast]); + + // Helper function to get original document filename for anonymized documents + const getOriginalFileName = ( + originalDocs: Document[], + documentId: string + ): string => { + const originalDoc = originalDocs.find((doc) => doc.id === documentId); + return originalDoc?.fileName || 'Unknown Document'; + }; + + const formatDate = (dateString: string) => { + try { + return new Date(dateString).toLocaleDateString(); + } catch { + return 'Invalid date'; + } + }; + + // Filter documents based on selected status const filteredDocuments = selectedStatus === 'All' - ? mockDocuments - : mockDocuments.filter((doc) => doc.status === selectedStatus); + ? documents + : documents.filter((doc) => doc.status === selectedStatus); + + if (loading) { + return ( +
+ +
+ {[1, 2, 3].map((i) => ( +
+
+
+
+
+
+
+
+ ))} +
+
+ ); + } + + if (error) { + return ( +
+

+ Error Loading Documents +

+

{error}

+ +
+ ); + } return (
@@ -49,26 +165,26 @@ const DocumentArchive = () => { key={doc.id} className="glass-panel p-6 rounded-xl transition-all duration-300 hover:scale-105 hover:shadow-lg hover:shadow-primary/10 hover:border-primary/20 cursor-pointer group" > -

- {doc.name} +

+ {doc.fileName}

- {doc.date} + {formatDate(doc.uploadDate)} {doc.status}
- +
@@ -82,7 +198,7 @@ const DocumentArchive = () => { ? "You haven't uploaded any documents yet." : `No ${selectedStatus.toLowerCase()} documents found.`}

- +
diff --git a/client/src/components/DocumentEditor.tsx b/client/src/components/DocumentEditor.tsx index 1d4fcab..5f72df9 100644 --- a/client/src/components/DocumentEditor.tsx +++ b/client/src/components/DocumentEditor.tsx @@ -1,20 +1,20 @@ -import React, { useState, useEffect } from 'react'; +import { useState, useEffect } from 'react'; import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; import { useAnonymization } from '@/hooks/useAnonymization'; import { useSummarization } from '@/hooks/useSummarization'; import AnonymizationPanel from './AnonymizationPanel'; import SummarizationPanel from './SummarizationPanel'; import EditAnonymizedDialog from './EditAnonymizedDialog'; +import ChatWidget from './ChatWidget'; import { DocumentContent } from '@/types/documentContent'; -import { toast } from "sonner"; - +import { Document } from '@/types/document'; +import { toast } from 'sonner'; type DocumentEditorProps = { - documentId?: string | null; + document?: Document | undefined; }; -const DocumentEditor = ({ documentId }: DocumentEditorProps) => { - +const DocumentEditor = ({ document }: DocumentEditorProps) => { const [activeTab, setActiveTab] = useState('anonymize'); const [documentData, setDocumentData] = useState(); const [selectedText, setSelectedText] = useState(''); @@ -28,7 +28,6 @@ const DocumentEditor = ({ documentId }: DocumentEditorProps) => { const [isAnonymized, setIsAnonymized] = useState(false); const [isSaved, setIsSaved] = useState(false); - // Custom hooks const { anonymizationLevel, @@ -39,7 +38,14 @@ const DocumentEditor = ({ documentId }: DocumentEditorProps) => { handleSaveEdit, handleDownload, handleSave, - } = useAnonymization(documentData, setDocumentData, isAnonymized, setIsAnonymized, isSaved, setIsSaved); + } = useAnonymization( + documentData, + setDocumentData, + isAnonymized, + setIsAnonymized, + isSaved, + setIsSaved + ); const { summarizationLevel, @@ -48,33 +54,41 @@ const DocumentEditor = ({ documentId }: DocumentEditorProps) => { handleSummarizationLevel, handleGenerateSummary, getSummarizationLevelDescription, - handleDownloadSummary + handleDownloadSummary, } = useSummarization(documentData, setDocumentData); useEffect(() => { if (documentData) { - console.log("Current documentData:", documentData); + console.log('Current documentData:', documentData); } }, [documentData]); - useEffect(() => { - console.log('Fetching document with ID:', documentId); + console.log('Fetching document with ID:', document); const documentInfo = sessionStorage.getItem('currentDocument'); - if (documentInfo) { - const parsedInfo = JSON.parse(documentInfo); + if (document) { const realContent: DocumentContent = { - title: (parsedInfo.fileName?.replace(/\.pdf$/i, "") || "Untitled"), - paragraph: parsedInfo.documentText || "", + title: document.fileName?.replace(/\.pdf$/i, '') || 'Untitled', + paragraph: document.documentText || '', sensitive: [], - summary: "" + summary: '', }; - - setDocumentData(realContent); + } else { + const documentInfo = sessionStorage.getItem('currentDocument'); + if (documentInfo) { + const parsedInfo = JSON.parse(documentInfo); + const realContent: DocumentContent = { + title: parsedInfo.fileName?.replace(/\.pdf$/i, '') || 'Untitled', + paragraph: parsedInfo.documentText || '', + sensitive: [], + summary: '', + }; + setDocumentData(realContent); + } } - }, [documentId]); + }, [document]); if (!documentData) { return
Loading document...
; @@ -119,6 +133,20 @@ const DocumentEditor = ({ documentId }: DocumentEditorProps) => { setCurrentEditItem(null); }; + // Get the current document content for the chat widget + const getCurrentDocumentContent = () => { + if (!documentData) return ''; + + // Return the current paragraph content (could be original or anonymized) + return documentData.paragraph; + }; + + const getDocumentStatus = () => { + if (isAnonymized || hasManualEdits) { + return 'anonymized'; + } + return 'original'; + }; return (
{ value={activeTab} onValueChange={(value) => { if (value === 'summarize' && !isAnonymized) { - toast.warning("Please anonymize your document before summarizing."); + toast.warning('Please anonymize your document before summarizing.'); return; } setActiveTab(value); @@ -136,7 +164,11 @@ const DocumentEditor = ({ documentId }: DocumentEditorProps) => { Anonymize Summarize @@ -181,6 +213,14 @@ const DocumentEditor = ({ documentId }: DocumentEditorProps) => { onSave={handleSaveEditInternal} onCancel={handleCancelEdit} /> + + {/* Chat Widget - available for any document content */} +
); }; diff --git a/client/src/components/DocumentStatusFilter.tsx b/client/src/components/DocumentStatusFilter.tsx index 28d8441..91d44c7 100644 --- a/client/src/components/DocumentStatusFilter.tsx +++ b/client/src/components/DocumentStatusFilter.tsx @@ -1,16 +1,18 @@ +import React from 'react'; +import { Button } from '@/components/ui/button'; -import React from "react"; -import { Button } from "@/components/ui/button"; - -type DocumentStatus = "All" | "Anonymized" | "Original" | "Summarized"; +type DocumentStatus = 'All' | 'Anonymized' | 'Original'; interface DocumentStatusFilterProps { selectedStatus: DocumentStatus; onStatusChange: (status: DocumentStatus) => void; } -const DocumentStatusFilter = ({ selectedStatus, onStatusChange }: DocumentStatusFilterProps) => { - const statuses: DocumentStatus[] = ["All", "Anonymized", "Original", "Summarized"]; +const DocumentStatusFilter = ({ + selectedStatus, + onStatusChange, +}: DocumentStatusFilterProps) => { + const statuses: DocumentStatus[] = ['All', 'Original', 'Anonymized']; return (
@@ -20,7 +22,7 @@ const DocumentStatusFilter = ({ selectedStatus, onStatusChange }: DocumentStatus {statuses.map((status) => (
diff --git a/client/src/pages/Index.tsx b/client/src/pages/Index.tsx index 02cbf0f..35304bb 100644 --- a/client/src/pages/Index.tsx +++ b/client/src/pages/Index.tsx @@ -70,7 +70,7 @@ const Index = () => { if (response.status == "PROCESSED") { toast.success("File uploaded successfully!"); sessionStorage.setItem("currentDocument", JSON.stringify(response)); - navigate(`/editor`); + navigate(`/editor`, { state: { document: response } }); setSelectedFile(null); if (fileInputRef.current) { fileInputRef.current.value = ""; diff --git a/client/src/services/anonymizeService.ts b/client/src/services/anonymizeService.ts index 69f8d97..8f02ae9 100644 --- a/client/src/services/anonymizeService.ts +++ b/client/src/services/anonymizeService.ts @@ -1,8 +1,8 @@ -import anonymizeApi from "@/api/anonymizeApi"; +import anonymizeApi from '@/api/anonymizeApi'; import type { AnonymizationRequestBody, AnonymizationDto, -} from "@/types/anonymize"; +} from '@/types/anonymize'; export async function saveAnonymization( documentId: string, @@ -20,15 +20,15 @@ export async function downloadAnonymizedPdf(anonymizationId: string) { responseType: 'blob', }); const url = window.URL.createObjectURL(response.data); - const link = document.createElement("a"); + const link = document.createElement('a'); link.href = url; - link.setAttribute("download", "anonymized_document.pdf"); + link.setAttribute('download', 'anonymized_document.pdf'); document.body.appendChild(link); link.click(); link.remove(); } export async function fetchAnonymizations(): Promise { - const { data } = await anonymizeApi.get("/"); + const { data } = await anonymizeApi.get('/'); return data; } diff --git a/client/src/services/chatService.ts b/client/src/services/chatService.ts new file mode 100644 index 0000000..e9e76d7 --- /dev/null +++ b/client/src/services/chatService.ts @@ -0,0 +1,42 @@ +import chatApi from '@/api/chatApi'; +import { + ConversationRequest, + ConversationResponse, + DocumentUploadRequest, + DocumentUploadResponse, + ChatMessage, +} from '@/types/chat'; + +export async function uploadTextDocument( + request: DocumentUploadRequest +): Promise { + const response = await chatApi.post( + '/documents/upload-text', + request + ); + return response.data; +} + +export async function chatWithDocuments( + request: ConversationRequest +): Promise { + const response = await chatApi.post( + '/conversation/chat', + request + ); + return response.data; +} + +export async function getConversationHistory( + conversationId: string +): Promise<{ conversation_id: string; history: ChatMessage[] }> { + const response = await chatApi.get(`/conversation/${conversationId}/history`); + return response.data; +} + +export async function clearConversation( + conversationId: string +): Promise<{ status: string; message: string }> { + const response = await chatApi.delete(`/conversation/${conversationId}`); + return response.data; +} diff --git a/client/src/services/documentService.ts b/client/src/services/documentService.ts index 0d2b85f..683b8fd 100644 --- a/client/src/services/documentService.ts +++ b/client/src/services/documentService.ts @@ -1,15 +1,18 @@ -import documentApi from "@/api/documentApi"; +// ABOUTME: Document service for API communication with backend document endpoints +// ABOUTME: Handles document upload and retrieval with proper typing and error handling +import documentApi from '@/api/documentApi'; +import { Document } from '@/types/document'; export async function uploadDocument(file: File): Promise { const form = new FormData(); - form.append("file", file); - const response = await documentApi.post("/upload", form, { - headers: { "Content-Type": "multipart/form-data" }, + form.append('file', file); + const response = await documentApi.post('/upload', form, { + headers: { 'Content-Type': 'multipart/form-data' }, }); return response.data; } export async function fetchDocuments(): Promise { - const response = await documentApi.get("/"); + const response = await documentApi.get('/'); return response.data; -} \ No newline at end of file +} diff --git a/client/src/types/anonymize.ts b/client/src/types/anonymize.ts index c9eb6a2..39be45c 100644 --- a/client/src/types/anonymize.ts +++ b/client/src/types/anonymize.ts @@ -11,12 +11,12 @@ export type AnonymizationRequestBody = { }; export type AnonymizationDto = { - id: string - created: string - documentId: string - userId: string - originalText: string - anonymizedText: string - anonymization_level: string - changedTerms: ChangedTerm[] -} \ No newline at end of file + id: string; + created: string; + documentId: string; + userId: string; + originalText: string; + anonymizedText: string; + anonymization_level: string; + changedTerms: ChangedTerm[]; +}; diff --git a/client/src/types/archiveDocument.ts b/client/src/types/archiveDocument.ts new file mode 100644 index 0000000..c7106a4 --- /dev/null +++ b/client/src/types/archiveDocument.ts @@ -0,0 +1,11 @@ +// ABOUTME: Unified document type for archive view combining original and anonymized documents +// ABOUTME: Provides consistent interface for displaying both document types in the archive +export interface ArchiveDocument { + id: string; + fileName: string; + uploadDate: string; + status: 'Original' | 'Anonymized'; + documentType: 'original' | 'anonymized'; + originalDocumentId?: string; // For anonymized docs, reference to original + documentText: string; +} diff --git a/client/src/types/chat.ts b/client/src/types/chat.ts new file mode 100644 index 0000000..175b861 --- /dev/null +++ b/client/src/types/chat.ts @@ -0,0 +1,35 @@ +export interface ChatMessage { + role: 'user' | 'assistant'; + content: string; + timestamp?: Date; +} + +export interface ConversationRequest { + conversation_id: string; + query: string; + document_ids?: string[]; +} + +export interface ConversationResponse { + response: string; + sources: Array<{ + filename: string; + chunk_index: number; + content: string; + }>; + conversation_id: string; + status: string; +} + +export interface DocumentUploadRequest { + content: string; + title?: string; + metadata?: Record; +} + +export interface DocumentUploadResponse { + document_id: string; + filename: string; + chunks_created: number; + status: string; +} diff --git a/client/src/types/document.ts b/client/src/types/document.ts index 0caa1ab..6ab9627 100644 --- a/client/src/types/document.ts +++ b/client/src/types/document.ts @@ -1,8 +1,11 @@ -import { UUID } from "crypto"; - +// ABOUTME: Document type definition matching backend DocumentDto structure +// ABOUTME: Includes all fields returned by the document service API export interface Document { - id: string, - userId : UUID - fileName: string, - status: string + id: string; + userId: string; + fileName: string; + fileUrl: string; + status: string; + uploadDate: string; // ISO string from backend Instant + documentText: string; } diff --git a/genai-service/conversation_chain.py b/genai-service/conversation_chain.py index abd87ca..59bd8a0 100644 --- a/genai-service/conversation_chain.py +++ b/genai-service/conversation_chain.py @@ -19,151 +19,125 @@ class ConversationState(TypedDict): document_ids: Optional[List[str]] query: str response: Optional[str] - sources: Optional[List[Dict]] class ConversationManager: def __init__(self, vector_store: VectorStoreManager): self.vector_store = vector_store - self.llm = init_chat_model("openai:gpt-4-0125-preview", temperature=0.7) + self.llm = init_chat_model( + "openai:gpt-4-0125-preview", temperature=0.7 + ) self.conversations: Dict[str, List[Dict]] = {} - - self.rag_prompt = ChatPromptTemplate.from_messages([ - ("system", """You are a helpful assistant with access to documents. - Use the provided context to answer questions accurately. When you use information from the context, - mention which document it came from by referencing the filename. - - Context from documents: - {context} + + self.rag_prompt = ChatPromptTemplate.from_messages( + [ + ( + "system", + """You are a helpful assistant with access to document content. + Use the provided context to answer questions accurately and naturally. + Provide clear, helpful responses based on the document information without mentioning + technical details like chunks or metadata. - Remember to cite your sources when using document information."""), - MessagesPlaceholder(variable_name="chat_history"), - ("human", "{query}") - ]) - + Document content: + {context}""", + ), + MessagesPlaceholder(variable_name="chat_history"), + ("human", "{query}"), + ] + ) + self.graph = self._build_graph() - + def _build_graph(self) -> StateGraph: graph_builder = StateGraph(ConversationState) - + graph_builder.add_node("retrieve_context", self._retrieve_context) graph_builder.add_node("generate_response", self._generate_response) - graph_builder.add_node("extract_sources", self._extract_sources) - + graph_builder.add_edge(START, "retrieve_context") graph_builder.add_edge("retrieve_context", "generate_response") - graph_builder.add_edge("generate_response", "extract_sources") - graph_builder.add_edge("extract_sources", END) - + graph_builder.add_edge("generate_response", END) + return graph_builder.compile() - + def _retrieve_context(self, state: ConversationState) -> Dict: query = state["query"] document_ids = state.get("document_ids") - + relevant_docs = self.vector_store.search_documents( - query=query, - k=5, - document_ids=document_ids + query=query, k=5, document_ids=document_ids ) - + + # Simply concatenate document content without chunk information context_parts = [] - sources = [] - + for doc in relevant_docs: - filename = doc.metadata.get("filename", "Unknown") - chunk_index = doc.metadata.get("chunk_index", 0) - document_id = doc.metadata.get("document_id", "") - - context_parts.append(f"[From {filename}, chunk {chunk_index}]:\n{doc.page_content}") - - sources.append({ - "document_id": document_id, - "filename": filename, - "chunk_index": chunk_index, - "content": doc.page_content[:200] + "..." - }) - + # Clean content without metadata references + context_parts.append(doc.page_content) + context = "\n\n".join(context_parts) - - return {"context": context, "sources": sources} - + + return {"context": context} + def _generate_response(self, state: ConversationState) -> Dict: messages = state["messages"] context = state.get("context", "") query = state["query"] - + chat_history = [] for msg in messages[:-1]: if msg["role"] == "user": chat_history.append(HumanMessage(content=msg["content"])) elif msg["role"] == "assistant": chat_history.append(AIMessage(content=msg["content"])) - + chain = self.rag_prompt | self.llm | StrOutputParser() - - response = chain.invoke({ - "context": context, - "chat_history": chat_history, - "query": query - }) - + + response = chain.invoke( + {"context": context, "chat_history": chat_history, "query": query} + ) + return {"response": response} - - def _extract_sources(self, state: ConversationState) -> Dict: - response = state.get("response", "") - sources = state.get("sources", []) - - referenced_files = set() - for source in sources: - filename = source.get("filename", "") - if filename and filename in response: - referenced_files.add(filename) - - filtered_sources = [ - source for source in sources - if source.get("filename") in referenced_files - ] - - return {"sources": filtered_sources} - - def chat(self, - conversation_id: str, - query: str, - document_ids: Optional[List[str]] = None) -> Dict: - + + def chat( + self, + conversation_id: str, + query: str, + document_ids: Optional[List[str]] = None, + ) -> Dict: + if conversation_id not in self.conversations: self.conversations[conversation_id] = [] - + messages = self.conversations[conversation_id].copy() messages.append({"role": "user", "content": query}) - - result = self.graph.invoke({ - "messages": messages, - "query": query, - "document_ids": document_ids, - "context": None, - "response": None, - "sources": None - }) - + + result = self.graph.invoke( + { + "messages": messages, + "query": query, + "document_ids": document_ids, + "context": None, + "response": None, + } + ) + response = result.get("response", "") - sources = result.get("sources", []) - + messages.append({"role": "assistant", "content": response}) self.conversations[conversation_id] = messages - + return { "response": response, - "sources": sources, - "conversation_id": conversation_id + "sources": [], # Empty sources array to maintain API compatibility + "conversation_id": conversation_id, } - + def get_conversation_history(self, conversation_id: str) -> List[Dict]: return self.conversations.get(conversation_id, []) - + def clear_conversation(self, conversation_id: str) -> bool: if conversation_id in self.conversations: del self.conversations[conversation_id] return True - return False \ No newline at end of file + return False diff --git a/genai-service/main.py b/genai-service/main.py index d9ddd8b..2aa09b9 100644 --- a/genai-service/main.py +++ b/genai-service/main.py @@ -8,6 +8,7 @@ ChatResponse, DocumentUploadResponse, MarkdownUploadRequest, + TextUploadRequest, ConversationRequest, ConversationResponse, DocumentListResponse, @@ -37,6 +38,8 @@ # Instrument the app and expose the metrics instrumentator.instrument(app).expose(app) + + # Add error handlers @app.exception_handler(RuntimeError) async def runtime_error_handler(request, exc): @@ -82,13 +85,11 @@ def anonymize(request: AnonymizeRequest): changed_terms = result["changed_terms"] anonymized_text = call_anonymization_service( - original_text=request.originalText, - changed_terms=changed_terms + original_text=request.originalText, changed_terms=changed_terms ) return GenAiResponse( - responseText=anonymized_text, - changedTerms=changed_terms + responseText=anonymized_text, changedTerms=changed_terms ) except RuntimeError as e: @@ -99,7 +100,6 @@ def anonymize(request: AnonymizeRequest): raise HTTPException(status_code=500, detail="Internal server error") - def call_anonymization_service( original_text: str, changed_terms: list[dict] ) -> str: @@ -126,6 +126,7 @@ def summarize(request: SummarizeRequest): ) return GenAiResponse(responseText=result["summarized_text"]) + @app.post("/api/v1/genai/chat", response_model=ChatResponse) def chat(request: ChatRequest): messages = [] @@ -189,6 +190,25 @@ def upload_markdown(request: MarkdownUploadRequest): raise HTTPException(status_code=500, detail=str(e)) +# New endpoint for uploading text content +@app.post("/documents/upload-text", response_model=DocumentUploadResponse) +def upload_text(request: TextUploadRequest): + try: + document_id, chunks = vector_store.ingest_text( + content=request.content, + title=request.title, + metadata=request.metadata, + ) + + return DocumentUploadResponse( + document_id=document_id, + filename=request.title, + chunks_created=chunks, + ) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + @app.get("/documents", response_model=DocumentListResponse) def list_documents(): documents = vector_store.list_documents() diff --git a/genai-service/models.py b/genai-service/models.py index 23ccc87..0d04ff4 100644 --- a/genai-service/models.py +++ b/genai-service/models.py @@ -12,18 +12,22 @@ class GenAiResponse(BaseModel): responseText: str changedTerms: Optional[List[ChangedTerm]] = None + class SummarizeRequest(BaseModel): originalText: str level: Literal["short", "medium", "long"] + class ChatMessage(BaseModel): role: Literal["user", "assistant"] content: str + class ChatRequest(BaseModel): messages: List[ChatMessage] document: Optional[str] = None + class ChatResponse(BaseModel): reply: str status: str = "success" @@ -41,6 +45,13 @@ class MarkdownUploadRequest(BaseModel): content: str +# New model for text upload +class TextUploadRequest(BaseModel): + content: str + title: Optional[str] = "Untitled Text" + metadata: Optional[Dict[str, Any]] = None + + class ConversationRequest(BaseModel): conversation_id: str query: str @@ -57,4 +68,4 @@ class ConversationResponse(BaseModel): class DocumentListResponse(BaseModel): documents: List[Dict[str, Any]] total: int - status: str = "success" \ No newline at end of file + status: str = "success" diff --git a/genai-service/requirements.txt b/genai-service/requirements.txt index 3b37162..d79b468 100644 --- a/genai-service/requirements.txt +++ b/genai-service/requirements.txt @@ -18,4 +18,4 @@ httpx==0.28.1 pytest-cov==6.2.1 faker==37.4.0 reportlab==4.4.2 -prometheus-fastapi-instrumentator \ No newline at end of file +prometheus-fastapi-instrumentator==7.1.0 \ No newline at end of file diff --git a/genai-service/vector_store.py b/genai-service/vector_store.py index 7bfc048..08d46a8 100644 --- a/genai-service/vector_store.py +++ b/genai-service/vector_store.py @@ -18,152 +18,224 @@ class VectorStoreManager: def __init__(self, persist_directory: str = "./chroma_db"): self.persist_directory = persist_directory os.makedirs(persist_directory, exist_ok=True) - + self.embeddings = OpenAIEmbeddings(model="text-embedding-3-small") - + self.client = chromadb.PersistentClient( path=persist_directory, - settings=Settings( - anonymized_telemetry=False, - allow_reset=True - ) + settings=Settings(anonymized_telemetry=False, allow_reset=True), ) - + self.collection_name = "documents" self.vectorstore = Chroma( client=self.client, collection_name=self.collection_name, embedding_function=self.embeddings, - persist_directory=persist_directory + persist_directory=persist_directory, ) - + self.text_splitter = RecursiveCharacterTextSplitter( chunk_size=1000, chunk_overlap=200, length_function=len, - separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""] + separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""], ) - + self.document_metadata: Dict[str, Dict] = {} - + def generate_document_id(self, filename: str, content: bytes) -> str: content_hash = hashlib.md5(content).hexdigest()[:8] return f"{filename}_{content_hash}" - - def ingest_pdf(self, file_path: str, filename: str, document_id: Optional[str] = None) -> Tuple[str, int]: + + def ingest_pdf( + self, file_path: str, filename: str, document_id: Optional[str] = None + ) -> Tuple[str, int]: if not document_id: - with open(file_path, 'rb') as f: + with open(file_path, "rb") as f: content = f.read() document_id = self.generate_document_id(filename, content) - + loader = PyPDFLoader(file_path) documents = loader.load() - + if not documents: - raise ValueError(f"No content could be extracted from PDF: {filename}") - + raise ValueError( + f"No content could be extracted from PDF: {filename}" + ) + # Filter out empty documents documents = [doc for doc in documents if doc.page_content.strip()] - + if not documents: raise ValueError(f"PDF contains no readable text: {filename}") - + chunks = self.text_splitter.split_documents(documents) - + # Filter out empty chunks chunks = [chunk for chunk in chunks if chunk.page_content.strip()] - + if not chunks: - raise ValueError(f"No text chunks could be created from PDF: {filename}") - + raise ValueError( + f"No text chunks could be created from PDF: {filename}" + ) + for i, chunk in enumerate(chunks): - chunk.metadata.update({ - "document_id": document_id, - "filename": filename, - "chunk_index": i, - "total_chunks": len(chunks), - "upload_time": datetime.utcnow().isoformat(), - "file_type": "pdf" - }) - + chunk.metadata.update( + { + "document_id": document_id, + "filename": filename, + "chunk_index": i, + "total_chunks": len(chunks), + "upload_time": datetime.utcnow().isoformat(), + "file_type": "pdf", + } + ) + self.vectorstore.add_documents(chunks) - + self.document_metadata[document_id] = { "filename": filename, "file_type": "pdf", "total_chunks": len(chunks), - "upload_time": datetime.utcnow().isoformat() + "upload_time": datetime.utcnow().isoformat(), } - + return document_id, len(chunks) - - def ingest_markdown(self, content: str, filename: str, document_id: Optional[str] = None) -> Tuple[str, int]: + + def ingest_markdown( + self, content: str, filename: str, document_id: Optional[str] = None + ) -> Tuple[str, int]: if not content.strip(): raise ValueError(f"Markdown content is empty: {filename}") - + if not document_id: document_id = self.generate_document_id(filename, content.encode()) - + doc = Document(page_content=content, metadata={"source": filename}) chunks = self.text_splitter.split_documents([doc]) - + # Filter out empty chunks chunks = [chunk for chunk in chunks if chunk.page_content.strip()] - + if not chunks: - raise ValueError(f"No text chunks could be created from markdown: {filename}") - + raise ValueError( + f"No text chunks could be created from markdown: {filename}" + ) + for i, chunk in enumerate(chunks): - chunk.metadata.update({ - "document_id": document_id, - "filename": filename, - "chunk_index": i, - "total_chunks": len(chunks), - "upload_time": datetime.utcnow().isoformat(), - "file_type": "markdown" - }) - + chunk.metadata.update( + { + "document_id": document_id, + "filename": filename, + "chunk_index": i, + "total_chunks": len(chunks), + "upload_time": datetime.utcnow().isoformat(), + "file_type": "markdown", + } + ) + self.vectorstore.add_documents(chunks) - + self.document_metadata[document_id] = { "filename": filename, "file_type": "markdown", "total_chunks": len(chunks), - "upload_time": datetime.utcnow().isoformat() + "upload_time": datetime.utcnow().isoformat(), } - + return document_id, len(chunks) - - def search_documents(self, query: str, k: int = 5, document_ids: Optional[List[str]] = None) -> List[Document]: + + # New method for ingesting plain text + def ingest_text( + self, + content: str, + title: str = "Untitled Text", + document_id: Optional[str] = None, + metadata: Optional[Dict] = None, + ) -> Tuple[str, int]: + if not content.strip(): + raise ValueError(f"Text content is empty: {title}") + + if not document_id: + document_id = self.generate_document_id(title, content.encode()) + + # Create document with additional metadata + doc_metadata = {"source": title} + if metadata: + doc_metadata.update(metadata) + + doc = Document(page_content=content, metadata=doc_metadata) + chunks = self.text_splitter.split_documents([doc]) + + # Filter out empty chunks + chunks = [chunk for chunk in chunks if chunk.page_content.strip()] + + if not chunks: + raise ValueError( + f"No text chunks could be created from text: {title}" + ) + + for i, chunk in enumerate(chunks): + chunk.metadata.update( + { + "document_id": document_id, + "filename": title, + "chunk_index": i, + "total_chunks": len(chunks), + "upload_time": datetime.utcnow().isoformat(), + "file_type": "text", + } + ) + # Add any additional metadata + if metadata: + chunk.metadata.update(metadata) + + self.vectorstore.add_documents(chunks) + + self.document_metadata[document_id] = { + "filename": title, + "file_type": "text", + "total_chunks": len(chunks), + "upload_time": datetime.utcnow().isoformat(), + **(metadata or {}), + } + + return document_id, len(chunks) + + def search_documents( + self, query: str, k: int = 5, document_ids: Optional[List[str]] = None + ) -> List[Document]: search_kwargs = {"k": k} - + if document_ids: search_kwargs["filter"] = {"document_id": {"$in": document_ids}} - + return self.vectorstore.similarity_search(query, **search_kwargs) - - def get_retriever(self, k: int = 5, document_ids: Optional[List[str]] = None): + + def get_retriever( + self, k: int = 5, document_ids: Optional[List[str]] = None + ): search_kwargs = {"k": k} - + if document_ids: search_kwargs["filter"] = {"document_id": {"$in": document_ids}} - + return self.vectorstore.as_retriever(search_kwargs=search_kwargs) - + def list_documents(self) -> List[Dict]: return [ {"document_id": doc_id, **metadata} for doc_id, metadata in self.document_metadata.items() ] - + def delete_document(self, document_id: str) -> bool: try: collection = self.client.get_collection(self.collection_name) collection.delete(where={"document_id": document_id}) - + if document_id in self.document_metadata: del self.document_metadata[document_id] - + return True except Exception: - return False \ No newline at end of file + return False diff --git a/server/anonymization-service/src/main/java/oopsops/app/anonymization/controller/AnonymizationController.java b/server/anonymization-service/src/main/java/oopsops/app/anonymization/controller/AnonymizationController.java index 3ba36ca..617b474 100644 --- a/server/anonymization-service/src/main/java/oopsops/app/anonymization/controller/AnonymizationController.java +++ b/server/anonymization-service/src/main/java/oopsops/app/anonymization/controller/AnonymizationController.java @@ -38,7 +38,7 @@ public AnonymizationController(AnonymizationService anonymizationService) { this.anonymizationService = anonymizationService; } - @GetMapping() + @GetMapping("/") public ResponseEntity> getAllAnonymizations(@AuthenticationPrincipal Jwt jwt) { UUID userId = UUID.fromString(jwt.getSubject()); List anonymizationDtos = anonymizationService.getAllAnonymizations(userId).stream()