Implement MCP and Web Search

.github/workflows/pytest_and_coverage.yml

@@ -53,6 +53,9 @@ jobs:
       # GitHub settings
       PROMETHEUS_GITHUB_ACCESS_TOKEN: github_access_token
 
+      # Tavily API key
+      PROMETHEUS_TAVILY_API_KEY: tavily_api_key
+
       # DATABASE settings
       PROMETHEUS_DATABASE_URL: postgresql://postgres:password@localhost:5432/postgres?sslmode=disable
 

docker-compose.win_mac.yml

@@ -69,6 +69,9 @@ services:
       - PROMETHEUS_BASE_MODEL_MAX_OUTPUT_TOKENS=${PROMETHEUS_BASE_MODEL_MAX_OUTPUT_TOKENS}
       - PROMETHEUS_BASE_MODEL_TEMPERATURE=${PROMETHEUS_BASE_MODEL_TEMPERATURE}
 
+      # Tavily API key
+      - PROMETHEUS_TAVILY_API_KEY=${PROMETHEUS_TAVILY_API_KEY}
+
       # Database settings
       - PROMETHEUS_DATABASE_URL=${PROMETHEUS_DATABASE_URL}
 

docker-compose.yml

@@ -88,6 +88,9 @@ services:
       - PROMETHEUS_BASE_MODEL_MAX_OUTPUT_TOKENS=${PROMETHEUS_BASE_MODEL_MAX_OUTPUT_TOKENS}
       - PROMETHEUS_BASE_MODEL_TEMPERATURE=${PROMETHEUS_BASE_MODEL_TEMPERATURE}
 
+      # Tavily API key
+      - PROMETHEUS_TAVILY_API_KEY=${PROMETHEUS_TAVILY_API_KEY}
+
       # Database settings
       - PROMETHEUS_DATABASE_URL=${PROMETHEUS_DATABASE_URL}
 

example.env

@@ -38,6 +38,9 @@ PROMETHEUS_BASE_MODEL_MAX_INPUT_TOKENS=64000
 PROMETHEUS_BASE_MODEL_TEMPERATURE=0.3
 PROMETHEUS_BASE_MODEL_MAX_OUTPUT_TOKENS=15000
 
+# Tavily API settings
+PROMETHEUS_TAVILY_API_KEY=your_tavily_api_key
+
 # Database settings
 PROMETHEUS_DATABASE_URL=postgresql+asyncpg://postgres:password@postgres:5432/postgres
 

prometheus/app/api/routes/repository.py

@@ -177,7 +177,11 @@ async def list_repositories(request: Request):
     response_model=Response,
 )
 @requireLogin
-async def delete(repository_id: int, request: Request):
+async def delete(
+    repository_id: int,
+    request: Request,
+    force: bool = False,
+):
     knowledge_graph_service: KnowledgeGraphService = request.app.state.service[
         "knowledge_graph_service"
     ]
@@ -189,7 +193,7 @@ async def delete(repository_id: int, request: Request):
     if not repository:
         raise ServerException(code=404, message="Repository not found")
     # Check if the repository is being processed
-    if repository.is_working:
+    if repository.is_working and not force:
         raise ServerException(
             code=400, message="Repository is currently being processed, please try again later"
         )

prometheus/app/main.py

@@ -1,5 +1,4 @@
 import inspect
-import logging
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
 
@@ -16,27 +15,10 @@
     register_login_required_routes,
 )
 from prometheus.configuration.config import settings
+from prometheus.utils.logger_manager import get_thread_logger
 
-# Create a logger for the application's namespace
-logger = logging.getLogger("prometheus")
-formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-console_handler = logging.StreamHandler()
-console_handler.setFormatter(formatter)
-logger.addHandler(console_handler)
-logger.setLevel(getattr(logging, settings.LOGGING_LEVEL))
-logger.propagate = False
-
-# Log the configuration settings
-logger.info(f"LOGGING_LEVEL={settings.LOGGING_LEVEL}")
-logger.info(f"ENVIRONMENT={settings.ENVIRONMENT}")
-logger.info(f"BACKEND_CORS_ORIGINS={settings.BACKEND_CORS_ORIGINS}")
-logger.info(f"ADVANCED_MODEL={settings.ADVANCED_MODEL}")
-logger.info(f"BASE_MODEL={settings.BASE_MODEL}")
-logger.info(f"NEO4J_BATCH_SIZE={settings.NEO4J_BATCH_SIZE}")
-logger.info(f"WORKING_DIRECTORY={settings.WORKING_DIRECTORY}")
-logger.info(f"KNOWLEDGE_GRAPH_MAX_AST_DEPTH={settings.KNOWLEDGE_GRAPH_MAX_AST_DEPTH}")
-logger.info(f"KNOWLEDGE_GRAPH_CHUNK_SIZE={settings.KNOWLEDGE_GRAPH_CHUNK_SIZE}")
-logger.info(f"KNOWLEDGE_GRAPH_CHUNK_OVERLAP={settings.KNOWLEDGE_GRAPH_CHUNK_OVERLAP}")
+# Create main thread logger with file handler - ONE LINE!
+logger, file_handler = get_thread_logger(__name__)
 
 
 @asynccontextmanager

prometheus/app/services/database_service.py

@@ -1,15 +1,14 @@
-import logging
-
 from sqlalchemy.ext.asyncio import create_async_engine
 from sqlmodel import SQLModel
 
 from prometheus.app.services.base_service import BaseService
+from prometheus.utils.logger_manager import get_thread_logger
 
 
 class DatabaseService(BaseService):
     def __init__(self, DATABASE_URL: str):
         self.engine = create_async_engine(DATABASE_URL, echo=True)
-        self._logger = logging.getLogger("prometheus.app.services.database_service")
+        self._logger, file_handler = get_thread_logger(__name__)
 
     # Create the database and tables
     async def create_db_and_tables(self):

prometheus/app/services/invitation_code_service.py

@@ -1,5 +1,4 @@
 import datetime
-import logging
 import uuid
 from typing import Sequence
 
@@ -9,13 +8,14 @@
 from prometheus.app.entity.invitation_code import InvitationCode
 from prometheus.app.services.base_service import BaseService
 from prometheus.app.services.database_service import DatabaseService
+from prometheus.utils.logger_manager import get_thread_logger
 
 
 class InvitationCodeService(BaseService):
     def __init__(self, database_service: DatabaseService):
         self.database_service = database_service
         self.engine = database_service.engine
-        self._logger = logging.getLogger("prometheus.app.services.invitation_code_service")
+        self._logger, file_handler = get_thread_logger(__name__)
 
     async def create_invitation_code(self) -> InvitationCode:
         """

prometheus/app/services/issue_service.py

@@ -1,7 +1,4 @@
-import logging
-import threading
 import traceback
-from datetime import datetime
 from pathlib import Path
 from typing import Mapping, Optional, Sequence
 
@@ -13,6 +10,7 @@
 from prometheus.graph.knowledge_graph import KnowledgeGraph
 from prometheus.lang_graph.graphs.issue_graph import IssueGraph
 from prometheus.lang_graph.graphs.issue_state import IssueType
+from prometheus.utils.logger_manager import get_thread_logger, remove_multi_threads_log_file_handler
 
 
 class IssueService(BaseService):
@@ -79,15 +77,8 @@ def answer_issue(
                 - issue_type (IssueType): The type of the issue (BUG or QUESTION).
         """
 
-        # Set up a dedicated logger for this thread
-        logger = logging.getLogger(f"thread-{threading.get_ident()}.prometheus")
-        logger.setLevel(getattr(logging, self.logging_level))
-        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        log_file = self.answer_issue_log_dir / f"{timestamp}_{threading.get_ident()}.log"
-        file_handler = logging.FileHandler(log_file)
-        file_handler.setFormatter(formatter)
-        logger.addHandler(file_handler)
+        # Create thread-specific logger with file handler - ONE LINE!
+        logger, file_handler = get_thread_logger(__name__, force_new_file=True)
 
         # Construct the working directory
         if dockerfile_content or image_name:
@@ -141,5 +132,5 @@ def answer_issue(
             logger.error(f"Error in answer_issue: {str(e)}\n{traceback.format_exc()}")
             return None, False, False, False, None, None
         finally:
-            logger.removeHandler(file_handler)
-            file_handler.close()
+            # Remove multi-thread file handler
+            remove_multi_threads_log_file_handler(file_handler, logger.name)

prometheus/app/services/knowledge_graph_service.py

@@ -1,13 +1,13 @@
 """Service for managing and interacting with Knowledge Graphs in Neo4j."""
 
 import asyncio
-import logging
 from pathlib import Path
 
 from prometheus.app.services.base_service import BaseService
 from prometheus.app.services.neo4j_service import Neo4jService
 from prometheus.graph.knowledge_graph import KnowledgeGraph
 from prometheus.neo4j import knowledge_graph_handler
+from prometheus.utils.logger_manager import get_thread_logger
 
 
 class KnowledgeGraphService(BaseService):
@@ -42,7 +42,7 @@ def __init__(
         self.chunk_size = chunk_size
         self.chunk_overlap = chunk_overlap
         self.writing_lock = asyncio.Lock()
-        self._logger = logging.getLogger("prometheus.app.services.knowledge_graph_service")
+        self._logger, file_handler = get_thread_logger(__name__)
 
     async def start(self):
         # Initialize the Neo4j database for Knowledge Graph operations

prometheus/app/services/neo4j_service.py

@@ -1,15 +1,14 @@
 """Service for managing Neo4j database driver."""
 
-import logging
-
 from neo4j import AsyncGraphDatabase
 
 from prometheus.app.services.base_service import BaseService
+from prometheus.utils.logger_manager import get_thread_logger
 
 
 class Neo4jService(BaseService):
     def __init__(self, neo4j_uri: str, neo4j_username: str, neo4j_password: str):
-        self._logger = logging.getLogger("prometheus.app.services.neo4j_service")
+        self._logger, file_handler = get_thread_logger(__name__)
         self.neo4j_driver = AsyncGraphDatabase.driver(
             neo4j_uri,
             auth=(neo4j_username, neo4j_password),

prometheus/app/services/user_service.py

@@ -1,4 +1,3 @@
-import logging
 from typing import Optional, Sequence
 
 from argon2 import PasswordHasher
@@ -11,13 +10,14 @@
 from prometheus.app.services.database_service import DatabaseService
 from prometheus.exceptions.server_exception import ServerException
 from prometheus.utils.jwt_utils import JWTUtils
+from prometheus.utils.logger_manager import get_thread_logger
 
 
 class UserService(BaseService):
     def __init__(self, database_service: DatabaseService):
         self.database_service = database_service
         self.engine = database_service.engine
-        self._logger = logging.getLogger("prometheus.app.services.user_service")
+        self._logger, file_handler = get_thread_logger(__name__)
         self.ph = PasswordHasher()
         self.jwt_utils = JWTUtils()
 

prometheus/configuration/config.py

@@ -66,5 +66,8 @@ class Settings(BaseSettings):
     # Default normal user repository number
     DEFAULT_USER_REPOSITORY_LIMIT: int = 5
 
+    # tool for Websearch
+    TAVILY_API_KEY: str
+
 
 settings = Settings()

prometheus/docker/base_container.py

@@ -2,13 +2,14 @@
 import shutil
 import tarfile
 import tempfile
-import threading
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Optional, Sequence
 
 import docker
 
+from prometheus.utils.logger_manager import get_thread_logger
+
 
 class BaseContainer(ABC):
     """An abstract base class for managing Docker containers with file synchronization capabilities.
@@ -41,9 +42,7 @@ def __init__(
         Args:
           project_path: Path to the project directory to be containerized.
         """
-        self._logger = logging.getLogger(
-            f"thread-{threading.get_ident()}.{self.__class__.__module__}.{self.__class__.__name__}"
-        )
+        self._logger, file_handler = get_thread_logger(__name__)
         temp_dir = Path(tempfile.mkdtemp())
         temp_project_path = temp_dir / project_path.name
         shutil.copytree(project_path, temp_project_path)

prometheus/exceptions/web_search_tool_exception.py

@@ -0,0 +1,4 @@
+class WebSearchToolException(Exception):
+    """Custom exception for web search tool errors."""
+
+    pass

prometheus/git/git_repository.py

@@ -1,14 +1,15 @@
 """Git repository management module."""
 
 import asyncio
-import logging
 import shutil
 import tempfile
 from pathlib import Path
 from typing import Optional, Sequence
 
 from git import Git, GitCommandError, InvalidGitRepositoryError, Repo
 
+from prometheus.utils.logger_manager import get_thread_logger
+
 
 class GitRepository:
     """A class for managing Git repositories with support for both local and remote operations.
@@ -23,12 +24,12 @@ def __init__(self):
         """
         Initialize a GitRepository instance.
         """
-        self._logger = logging.getLogger("prometheus.git.git_repository")
+        self._logger, file_handler = get_thread_logger(__name__)
 
         # Configure git command to use our logger
         g = Git()
         type(g).GIT_PYTHON_TRACE = "full"
-        git_cmd_logger = logging.getLogger("git.cmd")
+        git_cmd_logger, file_handler = get_thread_logger("git.cmd")
 
         # Ensure git command output goes to our logger
         for handler in git_cmd_logger.handlers:

prometheus/graph/knowledge_graph.py

@@ -19,7 +19,6 @@
 
 import asyncio
 import itertools
-import logging
 from collections import defaultdict, deque
 from pathlib import Path
 from typing import Mapping, Optional, Sequence
@@ -43,6 +42,7 @@
     Neo4jTextNode,
     TextNode,
 )
+from prometheus.utils.logger_manager import get_thread_logger
 
 
 class KnowledgeGraph:
@@ -79,7 +79,7 @@ def __init__(
         self._next_node_id = root_node_id + len(self._knowledge_graph_nodes)
 
         self._file_graph_builder = FileGraphBuilder(max_ast_depth, chunk_size, chunk_overlap)
-        self._logger = logging.getLogger("prometheus.graph.knowledge_graph")
+        self._logger, file_handler = get_thread_logger(__name__)
 
     async def build_graph(self, root_dir: Path):
         """Asynchronously builds knowledge graph for a codebase at a location.

prometheus/lang_graph/nodes/bug_fix_verification_subgraph_node.py

@@ -1,13 +1,11 @@
-import logging
-import threading
-
 from langchain_core.language_models.chat_models import BaseChatModel
 from langgraph.errors import GraphRecursionError
 
 from prometheus.docker.base_container import BaseContainer
 from prometheus.git.git_repository import GitRepository
 from prometheus.lang_graph.subgraphs.bug_fix_verification_subgraph import BugFixVerificationSubgraph
 from prometheus.lang_graph.subgraphs.issue_verified_bug_state import IssueVerifiedBugState
+from prometheus.utils.logger_manager import get_thread_logger
 
 
 class BugFixVerificationSubgraphNode:
@@ -17,9 +15,7 @@ def __init__(
         container: BaseContainer,
         git_repo: GitRepository,
     ):
-        self._logger = logging.getLogger(
-            f"thread-{threading.get_ident()}.prometheus.lang_graph.nodes.bug_fix_verification_subgraph_node"
-        )
+        self._logger, file_handler = get_thread_logger(__name__)
         self.git_repo = git_repo
         self.subgraph = BugFixVerificationSubgraph(
             model=model,