Merge branch 'main' into fix/benchmark-subprocess-leak-dead-code

Author: pengyuzhang

CONTRIBUTING.md

@@ -6,7 +6,7 @@ Thanks for taking the first step in contributing to our project.
 
 See the [Table of Contents](#table-of-contents) for different ways to contribute and details about how we treat each contribution. Please read the relevant section before making your contribution as it will not only make it a lot easier for us but also ensure you have the very best developer experience too.
 
-> ⭐ If you like the project, but don't have time to contribute just now, that's no problem at all! Give the Repo a star and we'll look forward to receiving your future contribution.
+> ⭐ If you like the project, but don't have time to contribute just now, that's no problem at all! Give the repo a star and we'll look forward to receiving your future contribution.
 
 ## Table of Contents
 
@@ -93,7 +93,7 @@ Bug reports shouldn't need the project maintainers to clarify or search for more
 
 - Make sure that you are using the latest version of the project.
 - **Determine if your bug is really a bug** and not an error on your side e.g. using incompatible environment components/versions.
-- To see if other users have experienced (and potentially already solved) the same issue you are having, **check if there is not already a bug report** existing for your bug or error in the [bug list](https://github.com/uber/ADR/issues?q=is%3Aissue%20state%3Aopen%20label%3Abug). If it has and the issue is still open, add a comment to the existing issue instead of opening a new one
+- To see if other users have experienced (and potentially already solved) the same issue you are having, **check if there is not already a bug report** existing for your bug or error in the [bug list](https://github.com/uber/ADR/issues?q=is%3Aissue%20state%3Aopen%20label%3Abug). If it has and the issue is still open, add a comment to the existing issue instead of opening a new one.
 - Collect information about the bug:
     - OS, Platform and Version (Windows, Linux, macOS, x86, ARM)
     - Version of the interpreter, compiler, SDK, runtime environment, package manager, depending on what seems relevant – for local instances only.
@@ -135,4 +135,4 @@ If you want to fix a bug or propose a new feature you'll do this through creatin
 - Provide a **short description of the solution you proposed** in as many details as possible.
 - **Use comments in the code** that you provide to give us more context to any code based submissions.
 
-Thanks for contributing into our project.
+Thanks for contributing to our project.

Detection/uv.lock

@@ -54,7 +54,7 @@ def __init__(self, output_dir: Optional[Path] = None, max_age_days: Optional[int
         )
         self.codex_parser = CodexParser()
         self.cline_parser = ClineParser()
-        self.warp_parser = WarpParser()
+        self.warp_parser = WarpParser(max_age_days=max_age_days) if max_age_days is not None else WarpParser()
 
         self.output_dir = output_dir if output_dir else Path("output")
         self.output_dir.mkdir(exist_ok=True)

Sensor/adr_sensor/observer.py

@@ -3,25 +3,31 @@
 Reads SQLite database from the Warp application data directory.
 
 Supports macOS path. Linux support can be added when Warp provides Linux paths.
+
+Performance-optimized: Skips conversations older than 2 weeks by default.
 """
 
 import json
 import sqlite3
 import traceback
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from ..schemas.agent_event_schema import AgentEvent, ChatMessage, ToolUsage
 from ..utils.timestamp_utils import normalize_timestamp
 from .base_parser import BaseParser
 
+MAX_CONVERSATION_AGE_DAYS = 14
+
 
 class WarpParser(BaseParser):
     """Parser for Warp Terminal SQLite database."""
 
-    def __init__(self):
+    def __init__(self, max_age_days: int = MAX_CONVERSATION_AGE_DAYS):
         self.base_path = Path.home() / "Library/Application Support/dev.warp.Warp-Stable"
         self.db_path = self.base_path / "warp.sqlite"
+        self.max_age_days = max_age_days
 
     def parse_all(self) -> List[AgentEvent]:
         """Parse all available Warp Terminal logs."""
@@ -42,7 +48,12 @@ def parse_all(self) -> List[AgentEvent]:
             conversations = self._get_all_conversations(conn)
             print(f"[WARP] Found {len(conversations)} conversations")
 
-            for conversation in conversations:
+            recent_conversations = self._filter_recent_conversations(conversations)
+            skipped_count = len(conversations) - len(recent_conversations)
+            if skipped_count > 0:
+                print(f"[WARP] Skipped {skipped_count} conversations older than {self.max_age_days} days")
+
+            for conversation in recent_conversations:
                 conversation_id = conversation["conversation_id"]
                 try:
                     exchanges = self._get_conversation_exchanges(conn, conversation_id)
@@ -61,15 +72,43 @@ def parse_all(self) -> List[AgentEvent]:
         return entries
 
     def _get_all_conversations(self, conn) -> List[Dict]:
-        """Get all conversations from the database."""
+        """Get all conversation ids and timestamps from the database.
+
+        Deliberately excludes the conversation_data column: it is not used
+        anywhere in this parser (exchange content comes from ai_queries /
+        ai_blocks instead), so fetching it here would be wasted I/O for
+        every conversation on every run.
+        """
         cursor = conn.cursor()
         cursor.execute("""
-            SELECT conversation_id, conversation_data, last_modified_at
+            SELECT conversation_id, last_modified_at
             FROM agent_conversations
             ORDER BY last_modified_at DESC
         """)
         return [dict(row) for row in cursor.fetchall()]
 
+    def _filter_recent_conversations(self, conversations: List[Dict]) -> List[Dict]:
+        """Filter out conversations whose last_modified_at is older than max_age_days.
+
+        A conversation with a missing or unparseable timestamp is kept rather than
+        dropped, since we can't determine its age.
+        """
+        cutoff_time = datetime.now(timezone.utc) - timedelta(days=self.max_age_days)
+        recent = []
+        for conversation in conversations:
+            last_modified = conversation.get("last_modified_at")
+            conv_timestamp = None
+            if last_modified is not None:
+                try:
+                    conv_timestamp = normalize_timestamp(last_modified)
+                except Exception:
+                    pass
+
+            if conv_timestamp is None or conv_timestamp >= cutoff_time:
+                recent.append(conversation)
+
+        return recent
+
     def _get_conversation_exchanges(self, conn, conversation_id: str) -> List[Dict]:
         """Get all exchanges for a conversation with LLM output."""
         cursor = conn.cursor()

Sensor/adr_sensor/parsers/warp_parser.py

@@ -1,8 +1,9 @@
 """Tests for ADR Sensor parsers."""
 
 import json
+import sqlite3
 import tempfile
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from unittest.mock import patch
 
@@ -11,6 +12,7 @@
 from adr_sensor.parsers.claude_parser import ClaudeParser
 from adr_sensor.parsers.cline_parser import ClineParser
 from adr_sensor.parsers.codex_parser import CodexParser
+from adr_sensor.parsers.warp_parser import WarpParser
 
 
 class TestClaudeParser:
@@ -211,3 +213,178 @@ def test_parse_no_directory(self):
         parser.base_path = Path("/nonexistent/path")
         entries = parser.parse_all()
         assert entries == []
+
+
+def _build_warp_db(db_path: Path, conversations: list) -> None:
+    """Create a synthetic warp.sqlite matching the schema WarpParser queries.
+
+    Each item in `conversations` is a dict with keys:
+        conversation_id, last_modified_at, exchanges
+    where `exchanges` is a list of (exchange_id, start_ts, input_json, llm_output_json).
+    """
+    conn = sqlite3.connect(str(db_path))
+    cursor = conn.cursor()
+    cursor.execute(
+        """
+        CREATE TABLE agent_conversations (
+            conversation_id TEXT PRIMARY KEY,
+            conversation_data TEXT,
+            last_modified_at TEXT
+        )
+        """
+    )
+    cursor.execute(
+        """
+        CREATE TABLE ai_queries (
+            exchange_id TEXT,
+            conversation_id TEXT,
+            start_ts TEXT,
+            input TEXT,
+            working_directory TEXT,
+            output_status TEXT,
+            model_id TEXT
+        )
+        """
+    )
+    cursor.execute("CREATE TABLE ai_blocks (exchange_id TEXT, output TEXT)")
+
+    for conv in conversations:
+        cursor.execute(
+            "INSERT INTO agent_conversations VALUES (?, ?, ?)",
+            (conv["conversation_id"], "unused_blob", conv["last_modified_at"]),
+        )
+        for exchange_id, start_ts, input_json, llm_output_json in conv.get("exchanges", []):
+            cursor.execute(
+                "INSERT INTO ai_queries VALUES (?, ?, ?, ?, ?, ?, ?)",
+                (exchange_id, conv["conversation_id"], start_ts, input_json, "/tmp/project", "success", "claude"),
+            )
+            if llm_output_json is not None:
+                cursor.execute("INSERT INTO ai_blocks VALUES (?, ?)", (exchange_id, llm_output_json))
+
+    conn.commit()
+    conn.close()
+
+
+class TestWarpParser:
+    def _make_parser(self, tmp_path: Path, max_age_days: int = 14) -> WarpParser:
+        parser = WarpParser(max_age_days=max_age_days)
+        parser.base_path = tmp_path
+        parser.db_path = tmp_path / "warp.sqlite"
+        return parser
+
+    def _query_exchange(self, exchange_id: str, timestamp: str) -> tuple:
+        input_json = json.dumps([{"Query": {"text": f"hello from {exchange_id}"}}])
+        llm_output_json = json.dumps({"Received": {"output": [{"Text": {"text": f"reply for {exchange_id}"}}]}})
+        return (exchange_id, timestamp, input_json, llm_output_json)
+
+    def test_skips_conversations_older_than_max_age(self, tmp_path):
+        """Old conversations should be filtered out before the expensive per-conversation query runs."""
+        now = datetime.now(timezone.utc)
+        recent_ts = now.isoformat()
+        old_ts = (now - timedelta(days=30)).isoformat()
+
+        conversations = [
+            {
+                "conversation_id": "recent-conv",
+                "last_modified_at": recent_ts,
+                "exchanges": [self._query_exchange("ex-recent", recent_ts)],
+            },
+            {
+                "conversation_id": "old-conv",
+                "last_modified_at": old_ts,
+                "exchanges": [self._query_exchange("ex-old", old_ts)],
+            },
+        ]
+        db_path = tmp_path / "warp.sqlite"
+        _build_warp_db(db_path, conversations)
+
+        parser = self._make_parser(tmp_path, max_age_days=14)
+        entries = parser.parse_all()
+
+        session_ids = {e.session_id for e in entries}
+        assert "warp_recent-conv" in session_ids
+        assert "warp_old-conv" not in session_ids
+
+    def test_all_history_via_large_max_age_days(self, tmp_path):
+        """A larger max_age_days should include conversations that would otherwise be skipped."""
+        now = datetime.now(timezone.utc)
+        old_ts = (now - timedelta(days=30)).isoformat()
+
+        conversations = [
+            {
+                "conversation_id": "old-conv",
+                "last_modified_at": old_ts,
+                "exchanges": [self._query_exchange("ex-old", old_ts)],
+            }
+        ]
+        db_path = tmp_path / "warp.sqlite"
+        _build_warp_db(db_path, conversations)
+
+        parser = self._make_parser(tmp_path, max_age_days=10000)
+        entries = parser.parse_all()
+
+        assert {e.session_id for e in entries} == {"warp_old-conv"}
+
+    def test_parses_conversation_content(self, tmp_path):
+        """Recent conversations should still be parsed into chat history correctly."""
+        now = datetime.now(timezone.utc)
+        ts1 = now.isoformat()
+        ts2 = (now + timedelta(seconds=1)).isoformat()
+
+        action_result_input = json.dumps(
+            [
+                {
+                    "ActionResult": {
+                        "id": "tool-1",
+                        "result": {
+                            "RequestCommandOutput": {
+                                "result": {"Success": {"command": "ls", "output": "file.txt", "exit_code": 0}}
+                            }
+                        },
+                    }
+                }
+            ]
+        )
+
+        conversations = [
+            {
+                "conversation_id": "conv-1",
+                "last_modified_at": ts2,
+                "exchanges": [
+                    self._query_exchange("ex-1", ts1),
+                    ("ex-2", ts2, action_result_input, None),
+                ],
+            }
+        ]
+        db_path = tmp_path / "warp.sqlite"
+        _build_warp_db(db_path, conversations)
+
+        parser = self._make_parser(tmp_path)
+        entries = parser.parse_all()
+
+        assert len(entries) == 1
+        entry = entries[0]
+        assert entry.source == "warp"
+        assert entry.session_id == "warp_conv-1"
+        assert len(entry.chat_history) == 2
+
+        user_msg = entry.chat_history[0]
+        assert user_msg.role == "user"
+        assert "hello from ex-1" in user_msg.content
+
+        assistant_msg = entry.chat_history[1]
+        assert assistant_msg.role == "assistant"
+        assert len(assistant_msg.tools) == 1
+        assert assistant_msg.tools[0].tool_name == "execute_command"
+        assert assistant_msg.tools[0].status == "success"
+
+    def test_parse_all_no_database(self, tmp_path):
+        """Test parse_all when the database file doesn't exist."""
+        parser = self._make_parser(tmp_path)
+        entries = parser.parse_all()
+        assert entries == []
+
+    def test_default_max_age_days(self):
+        """Constructor should default to the module-level constant when unset."""
+        parser = WarpParser()
+        assert parser.max_age_days == 14