Skip to content

Commit 83bda7c

Browse files
committed
fix(sensor): attribute MCP servers and failed tool calls in Claude Code sessions
The Claude Code parser hardcoded `tool_type="tool_use"` for every tool call, never populated `server_name`, and dropped the `is_error` flag from tool results. On a real corpus of 1,814 sessions / 90,626 tool calls this meant: - every call landed in a single `tool_use` bucket, so terminal commands were indistinguishable from MCP calls; - no MCP server attribution at all, even though Claude Code namespaces MCP tools as `mcp__<server>__<tool>` and the server is recoverable from the name; - zero errors reported across all 90,626 calls, because `is_error` was never read. After the fix, the same corpus yields `terminal_command` 48,008 / `tool_use` 24,617 / `mcp_tool` 18,001, 15 distinct MCP servers, and 5,327 failed calls (5.9%). Failed and blocked tool calls matter for detection, so losing them silently is a meaningful gap. This mirrors what `opencode_parser._classify_tool()` already does for opencode; the Claude Code parser had simply not received the same treatment. Also preserves `server_name` across the tool_use -> tool_result merge, which previously dropped it. Adds three tests: tool classification, server attribution surviving the result merge, and `is_error` producing `status="error"`.
1 parent d0e62f9 commit 83bda7c

2 files changed

Lines changed: 138 additions & 4 deletions

File tree

Sensor/adr_sensor/parsers/claude_parser.py

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,24 @@ def parse_all(self) -> List[AgentEvent]:
6767

6868
return entries
6969

70+
def _classify_tool(self, tool_name: str) -> tuple:
71+
"""Classify a Claude Code tool call and attribute MCP tools to their server.
72+
73+
Claude Code namespaces MCP tools as ``mcp__<server>__<tool>``, so the server is
74+
recoverable from the name alone. Without this, every call — including third-party
75+
MCP servers — collapses into a single ``tool_use`` bucket with no server attribution.
76+
"""
77+
if tool_name.startswith("mcp__"):
78+
parts = tool_name.split("__")
79+
if len(parts) >= 3 and parts[1]:
80+
return "mcp_tool", parts[1]
81+
return "mcp_tool", None
82+
83+
if tool_name in ("Bash", "PowerShell", "BashOutput", "KillShell"):
84+
return "terminal_command", None
85+
86+
return "tool_use", None
87+
7088
def _normalize_result_content(self, result_content: Any) -> str:
7189
"""Normalize result content which can be a string or list of content items."""
7290
if isinstance(result_content, str):
@@ -186,7 +204,13 @@ def _extract_message_data(self, obj: Dict[str, Any]) -> Optional[Dict[str, Any]]
186204

187205
if result_content and isinstance(result_content, str):
188206
result_content = truncate_middle(result_content, max_length=1000, edge_chars=400)
189-
tool_results.append({"tool_use_id": tool_use_id, "result": result_content})
207+
tool_results.append(
208+
{
209+
"tool_use_id": tool_use_id,
210+
"result": result_content,
211+
"is_error": bool(item.get("is_error")),
212+
}
213+
)
190214

191215
if tool_results:
192216
extracted["tool_results"] = tool_results
@@ -245,14 +269,23 @@ def _create_entry_from_extracted_session(
245269
for tool_result in tool_results:
246270
tool_use_id = tool_result.get("tool_use_id")
247271
result = tool_result.get("result")
272+
is_error = tool_result.get("is_error", False)
248273
if tool_use_id in pending_tools:
249274
old_tool = pending_tools[tool_use_id]
275+
if is_error:
276+
status = "error"
277+
elif result:
278+
status = "success"
279+
else:
280+
status = "unknown"
250281
updated_tool = ToolUsage(
251282
tool_name=old_tool.tool_name,
252283
tool_type=old_tool.tool_type,
284+
server_name=old_tool.server_name,
253285
arguments=old_tool.arguments,
254286
result=result,
255-
status="success" if result else "unknown",
287+
status=status,
288+
error=result if is_error else None,
256289
)
257290
for msg in entry.chat_history:
258291
if msg.role == "assistant":
@@ -274,9 +307,12 @@ def _create_entry_from_extracted_session(
274307
tools = []
275308

276309
for tool_data in msg_data.get("tools", []):
310+
tool_name = tool_data.get("name", "unknown")
311+
tool_type, server_name = self._classify_tool(tool_name)
277312
tool = ToolUsage(
278-
tool_name=tool_data.get("name", "unknown"),
279-
tool_type="tool_use",
313+
tool_name=tool_name,
314+
tool_type=tool_type,
315+
server_name=server_name,
280316
arguments=tool_data.get("input", {}),
281317
result=None,
282318
)

Sensor/tests/test_parsers.py

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,104 @@ def test_parse_jsonl_file(self, tmp_path):
7676
assert entry.model == "claude-sonnet-4-20250514"
7777
assert len(entry.chat_history) >= 1
7878

79+
def test_classify_tool(self):
80+
"""MCP tools are attributed to their server; shell tools are terminal commands."""
81+
parser = ClaudeParser()
82+
83+
assert parser._classify_tool("mcp__jarvis__ssh_run") == ("mcp_tool", "jarvis")
84+
assert parser._classify_tool("mcp__chrome-devtools__navigate") == (
85+
"mcp_tool",
86+
"chrome-devtools",
87+
)
88+
assert parser._classify_tool("Bash") == ("terminal_command", None)
89+
assert parser._classify_tool("PowerShell") == ("terminal_command", None)
90+
assert parser._classify_tool("Read") == ("tool_use", None)
91+
92+
def test_mcp_tool_keeps_server_after_result_merge(self, tmp_path):
93+
"""Server attribution survives the tool_use -> tool_result merge."""
94+
jsonl_file = tmp_path / "mcp.jsonl"
95+
messages = [
96+
{
97+
"type": "assistant",
98+
"sessionId": "session1",
99+
"timestamp": "2025-06-15T10:00:00Z",
100+
"message": {
101+
"model": "claude-sonnet-4-20250514",
102+
"content": [
103+
{
104+
"type": "tool_use",
105+
"id": "tool1",
106+
"name": "mcp__github__create_issue",
107+
"input": {"title": "hi"},
108+
}
109+
],
110+
},
111+
},
112+
{
113+
"type": "user",
114+
"sessionId": "session1",
115+
"timestamp": "2025-06-15T10:00:01Z",
116+
"message": {
117+
"content": [
118+
{"type": "tool_result", "tool_use_id": "tool1", "content": "created"}
119+
]
120+
},
121+
},
122+
]
123+
with open(jsonl_file, "w") as f:
124+
for msg in messages:
125+
f.write(json.dumps(msg) + "\n")
126+
127+
entries = ClaudeParser().parse_jsonl_file(jsonl_file)
128+
tools = [t for e in entries for m in e.chat_history for t in m.tools]
129+
130+
assert len(tools) == 1
131+
assert tools[0].tool_type == "mcp_tool"
132+
assert tools[0].server_name == "github"
133+
assert tools[0].status == "success"
134+
135+
def test_failed_tool_result_is_marked_error(self, tmp_path):
136+
"""A tool_result flagged is_error is recorded as an error, not a success."""
137+
jsonl_file = tmp_path / "error.jsonl"
138+
messages = [
139+
{
140+
"type": "assistant",
141+
"sessionId": "session1",
142+
"timestamp": "2025-06-15T10:00:00Z",
143+
"message": {
144+
"model": "claude-sonnet-4-20250514",
145+
"content": [
146+
{"type": "tool_use", "id": "tool1", "name": "Bash", "input": {"command": "false"}}
147+
],
148+
},
149+
},
150+
{
151+
"type": "user",
152+
"sessionId": "session1",
153+
"timestamp": "2025-06-15T10:00:01Z",
154+
"message": {
155+
"content": [
156+
{
157+
"type": "tool_result",
158+
"tool_use_id": "tool1",
159+
"content": "permission denied",
160+
"is_error": True,
161+
}
162+
]
163+
},
164+
},
165+
]
166+
with open(jsonl_file, "w") as f:
167+
for msg in messages:
168+
f.write(json.dumps(msg) + "\n")
169+
170+
entries = ClaudeParser().parse_jsonl_file(jsonl_file)
171+
tools = [t for e in entries for m in e.chat_history for t in m.tools]
172+
173+
assert len(tools) == 1
174+
assert tools[0].status == "error"
175+
assert tools[0].error == "permission denied"
176+
79177
def test_parse_empty_file(self, tmp_path):
80178
"""Test parsing an empty file."""
81179
jsonl_file = tmp_path / "empty.jsonl"

0 commit comments

Comments
 (0)