Skip to content

Commit 02eb11c

Browse files
AVADSA25Mikarina13claude
authored
refactor(dashboard): I1 — move Step-10 auto-escalate classifier → codec_chat_pipeline (SR-60) (#167)
codec_dashboard.py: 1,310 → 1,201 LOC (-109). Cumulative since pre-B6 baseline: 3,912 → 1,201 (-69.3%). The Phase 3 Step 10 auto-escalation classifier cluster (9 names: _AUTO_ESCALATE_SYSTEM_PROMPT, _qwen_chat_classify, _classify_chat_message, _AUTOESCALATE_SILENCE_LOCK, _autoescalate_silence_set, ESCALATE_CHECKPOINTS_THRESHOLD, silence_session_autoescalate, _reset_autoescalate_silence_for_test, _should_escalate_to_project) moves verbatim to codec_chat_pipeline.py — its natural home alongside _StepBudget / _is_conversational. The cluster is latent (no production caller yet; the chat→project "Promote?" prompt is deferred to Phase 3.5) but test-covered. codec_dashboard re-exports all 9 names identity-equal for back-compat. Test surface: - test_chat_escalation: monkeypatch target swapped codec_dashboard → codec_chat_pipeline. The functions call each other through the pipeline module namespace, so patching the in-module chain (_qwen_chat_classify → _classify_chat_message → _should_escalate_to_project) must target where they're DEFINED, not the re-export. All 8 monkeypatch tests updated. - test_dashboard_llm: codec_llm.call count >=2 → >=1 (classifier moved; only /api/command's Flash fallback remains in codec_dashboard). - new TestI1EscalationExtraction (3 pins): cluster-in-pipeline, 9-name identity-equal re-export, dashboard-no-longer-defines. - drive-by ruff F401: dropped now-unused `import threading` from codec_dashboard (the only consumer was _AUTOESCALATE_SILENCE_LOCK, which moved). codec_chat_pipeline gains `import threading` + `import codec_llm` for the moved code. test_dashboard_llm's 3 _qwen_chat_classify behavior tests are unaffected (they patch codec_llm.call at source, which the re-export honors). Full suite: 2,055 passed / 77 skipped (was 2,052 in H1). +3 net. Co-authored-by: Mickael Farina <farina.mickael@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent b04a303 commit 02eb11c

5 files changed

Lines changed: 202 additions & 137 deletions

File tree

codec_chat_pipeline.py

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@
2020
import json
2121
import logging
2222
import os
23+
import threading
2324
from typing import Optional
2425

2526
from codec_audit import STEP_BUDGET_EXHAUSTED, log_event
27+
import codec_llm # I1 / SR-60: auto-escalation classifier LLM call
2628

2729
log = logging.getLogger("codec_chat_pipeline")
2830

@@ -170,3 +172,137 @@ def _emit_exhausted(self, kind: str):
170172
)
171173
except Exception as e:
172174
log.warning("[step_budget] emit failed: %s", e)
175+
176+
177+
# ═══════════════════════════════════════════════════════════════
178+
# Phase 3 Step 10 — Auto-escalation classifier (I1 / SR-60)
179+
# Moved verbatim from codec_dashboard.py. Latent (no production caller
180+
# yet — the chat→project 'Promote?' prompt is deferred to Phase 3.5),
181+
# but exercised by tests. The functions call each other in-module, so
182+
# tests must monkeypatch codec_chat_pipeline.* (not the codec_dashboard
183+
# re-export). codec_dashboard re-exports all names for back-compat.
184+
# ═══════════════════════════════════════════════════════════════
185+
186+
# ── Phase 3 Step 10 — Auto-escalation classifier ──────────────────────────
187+
188+
_AUTO_ESCALATE_SYSTEM_PROMPT = """You are CODEC's chat-input classifier. \
189+
Given the user's chat message, decide if it represents a "project" — \
190+
multi-step work that would benefit from autonomous execution by an agent \
191+
(file writes, browser automation, multi-checkpoint plan) — or a "quick \
192+
question" suitable for single-shot LLM answer.
193+
194+
Return ONLY a JSON object:
195+
{
196+
"is_project": <bool>,
197+
"estimated_checkpoints": <int — best guess of plan size; 0 if not project>,
198+
"reason": <short string explaining the verdict>
199+
}
200+
201+
Rules:
202+
- Single-shot factual / conversational / explanatory questions → is_project=false.
203+
- "Build me X", "Set up Y", "Watch Z and tell me when W", "Plan launch of A" → is_project=true.
204+
- Be honest about checkpoint estimates; under 3 means not worth promoting.
205+
"""
206+
207+
208+
def _qwen_chat_classify(user_text: str, max_tokens: int = 300) -> str:
209+
"""Call Qwen-3.6 with the auto-escalation classifier prompt. Returns
210+
raw response string. Caller handles JSON parsing + error fallback.
211+
212+
Hotfix: URL + model resolved from codec_config (was hardcoded to the
213+
wrong dashboard port 8090; LLM lives at 8083 per ~/.codec/config.json)."""
214+
try:
215+
from codec_config import QWEN_BASE_URL, QWEN_MODEL as _qmodel
216+
# A-12 (PR-3E-dashboard): canonical codec_llm.call (never-raises -> "").
217+
# Now strips <think> + enable_thinking=False -> cleaner JSON for the
218+
# downstream _classify_chat_message parse.
219+
return codec_llm.call(
220+
[
221+
{"role": "system", "content": _AUTO_ESCALATE_SYSTEM_PROMPT},
222+
{"role": "user", "content": user_text[:2000]},
223+
],
224+
base_url=QWEN_BASE_URL, model=_qmodel,
225+
max_tokens=max_tokens, temperature=0.1, timeout=15,
226+
)
227+
except Exception as e:
228+
log.debug(f"_qwen_chat_classify failed: {e}")
229+
return ""
230+
231+
232+
def _classify_chat_message(user_text: str) -> tuple[bool, int, str]:
233+
"""Returns (is_project, estimated_checkpoints, reason). Falls back to
234+
(False, 0, reason) on any failure."""
235+
raw = _qwen_chat_classify(user_text)
236+
if not raw:
237+
return (False, 0, "qwen unavailable")
238+
239+
raw = raw.strip()
240+
if raw.startswith("```"):
241+
import re as _re
242+
raw = _re.sub(r"^```(?:json)?\s*", "", raw)
243+
raw = _re.sub(r"\s*```\s*$", "", raw)
244+
245+
try:
246+
d = json.loads(raw)
247+
except json.JSONDecodeError:
248+
return (False, 0, "qwen returned non-JSON")
249+
250+
return (
251+
bool(d.get("is_project", False)),
252+
int(d.get("estimated_checkpoints", 0)),
253+
str(d.get("reason", ""))[:200],
254+
)
255+
256+
257+
# ── Auto-escalation gate (in-memory session silence per Q11) ──────────────
258+
259+
_AUTOESCALATE_SILENCE_LOCK = threading.Lock()
260+
_autoescalate_silence_set: set[str] = set() # session_ids that said "no" once
261+
262+
ESCALATE_CHECKPOINTS_THRESHOLD = 3
263+
264+
265+
def silence_session_autoescalate(session_id: str) -> None:
266+
"""Q11: After user says No once, silence auto-escalation prompts for
267+
the rest of this conversation. Resets on new chat session."""
268+
with _AUTOESCALATE_SILENCE_LOCK:
269+
_autoescalate_silence_set.add(session_id)
270+
271+
272+
def _reset_autoescalate_silence_for_test() -> None:
273+
"""Test-only helper to clear in-memory silence state."""
274+
with _AUTOESCALATE_SILENCE_LOCK:
275+
_autoescalate_silence_set.clear()
276+
277+
278+
def _should_escalate_to_project(user_text: str, session_id: str) -> dict:
279+
"""2-signal gate (Step 10):
280+
Signal 1: classifier verdict (is_project=True)
281+
Signal 2: estimated_checkpoints >= ESCALATE_CHECKPOINTS_THRESHOLD
282+
283+
Plus 2 kill conditions:
284+
- AGENT_AUTO_ESCALATE_ENABLED=false
285+
- session_id in silence set (Q11)
286+
287+
Returns: {"escalate": bool, "estimated_checkpoints": int, "reason": str}
288+
"""
289+
import os as _os
290+
if _os.environ.get("AGENT_AUTO_ESCALATE_ENABLED", "true").lower() == "false":
291+
return {"escalate": False, "estimated_checkpoints": 0,
292+
"reason": "kill_switch_off"}
293+
294+
with _AUTOESCALATE_SILENCE_LOCK:
295+
if session_id in _autoescalate_silence_set:
296+
return {"escalate": False, "estimated_checkpoints": 0,
297+
"reason": "session_silenced", "silenced": True}
298+
299+
is_project, n_checkpoints, reason = _classify_chat_message(user_text)
300+
301+
escalate = is_project and n_checkpoints >= ESCALATE_CHECKPOINTS_THRESHOLD
302+
303+
return {
304+
"escalate": escalate,
305+
"estimated_checkpoints": n_checkpoints,
306+
"reason": reason,
307+
"is_project": is_project,
308+
}

codec_dashboard.py

Lines changed: 17 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import json
44
import time
55
import hmac
6-
import threading
76
import asyncio
87
from datetime import datetime, timedelta
98

@@ -862,135 +861,27 @@ async def vibe_page():
862861
_step_budget_for_route,
863862
_StepBudget,
864863
)
864+
# I1 / SR-60: auto-escalation classifier cluster (Step 10) moved to
865+
# codec_chat_pipeline. Re-exported here so any caller / test that imported them
866+
# from codec_dashboard keeps working. NOTE: the functions call each other via
867+
# the pipeline module namespace — tests that monkeypatch the chain must target
868+
# codec_chat_pipeline.*, not these re-exports (test_chat_escalation does).
869+
from codec_chat_pipeline import ( # noqa: E402,F401 (back-compat re-exports)
870+
ESCALATE_CHECKPOINTS_THRESHOLD,
871+
_AUTO_ESCALATE_SYSTEM_PROMPT,
872+
_autoescalate_silence_set,
873+
_AUTOESCALATE_SILENCE_LOCK,
874+
_classify_chat_message,
875+
_qwen_chat_classify,
876+
_reset_autoescalate_silence_for_test,
877+
_should_escalate_to_project,
878+
silence_session_autoescalate,
879+
)
865880

866881

867882
# H1 / SR-59: def _try_skill → moved to routes/chat.py
868883
# H1 / SR-59: def _try_skill_by_name → moved to routes/chat.py
869-
# ── Phase 3 Step 10 — Auto-escalation classifier ──────────────────────────
870-
871-
_AUTO_ESCALATE_SYSTEM_PROMPT = """You are CODEC's chat-input classifier. \
872-
Given the user's chat message, decide if it represents a "project" — \
873-
multi-step work that would benefit from autonomous execution by an agent \
874-
(file writes, browser automation, multi-checkpoint plan) — or a "quick \
875-
question" suitable for single-shot LLM answer.
876-
877-
Return ONLY a JSON object:
878-
{
879-
"is_project": <bool>,
880-
"estimated_checkpoints": <int — best guess of plan size; 0 if not project>,
881-
"reason": <short string explaining the verdict>
882-
}
883-
884-
Rules:
885-
- Single-shot factual / conversational / explanatory questions → is_project=false.
886-
- "Build me X", "Set up Y", "Watch Z and tell me when W", "Plan launch of A" → is_project=true.
887-
- Be honest about checkpoint estimates; under 3 means not worth promoting.
888-
"""
889-
890-
891-
def _qwen_chat_classify(user_text: str, max_tokens: int = 300) -> str:
892-
"""Call Qwen-3.6 with the auto-escalation classifier prompt. Returns
893-
raw response string. Caller handles JSON parsing + error fallback.
894-
895-
Hotfix: URL + model resolved from codec_config (was hardcoded to the
896-
wrong dashboard port 8090; LLM lives at 8083 per ~/.codec/config.json)."""
897-
try:
898-
from codec_config import QWEN_BASE_URL, QWEN_MODEL as _qmodel
899-
# A-12 (PR-3E-dashboard): canonical codec_llm.call (never-raises -> "").
900-
# Now strips <think> + enable_thinking=False -> cleaner JSON for the
901-
# downstream _classify_chat_message parse.
902-
return codec_llm.call(
903-
[
904-
{"role": "system", "content": _AUTO_ESCALATE_SYSTEM_PROMPT},
905-
{"role": "user", "content": user_text[:2000]},
906-
],
907-
base_url=QWEN_BASE_URL, model=_qmodel,
908-
max_tokens=max_tokens, temperature=0.1, timeout=15,
909-
)
910-
except Exception as e:
911-
log.debug(f"_qwen_chat_classify failed: {e}")
912-
return ""
913-
914-
915-
def _classify_chat_message(user_text: str) -> tuple[bool, int, str]:
916-
"""Returns (is_project, estimated_checkpoints, reason). Falls back to
917-
(False, 0, reason) on any failure."""
918-
raw = _qwen_chat_classify(user_text)
919-
if not raw:
920-
return (False, 0, "qwen unavailable")
921-
922-
raw = raw.strip()
923-
if raw.startswith("```"):
924-
import re as _re
925-
raw = _re.sub(r"^```(?:json)?\s*", "", raw)
926-
raw = _re.sub(r"\s*```\s*$", "", raw)
927-
928-
try:
929-
d = json.loads(raw)
930-
except json.JSONDecodeError:
931-
return (False, 0, "qwen returned non-JSON")
932-
933-
return (
934-
bool(d.get("is_project", False)),
935-
int(d.get("estimated_checkpoints", 0)),
936-
str(d.get("reason", ""))[:200],
937-
)
938-
939-
940-
# ── Auto-escalation gate (in-memory session silence per Q11) ──────────────
941-
942-
_AUTOESCALATE_SILENCE_LOCK = threading.Lock()
943-
_autoescalate_silence_set: set[str] = set() # session_ids that said "no" once
944-
945-
ESCALATE_CHECKPOINTS_THRESHOLD = 3
946-
947-
948-
def silence_session_autoescalate(session_id: str) -> None:
949-
"""Q11: After user says No once, silence auto-escalation prompts for
950-
the rest of this conversation. Resets on new chat session."""
951-
with _AUTOESCALATE_SILENCE_LOCK:
952-
_autoescalate_silence_set.add(session_id)
953-
954-
955-
def _reset_autoescalate_silence_for_test() -> None:
956-
"""Test-only helper to clear in-memory silence state."""
957-
with _AUTOESCALATE_SILENCE_LOCK:
958-
_autoescalate_silence_set.clear()
959-
960-
961-
def _should_escalate_to_project(user_text: str, session_id: str) -> dict:
962-
"""2-signal gate (Step 10):
963-
Signal 1: classifier verdict (is_project=True)
964-
Signal 2: estimated_checkpoints >= ESCALATE_CHECKPOINTS_THRESHOLD
965-
966-
Plus 2 kill conditions:
967-
- AGENT_AUTO_ESCALATE_ENABLED=false
968-
- session_id in silence set (Q11)
969-
970-
Returns: {"escalate": bool, "estimated_checkpoints": int, "reason": str}
971-
"""
972-
import os as _os
973-
if _os.environ.get("AGENT_AUTO_ESCALATE_ENABLED", "true").lower() == "false":
974-
return {"escalate": False, "estimated_checkpoints": 0,
975-
"reason": "kill_switch_off"}
976-
977-
with _AUTOESCALATE_SILENCE_LOCK:
978-
if session_id in _autoescalate_silence_set:
979-
return {"escalate": False, "estimated_checkpoints": 0,
980-
"reason": "session_silenced", "silenced": True}
981-
982-
is_project, n_checkpoints, reason = _classify_chat_message(user_text)
983-
984-
escalate = is_project and n_checkpoints >= ESCALATE_CHECKPOINTS_THRESHOLD
985-
986-
return {
987-
"escalate": escalate,
988-
"estimated_checkpoints": n_checkpoints,
989-
"reason": reason,
990-
"is_project": is_project,
991-
}
992-
993-
884+
# I1 / SR-60: Phase 3 Step 10 auto-escalation classifier cluster → moved to codec_chat_pipeline.py (re-exported below)
994885
# H1 / SR-59: def _chat_vision_response → moved to routes/chat.py
995886
# H1 / SR-59: def _build_chat_system_prompt → moved to routes/chat.py
996887
# H1 / SR-59: → moved to routes/chat.py

tests/test_chat_escalation.py

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
33
11 tests covering: classifier (3), 2-signal gate (3), session silence (2),
44
integration with chat handler (2), kill switch (1).
5+
6+
I1 / SR-60: the classifier cluster moved from codec_dashboard to
7+
codec_chat_pipeline. These tests monkeypatch the in-module call chain
8+
(`_qwen_chat_classify` → `_classify_chat_message` → `_should_escalate_to_project`),
9+
so they must patch where the functions are DEFINED (codec_chat_pipeline),
10+
not the codec_dashboard re-export. Hence `import codec_chat_pipeline as cd`.
511
"""
612
from __future__ import annotations
713

@@ -17,7 +23,7 @@
1723

1824
def test_classify_chat_message_returns_project_when_multi_step(monkeypatch):
1925
"""LLM verdict says multi-step → returns is_project=True with checkpoints estimate."""
20-
import codec_dashboard as cd
26+
import codec_chat_pipeline as cd
2127

2228
fake_response = json.dumps({
2329
"is_project": True,
@@ -35,7 +41,7 @@ def test_classify_chat_message_returns_project_when_multi_step(monkeypatch):
3541

3642

3743
def test_classify_chat_message_returns_not_project_for_quick_question(monkeypatch):
38-
import codec_dashboard as cd
44+
import codec_chat_pipeline as cd
3945

4046
fake_response = json.dumps({
4147
"is_project": False, "estimated_checkpoints": 0,
@@ -50,7 +56,7 @@ def test_classify_chat_message_returns_not_project_for_quick_question(monkeypatc
5056

5157
def test_classify_chat_message_handles_qwen_failure(monkeypatch):
5258
"""If Qwen call fails or returns garbage, classifier returns (False, 0, reason)."""
53-
import codec_dashboard as cd
59+
import codec_chat_pipeline as cd
5460

5561
monkeypatch.setattr(cd, "_qwen_chat_classify",
5662
lambda text: "garbage non-json")
@@ -62,7 +68,7 @@ def test_classify_chat_message_handles_qwen_failure(monkeypatch):
6268

6369
def test_should_escalate_when_both_signals_pass(monkeypatch):
6470
"""LLM says project + checkpoints >= 3 → escalate."""
65-
import codec_dashboard as cd
71+
import codec_chat_pipeline as cd
6672

6773
monkeypatch.setattr(cd, "_classify_chat_message",
6874
lambda text: (True, 5, "multi-step"))
@@ -74,7 +80,7 @@ def test_should_escalate_when_both_signals_pass(monkeypatch):
7480

7581
def test_should_not_escalate_when_checkpoints_below_3(monkeypatch):
7682
"""LLM says project but estimate=2 → don't escalate."""
77-
import codec_dashboard as cd
83+
import codec_chat_pipeline as cd
7884

7985
monkeypatch.setattr(cd, "_classify_chat_message",
8086
lambda text: (True, 2, "small"))
@@ -85,7 +91,7 @@ def test_should_not_escalate_when_checkpoints_below_3(monkeypatch):
8591

8692
def test_should_not_escalate_when_classifier_says_no(monkeypatch):
8793
"""LLM says not-a-project → don't escalate even if checkpoints>=3."""
88-
import codec_dashboard as cd
94+
import codec_chat_pipeline as cd
8995

9096
monkeypatch.setattr(cd, "_classify_chat_message",
9197
lambda text: (False, 5, "actually quick"))
@@ -96,7 +102,7 @@ def test_should_not_escalate_when_classifier_says_no(monkeypatch):
96102

97103
def test_session_silence_persists_across_calls(monkeypatch):
98104
"""Q11: After silence_session(s1), subsequent _should_escalate calls return escalate=False."""
99-
import codec_dashboard as cd
105+
import codec_chat_pipeline as cd
100106

101107
monkeypatch.setattr(cd, "_classify_chat_message",
102108
lambda text: (True, 5, "always-project"))
@@ -117,7 +123,7 @@ def test_session_silence_persists_across_calls(monkeypatch):
117123

118124
def test_kill_switch_disables_all_escalation(monkeypatch):
119125
"""AGENT_AUTO_ESCALATE_ENABLED=false → never escalate."""
120-
import codec_dashboard as cd
126+
import codec_chat_pipeline as cd
121127

122128
monkeypatch.setenv("AGENT_AUTO_ESCALATE_ENABLED", "false")
123129
monkeypatch.setattr(cd, "_classify_chat_message",

0 commit comments

Comments
 (0)