From 7e74a42e17c334bcd2400c1e1dc505b023b3e646 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Fri, 17 Jul 2026 22:14:34 -0400 Subject: [PATCH 01/15] feat: harden and streamline history search --- chatview/cli.py | 51 +- chatview/commands/analysis.py | 187 +++++-- chatview/commands/corrections.py | 22 +- chatview/commands/evidence.py | 257 +++++++++ chatview/commands/evolve.py | 25 +- chatview/commands/retrieval.py | 203 ++++--- chatview/commands/search_context.py | 83 ++- chatview/db/__init__.py | 13 +- chatview/db/core.py | 102 ++++ chatview/db/sessions.py | 10 +- chatview/search.py | 31 +- chatview/snippets.py | 86 +++ evals/retrieval/baseline.json | 3 +- evals/retrieval/cases.json | 42 +- scripts/run_retrieval_eval.py | 8 +- skills/distill-yourself/SKILL.md | 22 +- skills/distill-yourself/evals/evals.json | 28 + .../references/rules-signals-patterns.md | 6 +- .../references/twin-cognitive-model.md | 4 +- tests/test_commands_analysis.py | 294 +++++++++- tests/test_commands_evolve.py | 74 +++ tests/test_correction_events.py | 54 +- tests/test_distill_skill_static.py | 21 +- tests/test_retrieval_eval_runner.py | 4 +- tests/test_retrieval_tools.py | 194 ++++++- tests/test_search_robustness.py | 502 ++++++++++++++++++ tests/test_search_snippets.py | 31 ++ 27 files changed, 2062 insertions(+), 295 deletions(-) create mode 100644 chatview/commands/evidence.py create mode 100644 chatview/snippets.py create mode 100644 skills/distill-yourself/evals/evals.json create mode 100644 tests/test_search_robustness.py create mode 100644 tests/test_search_snippets.py diff --git a/chatview/cli.py b/chatview/cli.py index cc8ecd7..7df6271 100644 --- a/chatview/cli.py +++ b/chatview/cli.py @@ -35,7 +35,7 @@ cmd_profile_digest, ) from chatview.commands.retrieval import ( - cmd_search_plus, cmd_read_window, cmd_find_repeats, cmd_session_brief, + cmd_read_window, cmd_find_repeats, cmd_session_brief, cmd_evidence_audit, ) from chatview.commands.twin import ( @@ -58,6 +58,13 @@ ) +def _max_chars(value): + parsed = int(value) + if not 1 <= parsed <= 2000: + raise argparse.ArgumentTypeError("--max-chars must be between 1 and 2000") + return parsed + + def main(): parser = argparse.ArgumentParser( description="CLI tools for analyzing Claude Code / Codex conversation history.", @@ -89,7 +96,8 @@ def main(): "--project", default="", help="Filter by project name (substring match)" ) shared.add_argument( - "--limit", type=int, default=50, help="Max results (default: 50)" + "--limit", type=int, default=50, + help="Max results per page (search default: 20; other commands: 50)", ) shared.add_argument("--json", action="store_true", help="Output as JSON") shared.add_argument( @@ -117,7 +125,12 @@ def main(): ) p_search = sub.add_parser("search", parents=[shared], help="Search messages") + p_search.set_defaults(limit=20) p_search.add_argument("query", help="Search query") + p_search.add_argument( + "--recall", choices=("normal", "high"), default="normal", + help="Retrieval breadth (default: normal)", + ) p_search.add_argument( "--role", choices=("all", "user", "assistant"), @@ -136,29 +149,18 @@ def main(): p_search.add_argument( "-n", "--line-number", action="store_true", help="Prefix context lines with session:idx:role" ) - - p_search_plus = sub.add_parser( - "search-plus", - parents=[shared], - help="Hybrid higher-recall search with match reasons", - ) - p_search_plus.add_argument("query", help="Search query") - p_search_plus.add_argument( - "-C", "--context", type=int, default=0, help="Show N messages before/after each match" - ) - p_search_plus.add_argument( - "-B", "--before", type=int, help="Show N messages before each match" - ) - p_search_plus.add_argument( - "-A", "--after", type=int, help="Show N messages after each match" + p_search.add_argument( + "--format", choices=("standard", "lines", "jsonl"), default="standard", + help="Output format (default: standard)", ) - p_search_plus.add_argument( - "-n", "--line-number", action="store_true", help="Prefix context lines with session:idx:role" + p_search.add_argument("--page", type=int, default=1, help="1-based result page") + p_search.add_argument( + "--max-chars", type=_max_chars, default=500, + help="Maximum characters per result or inline context message (default: 500)", ) - p_search_plus.add_argument( - "--include-artifacts", - action="store_true", - help="Deprecated compatibility flag; artifacts are always marked and downranked", + p_search.add_argument( + "--evidence-only", action="store_true", + help="Exclude orchestration artifacts and meta-research prompts", ) p_read_window = sub.add_parser( @@ -166,7 +168,7 @@ def main(): parents=[shared], help="Read a small message window around a match index", ) - p_read_window.add_argument("session", nargs="?", help="Session ID or partial match") + p_read_window.add_argument("session", nargs="?", help="Exact session ID") p_read_window.add_argument("--idx", type=int, help="Target message index") p_read_window.add_argument( "--radius", @@ -441,7 +443,6 @@ def main(): "sessions": cmd_sessions, "read": cmd_read, "search": cmd_search, - "search-plus": cmd_search_plus, "read-window": cmd_read_window, "find-repeats": cmd_find_repeats, "session-brief": cmd_session_brief, diff --git a/chatview/commands/analysis.py b/chatview/commands/analysis.py index e062bb2..5905b5b 100644 --- a/chatview/commands/analysis.py +++ b/chatview/commands/analysis.py @@ -11,8 +11,15 @@ from chatview.index import build_index from chatview.session_loader import load_session_from_file from chatview.parsers.codex import _CODEX_TOOL_NAMES -from chatview.commands.search_context import format_grep_window, wants_grep_output +from chatview.commands.search_context import ( + attach_search_context, + format_grep_window, + format_search_lines, + wants_grep_output, +) +from chatview.commands.evidence import artifact_reason, dedupe_message_results, page_results from chatview.project_identity import project_matches +from chatview.snippets import make_query_snippet # --------------------------------------------------------------------------- @@ -143,6 +150,13 @@ def _ensure_project_identity_backfill(args): _init_index(force=True) +def _prepare_search_db(args): + """Prepare search without requiring writes on an already compatible cache.""" + from chatview import db as _db + + _db.prepare_search_db() + + def _get_filtered_db(args) -> list: """Return filtered sessions from SQLite DB as a list of dicts.""" from chatview import db as _db @@ -350,11 +364,46 @@ def flush_tools(): print("\n".join(output)) +def _render_search_results(results, args, total=None): + output_format = getattr(args, "format", "standard") + if args.json: + print(json.dumps(attach_search_context(results, args), ensure_ascii=False, indent=2)) + return None + if output_format == "jsonl": + for row in attach_search_context(results, args): + print(json.dumps(row, ensure_ascii=False)) + return None + if output_format == "lines": + for line in format_search_lines(results, args): + print(line) + return None + + print(f"Found {total if total is not None else len(results)} matches for '{args.query}':\n") + for row in results: + date = row.get("date", "")[:10] + reasons = ",".join(row.get("reasons", [])) + print(f" [{date}] {row.get('title', '')[:60]}") + print(f" {row.get('project', '')} · idx:{row.get('idx')} · {reasons}") + print(f" > {row.get('snippet', '')}") + print(f" session: {row.get('sessionId', '')}") + if wants_grep_output(args) and row.get("candidateType", "message") == "message": + print(f" read-window: distill read-window {row.get('sessionId', '')} --idx {row.get('idx', 0)}") + for line in format_grep_window(row.get("sessionId", ""), row.get("idx", 0), args): + print(line) + print() + return None + + def cmd_search(args): """Search messages across sessions (via FTS).""" from chatview import db as _db - _ensure_project_identity_backfill(args) + if getattr(args, "recall", "normal") == "high": + from chatview.commands.retrieval import search_high_data + + return _render_search_results(search_high_data(args.query, args), args) + + _prepare_search_db(args) now = datetime.now() days_map = {"1d": 1, "7d": 7, "30d": 30, "90d": 90} @@ -362,77 +411,97 @@ def cmd_search(args): if args.date and args.date != "all": min_date = (now - timedelta(days=days_map.get(args.date, 9999))).strftime("%Y-%m-%d") - # Push filters before FTS LIMIT so out-of-scope rows cannot exhaust candidates. - fts_results = _db.search_fts( - args.query, - limit=500, - role=getattr(args, "role", "all"), - source=getattr(args, "source", "all"), - project=getattr(args, "project", ""), - min_date=min_date, - ) - - # Retain defensive CLI filtering for callers using alternate DB backends. + page = max(getattr(args, "page", 1), 1) + page_size = max(getattr(args, "limit", 20), 1) + required = page * page_size + evidence_only = getattr(args, "evidence_only", False) out = [] + deduped = [] + fetch_limit = max(required, 64) + while True: + # Expand a stable FTS prefix only when filtering and event dedupe leave + # too few rows for the requested page. + fts_results = _db.search_fts( + args.query, + limit=fetch_limit, + role=getattr(args, "role", "all"), + source=getattr(args, "source", "all"), + project=getattr(args, "project", ""), + min_date=min_date, + ) - for r in fts_results: - if args.source and args.source != "all": - if (r.get("source") or "claude") != args.source: + # Rebuild from the current prefix so duplicate metadata stays complete. + # Retain defensive CLI filtering for alternate DB backends. + out = [] + for r in fts_results: + if args.source and args.source != "all": + if (r.get("source") or "claude") != args.source: + continue + if args.project and not project_matches( + { + "projectName": r.get("project_name", ""), + "projectKey": r.get("project_key", ""), + "projectDisplay": r.get("project_display", ""), + }, + args.project, + substring=True, + ): continue - if args.project and not project_matches( - { - "projectName": r.get("project_name", ""), - "projectKey": r.get("project_key", ""), - "projectDisplay": r.get("project_display", ""), - }, - args.project, - substring=True, - ): - continue - if args.date and args.date != "all": - date_str = r.get("ts") or "" - if date_str: - try: - d = datetime.fromisoformat(date_str.replace("Z", "+00:00")).replace( - tzinfo=None - ) - if (now - d).days > days_map.get(args.date, 9999): - continue - except Exception: - pass + if args.date and args.date != "all": + date_str = r.get("ts") or "" + if date_str: + try: + d = datetime.fromisoformat(date_str.replace("Z", "+00:00")).replace( + tzinfo=None + ) + if (now - d).days > days_map.get(args.date, 9999): + continue + except Exception: + pass - # Build output record - text = r.get("text", "") - snippet = text[:200] if text else "" - out.append( - { + text = r.get("text", "") + reason = artifact_reason(text, r.get("title", "")) + if evidence_only and reason: + continue + snippet_data = make_query_snippet( + text, + args.query, + max_chars=getattr(args, "max_chars", 500), + ) + item = { "sessionId": r.get("session_id", ""), "title": r.get("title", ""), "project": r.get("project_display") or r.get("project_name", ""), "date": (r.get("ts") or "")[:19], "messageIndex": r.get("idx", 0), "idx": r.get("idx", 0), - "snippet": snippet, + **snippet_data, "matchType": r.get("role", "content"), "score": 0, + "role": r.get("role", "content"), + "source": r.get("source", ""), + "candidateType": "message", + "reasons": ["message_fts"], + "_eventText": text, + "evidenceEligible": not bool(reason), } - ) + if reason: + item["artifactReason"] = reason + out.append(item) - if args.json: - print(json.dumps(out[: args.limit], ensure_ascii=False, indent=2)) - else: - print(f"Found {len(out)} matches for '{args.query}':\n") - for r in out[: args.limit]: - date = r.get("date", "")[:10] - print(f" [{date}] {r.get('title', '')[:60]}") - print(f" {r.get('project', '')} · match: {r.get('matchType', '')}") - print(f" > {r.get('snippet', '')[:200]}") - print(f" session: {r.get('sessionId', '')}") - if wants_grep_output(args): - print(f" read-window: distill read-window {r.get('sessionId', '')} --idx {r.get('idx', 0)}") - for line in format_grep_window(r.get("sessionId", ""), r.get("idx", 0), args): - print(line) - print() + deduped = dedupe_message_results(out) + if len(deduped) >= required or len(fts_results) < fetch_limit: + break + if fetch_limit < 160: + fetch_limit = 160 + elif fetch_limit < 500: + fetch_limit = 500 + else: + fetch_limit *= 2 + + shown = page_results(deduped, args) + + return _render_search_results(shown, args, total=len(out)) def cmd_queries(args): diff --git a/chatview/commands/corrections.py b/chatview/commands/corrections.py index a58c8b0..2f99177 100644 --- a/chatview/commands/corrections.py +++ b/chatview/commands/corrections.py @@ -6,9 +6,10 @@ from collections import defaultdict from chatview.commands.analysis import _get_filtered, _get_filtered_db, _get_messages_db +from chatview.commands.evidence import artifact_reason from chatview.utils.text import normalize_error as _normalize_error -_CORRECTION_EXTRACTOR_VERSION = "corrections-v3" +_CORRECTION_EXTRACTOR_VERSION = "corrections-v5" # --------------------------------------------------------------------------- @@ -221,7 +222,7 @@ def _correction_regexes(): } -def _skip_correction_noise(text): +def _skip_correction_noise(text, title=""): if len(text) < 5 or len(text) > 3000: return True if text.strip().startswith(("#", "```", "<", "{")): @@ -230,6 +231,8 @@ def _skip_correction_noise(text): return True if "Base directory for this skill:" in text or "skill:" in text[:30]: return True + if artifact_reason(text, title): + return True return False @@ -240,9 +243,12 @@ def _extract_corrections_from_session(meta, user_texts, assistant_snippets, rege seen_keys = set() asst_by_idx = {a["idx"]: a["text"] for a in assistant_snippets} + if artifact_reason(title=meta.get("title", "")): + return [] + for ut in user_texts: text = ut.get("text", "") - if _skip_correction_noise(text): + if _skip_correction_noise(text, meta.get("title", "")): continue matches = regexes["combined"].findall(text) if not matches: @@ -275,6 +281,8 @@ def _extract_corrections_from_session(meta, user_texts, assistant_snippets, rege user_corr_idxs = {k[1] for k in seen_keys if k[0] == meta["id"]} for asst in assistant_snippets: asst_text = asst.get("text", "") + if _skip_correction_noise(asst_text, meta.get("title", "")): + continue corr_matches = regexes["ai_correction"].findall(asst_text) insight_matches = regexes["ai_insight"].findall(asst_text) if not corr_matches and not insight_matches: @@ -293,6 +301,10 @@ def _extract_corrections_from_session(meta, user_texts, assistant_snippets, rege if ut["idx"] < asst["idx"]: preceding_user = ut["text"][:200] break + if preceding_user and _skip_correction_noise( + preceding_user, meta.get("title", "") + ): + continue corrections.append( { "sessionId": meta["id"], @@ -579,6 +591,8 @@ def _data_decisions(args): # Skip noise if len(text) < 10 or len(text) > 2000: continue + if artifact_reason(text, meta.get("title", "")): + continue if text.strip().startswith(("<", "{", "```", "#")): continue if ( @@ -611,6 +625,8 @@ def _data_decisions(args): text = ut.get("text", "") if len(text) < 10 or len(text) > 2000: continue + if artifact_reason(text, meta.get("title", "")): + continue if text.strip().startswith(("<", "{", "```", "#")): continue if ( diff --git a/chatview/commands/evidence.py b/chatview/commands/evidence.py new file mode 100644 index 0000000..cd67bc9 --- /dev/null +++ b/chatview/commands/evidence.py @@ -0,0 +1,257 @@ +"""Shared evidence hygiene, duplicate folding, and result paging.""" + +from __future__ import annotations + +import re + + +_ARTIFACT_PATTERNS = [ + ("task_notification", re.compile(r"||toolu_", re.I)), + ("continuation_summary", re.compile(r"This session is being continued from a previous conversation|Summary:\s*\n", re.I)), + ("ide_context", re.compile(r"# Context from my IDE setup|Active file:|Open tabs:", re.I)), + ("retrieval_work_product", re.compile( + r"Pre-collected Data \(do NOT re-run these\)|Full-text search across all sessions Options:|" + r"pre-computed project distribution \+ daily activity as JSON|=== STATS ===", + re.I, + )), + ("subagent_prompt", re.compile( + r"(?:You are|你是(?:一个|一名|\s*[A-Z]))[\s\S]{0,120}" + r"(?:子.{0,8}代理|子\s*agent|sub-?agent)|" + r"你是\s*Agent\s*\d+|" + r"(?:Subagent task|子代理任务)\s*[:：]", + re.I, + )), + ("review_prompt", re.compile( + r"You are (?:an? )?(?:independent |strict )?(?:reviewer|grader|evaluator)|" + r"你是.{0,30}(?:独立审阅员|复核员|评测员)|" + r"(?:独立\s*Review\s*请求|请独立回答|对抗式交叉评审)|" + r"(?:设计|架构|代码)?评审请求.{0,40}独立判断", + re.I, + )), + ("capability_probe", re.compile( + r"tool_search.{0,80}\bNOW\b|(?:call|invoke|use).{0,30}(?:tool|skill).{0,30}exactly once|" + r"只说明.{0,40}(?:skill|工具).{0,60}(?:停止|不要真正|不要实际)", + re.I | re.S, + )), + ("acceptance_probe", re.compile( + r"Acceptance self-test|\bFLAG_[A-Z0-9_]+\b|(?:pass|fail).{0,30}(?:acceptance|assertion)", + re.I, + )), + ("routing_test", re.compile( + r"(?:search|检索).{0,30}exactly once|(?:只|仅).{0,20}(?:调用|运行).{0,20}(?:一次|一遍)", + re.I, + )), + ("generated_dump", re.compile( + r"(?:generated|synthetic).{0,30}(?:dump|fixture|transcript)|(?:BEGIN|END) GENERATED DATA|" + r"# Judgment Cards \(input data\)|\"strength\"\s*:\s*0\.\d+.{0,120}\"status\"\s*:\s*\"confirmed\"|" + r"\"run_id\"\s*:\s*\"run_[a-z0-9]+\".{0,120}\"operations\"", + re.I, + )), + ("agent_prompt", re.compile( + r"You are Agent\b|You are independently|You are review|You are extracting structured|" + r"You have a CLI tool for analyzing conversation history|Acceptance self-test|" + r"Base directory for this skill|你是一个严格、独立的标注员|任务：判断每条|" + r"你是.{0,80}(?:子\s*agent|subagent)|任务：不要写泛泛趋势", + re.I, + )), + ("structured_noise", re.compile(r"(?:^|\n)\s*(?:<|\{|\[Request interrupted|```|toolu_)", re.I)), +] + +_META_RESEARCH_RE = re.compile( + r"(?:搜索|检索|查找|找出|分析).{0,30}(?:历史|对话).{0,40}(?:哪些|场景|记录|证据)|" + r"(?:历史|对话).{0,30}(?:搜索|检索|查找|找出|分析).{0,40}(?:哪些|场景|记录|证据)|" + r"(?:search|analy[sz]e|find).{0,40}(?:history|conversations?).{0,50}(?:cases|examples|evidence)", + re.I | re.S, +) + +_ORCHESTRATION_ROLE_RE = re.compile( + r"(?:第\s*[二2]\s*轮|(?:round|phase)\s*2).{0,50}(?:qa|测试|review|审查|复核)|" + r"(?:修复已提交|修复完成|已修复|implementation complete).{0,100}(?:只验|复核|review|verify)|" + r"(?:研究任务|测试任务|审查任务|review task)\s*[A-Z0-9一二三]?\s*[:：]|" + r"(?:Resume\s+/goal|Fallback\s*:\s*(?:check|review|verify))|" + r"(?:Act as|作为).{0,80}(?:sheriff|verification\s+(?:lead|owner)|负责人|红队|" + r"审阅|评审|复核|reviewer|tester|validator|researcher|owner|lead)|" + r"(?:你是|You are).{0,80}(?:顾问|审阅|评审|复核|测试|研究员|设计专家|analyzing|reviewing|testing|" + r"AI\s+Coding\s+Agent|reviewer|tester|researcher|worker|design expert)", + re.I | re.S, +) +_TASK_HEADING_RE = re.compile( + r"(?:^|\n)\s*(?:#{1,4}\s*(?:.{0,12}任务|Task(?:\s+\d+)?)\s*[:：]?|" + r"(?:任务|Task)\s*[:：]|Task\s+\d+\b|(?:read[- ]only\s+)?verification task\b)", + re.I, +) +_CONTINUATION_REVIEW_RE = re.compile( + r"(?:上一轮|第二轮|修订|finding|implementation).{0,180}(?:只验|只验证|复核|核验|review)|" + r"(?:请只验证|请复核|请重新读取并逐条核验)|" + r"(?:^|\n)\s*(?:Review this diff|Focus on .{0,100}(?:regression|compatibility))", + re.I | re.S, +) +_STRUCTURED_LABEL_RE = re.compile( + r"(?:^|\n)\s*(?:任务|背景|目标|范围|要求|约束|验收|输出|交付|步骤|文件|" + r"task|context|goal|scope|requirements?|constraints?|acceptance|output|deliverables?)\s*[:：]", + re.I, +) +_STRUCTURED_LIST_RE = re.compile(r"(?:^|\n)\s*(?:[-*]|\d+[.)、])\s+", re.M) +_INLINE_NUMBERED_RE = re.compile(r"(?:^|[\s:：;；])\d+[.)、]\s*") +_OUTPUT_PROTOCOL_RE = re.compile( + r"(?:输出格式|返回格式|完成后.{0,30}(?:返回|报告)|不得修改|不要修改|只读|read[- ]only|" + r"do not edit|report (?:only|exactly)|respond with|验收标准|acceptance criteria|" + r"(?:return|返回).{0,50}(?:verdict|结论|报告)|(?:最终|明确)\s*verdict)", + re.I | re.S, +) +_PROTOCOL_PAIR_RE = re.compile( + r"(?:Resume\s+/goal|Fallback\s*:).{0,500}(?:do not edit|只读|return\s*[:：]|verdict)|" + r"(?:do not edit|不要修改).{0,300}(?:return\s*[:：].{0,50}verdict)|" + r"(?:return\s*[:：].{0,50}verdict).{0,300}(?:do not edit|不要修改)", + re.I | re.S, +) +_GENERIC_ASSIGNMENT_RE = re.compile( + r"(?:\b(?:act|operate|serve|work)\s+as\b|\btake\s+(?:the\s+)?chair\s+as\b|" + r"(?:请)?担任.{0,80}|" + r"(?:independent(?:ly)?|独立).{0,60}(?:audit|verify|check|审计|审查|验证|核验|检查)|" + r"(?:audit|verify|check|审计|审查|验证|核验|检查).{0,60}(?:independent(?:ly)?|独立))", + re.I | re.S, +) +_WRITE_OPERATION_PHRASE = ( + r"(?:write|mutating)(?:\s*(?:or|and|/)\s*(?:write|mutating))?\s+operations?" +) +_FALSE_POSITIVE_PHRASE = r"false[-_\s]+positive" +_NO_WRITE_CONSTRAINT_RE = re.compile( + r"read[- ]only|do not (?:edit|modify|change|touch)|" + rf"do not execute {_WRITE_OPERATION_PHRASE}|" + r"leave.{0,40}(?:tree|worktree).{0,30}(?:untouched|unchanged)|" + r"只读|不要(?:编辑|修改|改动|碰)|不要执行.{0,10}写(?:入)?操作|" + r"保持.{0,30}(?:工作树|仓库).{0,30}(?:不变|原样)", + re.I | re.S, +) +_DECISION_RE = re.compile( + r"\bverdict\b|\bpass\s*/\s*fail\b|\bgo\s*/\s*no[- ]go\b|" + r"通过\s*/\s*不通过|放行\s*/\s*阻断|明确结论", + re.I, +) +_EVIDENCE_RE = re.compile(r"\bevidence\b|\bproof\b|证据|依据", re.I) +_OUTPUT_ACTION_RE = re.compile( + r"\b(?:return|report|provide|respond with)\b|输出|返回|给出", re.I +) +_QUOTE_DISCUSSION_RE = re.compile( + r"(?:下面|以下|这段|这份|该段|引用|prompt|提示词|指令文本).{0,100}" + r"(?:(?:不要|请勿|无需)(?:执行|照做|运行)(?!(?:任何)?写(?:入)?操作|命令)|" + r"仅供.{0,30}(?:讨论|分析)|" + r"只是引用|用于.{0,30}(?:误判|假阳性))|" + r"(?:(?:不要|请勿|无需)(?:执行|照做|运行)(?!(?:任何)?写(?:入)?操作|命令)).{0,100}" + r"(?:下面|以下|这段|这份|引用|prompt|提示词|误判|假阳性)|" + r"(?:below|following|this (?:prompt|passage|quote)|quoted text).{0,100}" + rf"(?:do not execute|do not follow|for discussion only|discuss.{{0,30}}{_FALSE_POSITIVE_PHRASE})|" + rf"(?:do not execute(?!\s+{_WRITE_OPERATION_PHRASE})|do not follow).{{0,100}}" + rf"(?:below|following|this (?:prompt|passage|quote)|quoted|{_FALSE_POSITIVE_PHRASE})|" + r"(?:this|the)\s+(?:quote|prompt|passage).{0,40}(?:must|should)\s+not\s+be\s+" + rf"(?:executed|followed).{{0,140}}(?:analy[sz]e|discuss).{{0,50}}(?:a\s+)?{_FALSE_POSITIVE_PHRASE}", + re.I | re.S, +) +_QUOTE_SUPPRESSIBLE_REASONS = { + "subagent_prompt", + "review_prompt", + "agent_prompt", + "structured_noise", +} + + +def _is_structured_orchestration_prompt(sample: str) -> bool: + """Conservatively detect long, protocol-shaped agent work orders.""" + if len(sample) < 50: + return False + if ( + _GENERIC_ASSIGNMENT_RE.search(sample) + and _NO_WRITE_CONSTRAINT_RE.search(sample) + and _DECISION_RE.search(sample) + and _EVIDENCE_RE.search(sample) + and _OUTPUT_ACTION_RE.search(sample) + ): + return True + if _PROTOCOL_PAIR_RE.search(sample): + return True + role_gate = bool(_ORCHESTRATION_ROLE_RE.search(sample)) + task_gate = bool(_TASK_HEADING_RE.search(sample)) + continuation_gate = bool(_CONTINUATION_REVIEW_RE.search(sample)) + if not role_gate and not task_gate and not continuation_gate: + return False + signals = 0 + if role_gate and re.search(r"(?:^|\n)\s*#{1,4}\s+", sample): + signals += 1 + if len(_STRUCTURED_LIST_RE.findall(sample)) >= 2: + signals += 1 + if len(_INLINE_NUMBERED_RE.findall(sample)) >= 2: + signals += 1 + if len(_STRUCTURED_LABEL_RE.findall(sample)) >= 2: + signals += 1 + if _OUTPUT_PROTOCOL_RE.search(sample): + signals += 1 + return signals >= 1 + + +def artifact_reason(text: str = "", title: str = "", include_meta_research: bool = True) -> str: + """Return a stable reason when content is unsafe as durable user evidence.""" + sample = f"{title}\n{text}"[:5000] + quoted_discussion = bool(_QUOTE_DISCUSSION_RE.search(text or "")) + for reason, pattern in _ARTIFACT_PATTERNS: + if pattern.search(sample): + if quoted_discussion and reason in _QUOTE_SUPPRESSIBLE_REASONS: + continue + return reason + if _is_structured_orchestration_prompt(sample): + if quoted_discussion: + return "" + return "structured_orchestration_prompt" + if include_meta_research and _META_RESEARCH_RE.search(text or ""): + return "meta_research_request" + return "" + + +def is_noise_text(text: str) -> bool: + return bool(artifact_reason(text, include_meta_research=False)) + + +def dedupe_message_results(results: list) -> list: + """Fold mirrored copies of the same timestamped message before paging.""" + deduped = [] + by_fingerprint = {} + for row in results: + if row.get("candidateType", "message") != "message": + row.pop("_eventText", None) + deduped.append(row) + continue + text = row.get("_eventText") or row.get("snippet") or "" + date = row.get("date") or "" + if not text or not date: + deduped.append(row) + continue + normalized = " ".join(text.lower().split()) + fingerprint = ( + row.get("role") or row.get("matchType") or "", + normalized, + date, + (row.get("project") or "").strip().lower(), + ) + existing = by_fingerprint.get(fingerprint) + if existing is None: + by_fingerprint[fingerprint] = row + deduped.append(row) + continue + session_id = row.get("sessionId", "") + sessions = existing.setdefault("duplicateSessionIds", []) + if session_id and session_id != existing.get("sessionId") and session_id not in sessions: + sessions.append(session_id) + existing["duplicateCount"] = 1 + len(sessions) + for row in deduped: + row.pop("_eventText", None) + if row.get("candidateType", "message") == "message": + row.setdefault("duplicateCount", 1) + row.setdefault("duplicateSessionIds", []) + return deduped + + +def page_results(results: list, args) -> list: + page = max(int(getattr(args, "page", 1) or 1), 1) + limit = max(int(getattr(args, "limit", 50) or 50), 1) + start = (page - 1) * limit + return results[start:start + limit] diff --git a/chatview/commands/evolve.py b/chatview/commands/evolve.py index 6d90a4e..0e681e1 100644 --- a/chatview/commands/evolve.py +++ b/chatview/commands/evolve.py @@ -8,8 +8,10 @@ from pathlib import Path from chatview.commands.analysis import _get_filtered_db, _get_messages_db +from chatview.commands.evidence import artifact_reason from chatview.commands.corrections import ( _data_corrections, + _data_errors, ) @@ -94,8 +96,26 @@ def _is_profile_decision_text(text): return bool(_PROFILE_DECISION_RE.search(text)) +def _filter_profile_evidence_rows(db_sessions, user_rows): + """Remove rows that must not contribute to durable profile evidence.""" + titles = {session["id"]: session.get("title", "") for session in db_sessions} + return [ + row + for row in user_rows + if not artifact_reason( + row.get("text", ""), titles.get(row.get("session_id", ""), "") + ) + ] + + def _profile_digest_scan_messages(db_sessions, user_rows): """Build decision and highlight summaries from already-fetched user rows.""" + user_rows = _filter_profile_evidence_rows(db_sessions, user_rows) + db_sessions = [ + session + for session in db_sessions + if not artifact_reason(title=session.get("title", "")) + ] session_meta = {s["id"]: s for s in db_sessions} rows_by_session = defaultdict(list) for row in user_rows: @@ -904,7 +924,10 @@ def cmd_profile_digest(args): dates.sort() # Count total queries - all_queries_raw = _get_messages_db(args, role="user", limit=99999) + all_queries_raw = _filter_profile_evidence_rows( + db_sessions, + _get_messages_db(args, role="user", limit=99999), + ) all_decisions, all_highlights = _profile_digest_scan_messages( db_sessions, all_queries_raw ) diff --git a/chatview/commands/retrieval.py b/chatview/commands/retrieval.py index 9cd5323..adc83d3 100644 --- a/chatview/commands/retrieval.py +++ b/chatview/commands/retrieval.py @@ -8,7 +8,17 @@ from datetime import datetime, timedelta from math import log2 -from chatview.commands.search_context import format_grep_window, wants_grep_output +from chatview.commands.evidence import ( + artifact_reason, + dedupe_message_results, + is_noise_text, + page_results, +) +from chatview.snippets import make_query_snippet + + +_SEARCH_HIGH_RETRIEVAL_WINDOW = 20 +_SEARCH_HIGH_EXACT_SKIP_THRESHOLD = 2 _CJK_RE = re.compile(r"[\u4e00-\u9fff]+") @@ -32,26 +42,6 @@ "趋势": ["trend"], } -_NOISE_PATTERNS = [ - ("task_notification", re.compile(r"||toolu_", re.I)), - ("continuation_summary", re.compile(r"This session is being continued from a previous conversation|Summary:\s*\n", re.I)), - ("ide_context", re.compile(r"# Context from my IDE setup|Active file:|Open tabs:", re.I)), - ("retrieval_work_product", re.compile( - r"Pre-collected Data \(do NOT re-run these\)|Full-text search across all sessions Options:|" - r"pre-computed project distribution \+ daily activity as JSON|=== STATS ===", - re.I, - )), - ("agent_prompt", re.compile( - r"You are Agent\b|You are independently|You are review|You are extracting structured|" - r"You have a CLI tool for analyzing conversation history|Acceptance self-test|" - r"Base directory for this skill|你是一个严格、独立的标注员|任务：判断每条|" - r"你是.{0,80}(?:子\s*agent|subagent)|任务：不要写泛泛趋势", - re.I, - )), - ("structured_noise", re.compile(r"(?:^|\n)\s*(?:<|\{|\[Request interrupted|```|toolu_)", re.I)), -] - - def query_tokens(query: str) -> list: """Return deduplicated lexical tokens plus small CJK bigrams and synonyms.""" raw_tokens = [t for t in _SPLIT_RE.split((query or "").strip()) if len(t) >= 2] @@ -73,18 +63,8 @@ def query_tokens(query: str) -> list: return deduped -def is_noise_text(text: str) -> bool: - """Return True when text is likely orchestration/system noise, not user signal.""" - sample = (text or "")[:5000] - return any(pattern.search(sample) for _, pattern in _NOISE_PATTERNS) - - def _noise_reason(text: str) -> str: - sample = (text or "")[:5000] - for reason, pattern in _NOISE_PATTERNS: - if pattern.search(sample): - return reason - return "" + return artifact_reason(text, include_meta_research=False) def _sanitize_fts_token(token: str) -> str: @@ -104,7 +84,11 @@ def _message_fts_rows(match: str, limit: int, eligible: set | None = None) -> li JOIN messages m ON fts.rowid = m.id JOIN sessions s ON m.session_id = s.id WHERE messages_fts MATCH ? - ORDER BY rank + ORDER BY rank, + COALESCE(NULLIF(m.ts, ''), s.date, '') DESC, + m.session_id, + m.idx, + m.id LIMIT ? """ try: @@ -120,7 +104,11 @@ def _message_fts_rows(match: str, limit: int, eligible: set | None = None) -> li JOIN messages m ON fts.rowid = m.id JOIN sessions s ON m.session_id = s.id WHERE m.session_id IN ({placeholders}) AND messages_fts MATCH ? - ORDER BY rank + ORDER BY rank, + COALESCE(NULLIF(m.ts, ''), s.date, '') DESC, + m.session_id, + m.idx, + m.id LIMIT ? """ try: @@ -130,6 +118,8 @@ def _message_fts_rows(match: str, limit: int, eligible: set | None = None) -> li except sqlite3.OperationalError: return [] result = [dict(row) for row in rows] + result.sort(key=lambda row: (row.get("session_id", ""), row.get("idx", 0), row.get("id", 0))) + result.sort(key=lambda row: row.get("ts") or "", reverse=True) result.sort(key=lambda row: row.get("fts_rank", 0)) for row in result: row.pop("fts_rank", None) @@ -146,11 +136,12 @@ def _fts_or_rows(query: str, limit: int, eligible: set | None = None) -> list: return _message_fts_rows(match, limit, eligible) -def _eligible_session_ids(args) -> set: +def _eligible_session_ids(args, prepare: bool = True) -> set: from chatview import db as _db from chatview.commands.analysis import _ensure_project_identity_backfill - _ensure_project_identity_backfill(args) + if prepare: + _ensure_project_identity_backfill(args) days_map = {"1d": 1, "7d": 7, "30d": 30, "90d": 90} max_days = days_map.get(getattr(args, "date", ""), 99999) sessions = _db.get_filtered_sessions( @@ -181,22 +172,7 @@ def _sql_scope_ids(args, eligible: set) -> set | None: def _make_snippet(text: str, query: str, tokens: list, ctx: int = 80) -> str: - lower = (text or "").lower() - probes = [query.lower()] + [t.lower() for t in tokens] - idx = -1 - probe_len = len(query) - for probe in probes: - if not probe: - continue - idx = lower.find(probe) - if idx != -1: - probe_len = len(probe) - break - if idx == -1: - return (text or "")[:180] - start = max(0, idx - ctx) - end = min(len(text), idx + probe_len + ctx) - return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") + return make_query_snippet(text, query, tokens, max_chars=500)["snippet"] def _add_candidate( @@ -207,6 +183,7 @@ def _add_candidate( reason: str, base_score: float, candidate_type: str = "message", + max_chars: int = 500, ): sid = row.get("session_id") or row.get("sessionId") idx = row.get("idx") if candidate_type == "message" else None @@ -228,8 +205,8 @@ def _add_candidate( score += 12 if token_hits and text and all(t.lower() in text.lower() for t in token_hits[:2]): score += 3 - artifact_reason = _noise_reason(f"{title}\n{text}") - if artifact_reason: + artifact = artifact_reason(text, title) + if artifact: score -= 18 reason = f"{reason}:artifact_downranked" @@ -242,10 +219,12 @@ def _add_candidate( existing["score"] = max(existing["_pathScores"].values()) + corroboration_bonus existing["reasons"].add(reason) existing["tokenHits"].update(token_hits) - if artifact_reason and not existing.get("artifactReason"): - existing["artifactReason"] = artifact_reason + if artifact and not existing.get("artifactReason"): + existing["artifactReason"] = artifact + existing["evidenceEligible"] = False return + snippet_data = make_query_snippet(text or title, query, tokens, max_chars=max_chars) item = { "sessionId": sid, "title": title, @@ -256,14 +235,16 @@ def _add_candidate( "idx": idx, "candidateType": candidate_type, "role": role, - "snippet": _make_snippet(text or title, query, tokens), + **snippet_data, + "_eventText": text, "score": score, "_pathScores": {reason: score}, "reasons": {reason}, "tokenHits": set(token_hits), } - if artifact_reason: - item["artifactReason"] = artifact_reason + item["evidenceEligible"] = not bool(artifact) + if artifact: + item["artifactReason"] = artifact candidates[key] = item @@ -319,19 +300,26 @@ def _token_scan_rows(eligible: set, tokens: list, limit: int = 500) -> list: return result[:max(limit, 1)] -def search_plus_data(query: str, args) -> list: +def search_high_data(query: str, args) -> list: """Return ranked hybrid search results with match reasons.""" if not query or len(query.strip()) < 2: return [] from chatview import db as _db - _db.init_db() - eligible = _eligible_session_ids(args) + from chatview.commands.analysis import _prepare_search_db + + _prepare_search_db(args) + eligible = _eligible_session_ids(args, prepare=False) if not eligible: return [] - limit = max(getattr(args, "limit", 50), 1) + # Candidate collection must not depend on the requested page. Otherwise a + # larger page expands the pool, inserts newly discovered high-score rows, + # and makes adjacent pages overlap. + retrieval_window = _SEARCH_HIGH_RETRIEVAL_WINDOW + max_chars = max(min(getattr(args, "max_chars", 500), 2000), 1) + requested_role = getattr(args, "role", "all") sql_scope_ids = _sql_scope_ids(args, eligible) tokens = query_tokens(query) candidates = {} @@ -339,27 +327,34 @@ def search_plus_data(query: str, args) -> list: # 1. Existing exact/AND-ish FTS path. for row in _db.search_fts( query, - limit=max(limit * 6, 100), + limit=retrieval_window * 6, source=getattr(args, "source", "all"), project=getattr(args, "project", ""), min_date=_min_date_for_args(args), + role=requested_role, ): if row["session_id"] in eligible: - _add_candidate(candidates, row, query, tokens, "message_fts", 30) + _add_candidate( + candidates, row, query, tokens, "message_fts", 30, + max_chars=max_chars, + ) # 2. Wider FTS OR path. - for row in _fts_or_rows(query, limit=max(limit * 10, 150), eligible=sql_scope_ids): - if row["session_id"] in eligible: - _add_candidate(candidates, row, query, tokens, "fts_or", 14) + for row in _fts_or_rows(query, limit=retrieval_window * 10, eligible=sql_scope_ids): + if row["session_id"] in eligible and requested_role in ("all", row.get("role")): + _add_candidate( + candidates, row, query, tokens, "fts_or", 14, + max_chars=max_chars, + ) # 3. Title/project FTS path. - for row in _db.search_title_fts( + for row in ([] if requested_role != "all" else _db.search_title_fts( query, - limit=max(limit * 6, 100), + limit=retrieval_window * 6, source=getattr(args, "source", "all"), project=getattr(args, "project", ""), min_date=_min_date_for_args(args), - ): + )): if row["session_id"] not in eligible: continue title_row = { @@ -373,21 +368,26 @@ def search_plus_data(query: str, args) -> list: } _add_candidate( candidates, title_row, query, tokens, "title_project", 22, - candidate_type="session", + candidate_type="session", max_chars=max_chars, ) # 4. Token scan catches compact Chinese phrases and synonym rewrites. # It is the expensive path, so skip it when exact FTS already produced # enough clean user evidence for the requested result size. - if tokens and not _has_enough_clean_exact_user_matches(candidates, query, limit): - for data in _token_scan_rows(eligible, tokens, limit=max(limit * 20, 200)): + if requested_role in ("all", "user") and tokens and not _has_enough_clean_exact_user_matches( + candidates, query, _SEARCH_HIGH_EXACT_SKIP_THRESHOLD + ): + for data in _token_scan_rows(eligible, tokens, limit=retrieval_window * 20): hay = f"{data.get('title', '')} {data.get('project_name', '')} {data.get('text', '')}".lower() hits = [t for t in tokens if t.lower() in hay] if not hits: continue # A single synonym hit is useful for short Chinese queries, but score keeps it below exact FTS. base = 6 if len(hits) == 1 else 10 - _add_candidate(candidates, data, query, tokens, "token_scan", base) + _add_candidate( + candidates, data, query, tokens, "token_scan", base, + max_chars=max_chars, + ) results = [] for item in candidates.values(): @@ -402,10 +402,15 @@ def search_plus_data(query: str, args) -> list: r["score"], 1 if r.get("role") == "user" else 0, r.get("date", ""), + r.get("sessionId", ""), + r.get("candidateType", ""), + -1 if r.get("idx") is None else r.get("idx"), ), reverse=True, ) - return results[:limit] + if getattr(args, "evidence_only", False): + results = [row for row in results if row.get("evidenceEligible", True)] + return page_results(dedupe_message_results(results), args) _LATIN_RE = re.compile(r"[a-z0-9]+", re.I) @@ -1108,22 +1113,20 @@ def evidence_audit_data(args, kind: str = "all") -> dict: return summary -def cmd_search_plus(args): - results = search_plus_data(args.query, args) - if args.json: - print(json.dumps(results, ensure_ascii=False, indent=2)) - return - print(f"Found {len(results)} hybrid matches for '{args.query}':\n") - for row in results: - reasons = ",".join(row.get("reasons", [])) - print(f" [{row.get('score')}] {row.get('date', '')[:10]} {row.get('title', '')[:70]}") - print(f" {row.get('project', '')} · idx:{row.get('messageIndex')} · {reasons}") - print(f" > {row.get('snippet', '')[:220]}") - print(f" session: {row.get('sessionId', '')}") - if wants_grep_output(args) and row.get("candidateType") == "message": - print(f" read-window: distill read-window {row.get('sessionId', '')} --idx {row.get('idx', 0)}") - for line in format_grep_window(row.get("sessionId", ""), row.get("idx", 0), args): - print(line) +def _human_window_text(text: str, max_chars: int = 1200) -> str: + value = (text or "").strip() + if len(value) <= max_chars: + return value + marker = f"\n[… truncated from {len(value)} chars]" + return value[:max(max_chars - len(marker), 0)] + marker[:max_chars] + + +def _print_human_window(window: dict): + print(f"# {window['title']}") + print(f"# {window['project']} | target idx:{window['targetIndex']} radius:{window['radius']}\n") + for msg in window["messages"]: + print(f"--- {msg.get('role', '').upper()} idx:{msg.get('idx')} {msg.get('ts', '')[:16]} ---") + print(_human_window_text(msg.get("text") or "")) print() @@ -1141,13 +1144,7 @@ def cmd_read_window(args): print(json.dumps(data, ensure_ascii=False, indent=2)) return for window in data["windows"]: - print(f"# {window['title']}") - print(f"# {window['project']} | target idx:{window['targetIndex']} radius:{window['radius']}\n") - for msg in window["messages"]: - text = (msg.get("text") or "").strip() - print(f"--- {msg.get('role', '').upper()} idx:{msg.get('idx')} {msg.get('ts', '')[:16]} ---") - print(text[:1200]) - print() + _print_human_window(window) return if not args.session or args.idx is None: @@ -1161,13 +1158,7 @@ def cmd_read_window(args): if args.json: print(json.dumps(data, ensure_ascii=False, indent=2)) return - print(f"# {data['title']}") - print(f"# {data['project']} | target idx:{data['targetIndex']} radius:{data['radius']}\n") - for msg in data["messages"]: - text = (msg.get("text") or "").strip() - print(f"--- {msg.get('role', '').upper()} idx:{msg.get('idx')} {msg.get('ts', '')[:16]} ---") - print(text[:1200]) - print() + _print_human_window(data) def cmd_find_repeats(args): diff --git a/chatview/commands/search_context.py b/chatview/commands/search_context.py index 9379771..824ae78 100644 --- a/chatview/commands/search_context.py +++ b/chatview/commands/search_context.py @@ -1,5 +1,7 @@ """Shared grep-style formatting helpers for search commands.""" +from chatview.snippets import make_query_snippet + def grep_bounds(args): context = max(getattr(args, "context", 0) or 0, 0) @@ -27,11 +29,86 @@ def format_grep_window(session_id, idx, args): for msg in messages: role = msg.get("role", "") msg_idx = msg.get("idx", 0) - text = " ".join((msg.get("text") or "").strip().split()) - if len(text) > 500: - text = text[:500] + "..." + data = make_query_snippet( + msg.get("text") or "", + getattr(args, "query", ""), + max_chars=getattr(args, "max_chars", 500), + ) + text = " ".join(data["snippet"].strip().split()) if getattr(args, "line_number", False): lines.append(f"{session_id}:{msg_idx}:{role}: {text}") else: lines.append(f" {role} idx:{msg_idx}: {text}") return lines + + +def attach_search_context(results: list, args) -> list: + """Attach nested context only when inline context was explicitly requested.""" + from chatview import db as _db + + before, after = grep_bounds(args) + if before == 0 and after == 0: + return results + max_chars = getattr(args, "max_chars", 500) + query = getattr(args, "query", "") + for row in results: + if row.get("candidateType", "message") != "message" or row.get("idx") is None: + continue + messages = _db.get_message_window( + row.get("sessionId", ""), row["idx"] - before, row["idx"] + after + ) + context = [] + for msg in messages: + snippet_data = make_query_snippet( + msg.get("text") or "", query, max_chars=max_chars + ) + context.append({ + "idx": msg.get("idx"), + "role": msg.get("role", ""), + "ts": msg.get("ts", ""), + **snippet_data, + }) + row["context"] = context + return results + + +def format_search_lines(results: list, args) -> list: + """Render grep-style results, folding overlapping inline windows.""" + from chatview import db as _db + + before, after = grep_bounds(args) + max_chars = getattr(args, "max_chars", 500) + query = getattr(args, "query", "") + seen = set() + lines = [] + for row in results: + session_id = row.get("sessionId", "") + idx = row.get("idx") + if row.get("candidateType", "message") != "message" or idx is None: + lines.append(f"{session_id}:session:{row.get('role', 'session')}: {row.get('snippet', '')}") + continue + if before == 0 and after == 0: + messages = [{ + "idx": idx, + "role": row.get("role", ""), + "snippet": row.get("snippet", ""), + }] + else: + messages = [] + for msg in _db.get_message_window(session_id, idx - before, idx + after): + data = make_query_snippet( + msg.get("text") or "", query, max_chars=max_chars + ) + messages.append({ + "idx": msg.get("idx"), + "role": msg.get("role", ""), + "snippet": data["snippet"], + }) + for msg in messages: + key = (session_id, msg.get("idx")) + if key in seen: + continue + seen.add(key) + text = " ".join((msg.get("snippet") or "").strip().split()) + lines.append(f"{session_id}:{msg.get('idx')}:{msg.get('role', '')}: {text}") + return lines diff --git a/chatview/db/__init__.py b/chatview/db/__init__.py index 1b3744b..f5b60a7 100644 --- a/chatview/db/__init__.py +++ b/chatview/db/__init__.py @@ -4,7 +4,17 @@ from chatview.db import get_conn, init_db, upsert_session, ... """ -from .core import get_conn, init_db, query_in_chunks, DB_PATH, CACHE_DIR, begin_bulk, end_bulk, bulk_commit +from .core import ( + get_conn, + init_db, + prepare_search_db, + query_in_chunks, + DB_PATH, + CACHE_DIR, + begin_bulk, + end_bulk, + bulk_commit, +) from .sessions import ( upsert_session, rename_session, @@ -105,6 +115,7 @@ __all__ = [ "get_conn", "init_db", + "prepare_search_db", "query_in_chunks", "DB_PATH", "CACHE_DIR", diff --git a/chatview/db/core.py b/chatview/db/core.py index ddfaad9..d09a6cf 100644 --- a/chatview/db/core.py +++ b/chatview/db/core.py @@ -4,6 +4,7 @@ import sqlite3 import threading from pathlib import Path +from urllib.parse import quote # --------------------------------------------------------------------------- # Configuration @@ -28,9 +29,106 @@ def get_conn() -> sqlite3.Connection: conn.execute("PRAGMA synchronous=NORMAL") conn.execute("PRAGMA busy_timeout=30000") _local.conn = conn + _local.readonly = False return conn +def _close_thread_connection() -> None: + conn = getattr(_local, "conn", None) + if conn is not None: + conn.close() + delattr(_local, "conn") + if hasattr(_local, "readonly"): + delattr(_local, "readonly") + + +def _open_readonly_connection() -> sqlite3.Connection: + if not DB_PATH.is_file(): + raise RuntimeError(f"Search database does not exist: {DB_PATH}") + wal_path = Path(f"{DB_PATH}-wal") + shm_path = Path(f"{DB_PATH}-shm") + wal_nonempty = wal_path.is_file() and wal_path.stat().st_size > 0 + if wal_nonempty and not shm_path.is_file(): + raise RuntimeError( + "Read-only search snapshot is incomplete: a non-empty WAL exists " + "without its SHM file. Run `distill refresh` or checkpoint the " + "database with write access before searching read-only." + ) + immutable = not wal_nonempty + immutable_arg = "&immutable=1" if immutable else "" + uri = f"file:{quote(str(DB_PATH))}?mode=ro{immutable_arg}" + conn = sqlite3.connect(uri, uri=True, check_same_thread=False) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA query_only=ON") + conn.execute("PRAGMA busy_timeout=30000") + _local.conn = conn + _local.readonly = True + return conn + + +def _validate_search_schema(conn: sqlite3.Connection) -> None: + required_tables = {"sessions", "messages", "messages_fts", "sessions_fts"} + rows = conn.execute( + "SELECT name, type, sql FROM sqlite_master WHERE type IN ('table', 'view')" + ).fetchall() + schema = {row["name"]: row for row in rows} + missing_tables = sorted(required_tables - set(schema)) + invalid_fts = [] + for table in ("messages_fts", "sessions_fts"): + row = schema.get(table) + if row is None: + continue + sql = row["sql"] or "" + if row["type"] != "table" or not re.search(r"\bUSING\s+fts5\b", sql, re.I): + invalid_fts.append(table) + + required_columns = { + "sessions": { + "id", "title", "date", "project_name", "project_key", + "project_display", "project_identity_version", "source", + }, + "messages": {"id", "session_id", "idx", "role", "text", "ts"}, + "messages_fts": {"text"}, + "sessions_fts": {"title", "project_name"}, + } + missing_columns = [] + for table, expected in required_columns.items(): + if table in missing_tables: + continue + actual = { + row[1] for row in conn.execute(f"PRAGMA table_info({table})").fetchall() + } + missing_columns.extend(f"{table}.{name}" for name in sorted(expected - actual)) + + if missing_tables or missing_columns or invalid_fts: + details = [] + if missing_tables: + details.append(f"missing tables: {', '.join(missing_tables)}") + if missing_columns: + details.append(f"missing columns: {', '.join(missing_columns)}") + if invalid_fts: + details.append( + "not FTS5 virtual tables: " + ", ".join(sorted(invalid_fts)) + ) + raise RuntimeError( + "Search database schema is incompatible (" + + "; ".join(details) + + "). Run `distill refresh` with write access." + ) + + +def prepare_search_db() -> bool: + """Open and validate search state without writing an existing database.""" + conn = getattr(_local, "conn", None) + if conn is None and DB_PATH.is_file(): + conn = _open_readonly_connection() + elif conn is None: + init_db() + conn = get_conn() + _validate_search_schema(conn) + return bool(getattr(_local, "readonly", False)) + + # --------------------------------------------------------------------------- # Bulk mode — suppress per-row commits during index rebuilds # --------------------------------------------------------------------------- @@ -243,6 +341,10 @@ def _validate_column_definition(definition: str) -> None: # --------------------------------------------------------------------------- def init_db(): """Create all tables and indexes if they don't exist.""" + # A prior search may have opened a query-only connection in this process. + # Explicit initialization is a write intent, so switch back to read/write. + if getattr(_local, "readonly", False): + _close_thread_connection() conn = get_conn() conn.executescript(""" CREATE TABLE IF NOT EXISTS sessions ( diff --git a/chatview/db/sessions.py b/chatview/db/sessions.py index 8fc5cf0..07e741f 100644 --- a/chatview/db/sessions.py +++ b/chatview/db/sessions.py @@ -385,7 +385,11 @@ def search_fts( JOIN messages m ON fts.rowid = m.id JOIN sessions s ON m.session_id = s.id WHERE messages_fts MATCH ?{filter_sql} - ORDER BY rank + ORDER BY rank, + COALESCE(NULLIF(m.ts, ''), s.date, '') DESC, + m.session_id, + m.idx, + m.id LIMIT ? """ safe_query = _sanitize_fts_query(query) @@ -404,7 +408,7 @@ def search_fts( FROM messages m JOIN sessions s ON m.session_id = s.id WHERE m.text LIKE ?{filter_sql} - ORDER BY m.ts DESC + ORDER BY m.ts DESC, m.session_id, m.idx, m.id LIMIT ? """ try: @@ -448,7 +452,7 @@ def search_title_fts( FROM sessions_fts JOIN sessions s ON sessions_fts.rowid = s.rowid WHERE sessions_fts MATCH ?{filter_sql} - ORDER BY rank + ORDER BY rank, COALESCE(s.date, '') DESC, s.id LIMIT ? """ safe_query = _sanitize_fts_query(query) diff --git a/chatview/search.py b/chatview/search.py index d4680de..0fc431a 100644 --- a/chatview/search.py +++ b/chatview/search.py @@ -5,6 +5,7 @@ from chatview import index as _idx from chatview.index import schedule_index_refresh_if_stale +from chatview.snippets import make_query_snippet _QUERY_STOP_WORDS = { @@ -90,7 +91,7 @@ def search_sessions( "project": row.get("project_name", ""), "date": row.get("ts", ""), "messageIndex": row["idx"], - "snippet": _make_snippet(text, query.lower()), + **_make_snippet_data(text, query.lower()), "timestamp": row.get("ts", ""), "matchType": "content", "role": row.get("role", ""), @@ -255,7 +256,7 @@ def _append_anchor_content_matches( "project": row["project_name"] or "", "date": row["ts"] or "", "messageIndex": row["idx"], - "snippet": _make_snippet(text, query_lower, _tokenize_query(query_lower)), + **_make_snippet_data(text, query_lower, _tokenize_query(query_lower)), "timestamp": row["ts"] or "", "matchType": "content", "role": row["role"] or "", @@ -322,7 +323,7 @@ def _append_adjacent_assistant_matches( "project": item["project"], "date": assistant["ts"] or item.get("date", ""), "messageIndex": assistant["idx"], - "snippet": _make_snippet( + **_make_snippet_data( assistant_text, query_lower, _meaningful_query_tokens(query_lower) ), "timestamp": assistant["ts"] or item.get("timestamp", ""), @@ -397,7 +398,7 @@ def _append_fuzzy_content_matches( "project": row["project_name"] or "", "date": row["ts"] or "", "messageIndex": row["idx"], - "snippet": _make_snippet(text, query_lower, tokens), + **_make_snippet_data(text, query_lower, tokens), "timestamp": row["ts"] or "", "matchType": "content", "role": row["role"] or "", @@ -508,20 +509,10 @@ def _dedupe_tokens(tokens: list) -> list: return out +def _make_snippet_data(text: str, query: str, tokens: list = None) -> dict: + return make_query_snippet(text, query, tokens, max_chars=500) + + def _make_snippet(text: str, query: str, tokens: list = None, ctx: int = 80) -> str: - """Create a context snippet around the first match.""" - idx = text.lower().find(query) - if idx == -1 and tokens: - # Find first matching token for snippet context - for t in tokens: - idx = text.lower().find(t) - if idx != -1: - break - if idx == -1: - return text[:160] - start = max(0, idx - ctx) - end = min(len(text), idx + len(query) + ctx) - snippet = ( - ("…" if start > 0 else "") + text[start:end] + ("…" if end < len(text) else "") - ) - return snippet + """Compatibility wrapper for callers that only need the canonical text.""" + return _make_snippet_data(text, query, tokens)["snippet"] diff --git a/chatview/snippets.py b/chatview/snippets.py new file mode 100644 index 0000000..da0cd5d --- /dev/null +++ b/chatview/snippets.py @@ -0,0 +1,86 @@ +"""Canonical query-centered snippets shared by CLI and web search.""" + +from __future__ import annotations + + +def _match_bounds(text: str, query: str, tokens: list | None) -> tuple[int, int]: + lower = text.lower() + probes = [query] + list(tokens or []) + seen = set() + for probe in probes: + probe = (probe or "").strip() + key = probe.lower() + if not key or key in seen: + continue + seen.add(key) + start = lower.find(key) + if start >= 0: + return start, start + len(probe) + return -1, -1 + + +def make_query_snippet( + text: str, + query: str = "", + tokens: list | None = None, + max_chars: int = 500, +) -> dict: + """Return one bounded snippet plus explicit truncation metadata.""" + value = text or "" + max_chars = max(int(max_chars or 500), 1) + match_start, match_end = _match_bounds(value, query or "", tokens) + + if match_start < 0: + if len(value) <= max_chars: + snippet = value + elif max_chars == 1: + snippet = "…" + else: + snippet = value[:max_chars - 1] + "…" + return { + "snippet": snippet, + "snippetTruncated": len(value) > len(snippet), + "originalChars": len(value), + "matchStart": None, + } + + match = value[match_start:match_end] + if len(match) >= max_chars: + snippet = match[:max_chars] + return { + "snippet": snippet, + "snippetTruncated": len(value) > len(snippet), + "originalChars": len(value), + "matchStart": match_start, + } + + left_count = min(200, match_start) + right_available = len(value) - match_end + right_count = min(200, right_available) + + def bounds() -> tuple[int, int]: + return match_start - left_count, match_end + right_count + + def render() -> str: + start, end = bounds() + return ( + ("…" if start > 0 else "") + + value[start:end] + + ("…" if end < len(value) else "") + ) + + snippet = render() + while len(snippet) > max_chars and (left_count or right_count): + if left_count >= right_count and left_count: + left_count -= 1 + elif right_count: + right_count -= 1 + snippet = render() + + start, end = bounds() + return { + "snippet": snippet[:max_chars], + "snippetTruncated": start > 0 or end < len(value), + "originalChars": len(value), + "matchStart": match_start, + } diff --git a/evals/retrieval/baseline.json b/evals/retrieval/baseline.json index 7f86208..5a6111d 100644 --- a/evals/retrieval/baseline.json +++ b/evals/retrieval/baseline.json @@ -1,6 +1,7 @@ { - "dataset_sha256": "ebb6b2b9ff5b225d77f37b0467d4fbc46a7ac3af204acbbc7f9112aa3f2d5203", + "dataset_sha256": "ff952647bcc59a711844b585b21049de91774b5197e7ba7f5eb95db508de6106", "generated_from_commit": "652ef16", + "dataset_migration": "Only the command label changed from the removed legacy entry point to search-high; queries, sessions, relevance labels, and thresholds are unchanged.", "dev": { "cases": 12, "passed": 10, diff --git a/evals/retrieval/cases.json b/evals/retrieval/cases.json index 0aa52d6..9ebbe4f 100644 --- a/evals/retrieval/cases.json +++ b/evals/retrieval/cases.json @@ -34,29 +34,29 @@ {"id":"holdout-schema-artifact","title":"You are Agent 7 reviewing schema rollback","project":"eval/schema","source":"codex","age_days":1,"messages":[{"idx":0,"role":"user","text":"数据库字段变更必须准备回滚路径。"}]} ], "cases": [ - {"id":"dev-exact-zh","split":"dev","family":"exact_phrase","command":"search-plus","query":"保持实现简洁","relevant":["dev-zh-clean"],"top_role":"user"}, - {"id":"dev-echo-zh","split":"dev","family":"assistant_echo","command":"search-plus","query":"不要引入额外层级","relevant":["dev-zh-clean"],"top_role":"user"}, - {"id":"dev-artifact-zh","split":"dev","family":"artifact_ranking","command":"search-plus","query":"保持实现简洁 不要引入额外层级","relevant":["dev-zh-clean"],"top_role":"user","artifact_top1_forbidden":true}, - {"id":"dev-concept-memory","split":"dev","family":"concept_zh","command":"search-plus","query":"记忆写入需要确认","relevant":["dev-memory-clean"]}, - {"id":"dev-artifact-memory","split":"dev","family":"artifact_ranking","command":"search-plus","query":"长期记忆写入前确认","relevant":["dev-memory-clean"],"artifact_top1_forbidden":true}, - {"id":"dev-concept-browser","split":"dev","family":"concept_zh","command":"search-plus","query":"前端截图验证","relevant":["dev-browser-clean"],"artifact_top1_forbidden":true}, - {"id":"dev-identifier-positive","split":"dev","family":"identifier","command":"search-plus","query":"cache_key_v9f3_alpha","relevant":["dev-identifier-positive"]}, - {"id":"dev-identifier-empty","split":"dev","family":"abstention","command":"search-plus","query":"cache_key_q7x9_unseen","expected_empty":true}, - {"id":"dev-title-only","split":"dev","family":"title_only","command":"search-plus","query":"AuroraGraph","relevant":["dev-title-only"]}, - {"id":"dev-project-scope","split":"dev","family":"scope","command":"search-plus","query":"scope_marker_delta","project":"eval/alpha","relevant":["dev-scope-target","dev-scope-source-noise"],"allowed_projects":["eval/alpha"]}, - {"id":"dev-source-scope","split":"dev","family":"scope","command":"search-plus","query":"scope_marker_delta","project":"eval/alpha","source":"codex","relevant":["dev-scope-target"],"allowed_sources":["codex"]}, + {"id":"dev-exact-zh","split":"dev","family":"exact_phrase","command":"search-high","query":"保持实现简洁","relevant":["dev-zh-clean"],"top_role":"user"}, + {"id":"dev-echo-zh","split":"dev","family":"assistant_echo","command":"search-high","query":"不要引入额外层级","relevant":["dev-zh-clean"],"top_role":"user"}, + {"id":"dev-artifact-zh","split":"dev","family":"artifact_ranking","command":"search-high","query":"保持实现简洁 不要引入额外层级","relevant":["dev-zh-clean"],"top_role":"user","artifact_top1_forbidden":true}, + {"id":"dev-concept-memory","split":"dev","family":"concept_zh","command":"search-high","query":"记忆写入需要确认","relevant":["dev-memory-clean"]}, + {"id":"dev-artifact-memory","split":"dev","family":"artifact_ranking","command":"search-high","query":"长期记忆写入前确认","relevant":["dev-memory-clean"],"artifact_top1_forbidden":true}, + {"id":"dev-concept-browser","split":"dev","family":"concept_zh","command":"search-high","query":"前端截图验证","relevant":["dev-browser-clean"],"artifact_top1_forbidden":true}, + {"id":"dev-identifier-positive","split":"dev","family":"identifier","command":"search-high","query":"cache_key_v9f3_alpha","relevant":["dev-identifier-positive"]}, + {"id":"dev-identifier-empty","split":"dev","family":"abstention","command":"search-high","query":"cache_key_q7x9_unseen","expected_empty":true}, + {"id":"dev-title-only","split":"dev","family":"title_only","command":"search-high","query":"AuroraGraph","relevant":["dev-title-only"]}, + {"id":"dev-project-scope","split":"dev","family":"scope","command":"search-high","query":"scope_marker_delta","project":"eval/alpha","relevant":["dev-scope-target","dev-scope-source-noise"],"allowed_projects":["eval/alpha"]}, + {"id":"dev-source-scope","split":"dev","family":"scope","command":"search-high","query":"scope_marker_delta","project":"eval/alpha","source":"codex","relevant":["dev-scope-target"],"allowed_sources":["codex"]}, {"id":"dev-repeat-memory","split":"dev","family":"find_repeats","command":"find-repeats","query":"记忆写入前需要确认","relevant":["dev-memory-clean"],"expected_bucket":"strong_evidence"}, - {"id":"holdout-exact-en","split":"holdout","family":"exact_phrase","command":"search-plus","query":"reversible changes before broad refactors","relevant":["holdout-en-clean"],"top_role":"user"}, - {"id":"holdout-artifact-en","split":"holdout","family":"artifact_ranking","command":"search-plus","query":"Prefer reversible changes","relevant":["holdout-en-clean"],"artifact_top1_forbidden":true}, - {"id":"holdout-concept-dependency","split":"holdout","family":"concept_zh","command":"search-plus","query":"依赖选择要优先复用已有能力","relevant":["holdout-dependency-clean","holdout-dependency-related"]}, - {"id":"holdout-mixed-language","split":"holdout","family":"mixed_language","command":"search-plus","query":"Playwright 验证 checkout","relevant":["holdout-mixed-clean"],"artifact_top1_forbidden":true}, - {"id":"holdout-identifier-positive","split":"holdout","family":"identifier","command":"search-plus","query":"release-channel-zeta-42","relevant":["holdout-identifier-positive"]}, - {"id":"holdout-identifier-empty","split":"holdout","family":"abstention","command":"search-plus","query":"release-channel-unseen-991","expected_empty":true}, - {"id":"holdout-title-only","split":"holdout","family":"title_only","command":"search-plus","query":"NebulaCache","relevant":["holdout-title-only"]}, - {"id":"holdout-date-scope","split":"holdout","family":"scope","command":"search-plus","query":"date_scope_omega","date":"7d","relevant":["holdout-date-recent"],"max_age_days":7}, - {"id":"holdout-echo-clones","split":"holdout","family":"assistant_echo","command":"search-plus","query":"Prefer reversible changes before broad refactors","relevant":["holdout-en-clean"],"top_role":"user","clone_session":"holdout-en-echo","clone_count":20}, - {"id":"holdout-artifact-clones","split":"holdout","family":"artifact_ranking","command":"search-plus","query":"Playwright verify checkout flow","relevant":["holdout-mixed-clean"],"artifact_top1_forbidden":true,"clone_session":"holdout-mixed-artifact","clone_count":20}, + {"id":"holdout-exact-en","split":"holdout","family":"exact_phrase","command":"search-high","query":"reversible changes before broad refactors","relevant":["holdout-en-clean"],"top_role":"user"}, + {"id":"holdout-artifact-en","split":"holdout","family":"artifact_ranking","command":"search-high","query":"Prefer reversible changes","relevant":["holdout-en-clean"],"artifact_top1_forbidden":true}, + {"id":"holdout-concept-dependency","split":"holdout","family":"concept_zh","command":"search-high","query":"依赖选择要优先复用已有能力","relevant":["holdout-dependency-clean","holdout-dependency-related"]}, + {"id":"holdout-mixed-language","split":"holdout","family":"mixed_language","command":"search-high","query":"Playwright 验证 checkout","relevant":["holdout-mixed-clean"],"artifact_top1_forbidden":true}, + {"id":"holdout-identifier-positive","split":"holdout","family":"identifier","command":"search-high","query":"release-channel-zeta-42","relevant":["holdout-identifier-positive"]}, + {"id":"holdout-identifier-empty","split":"holdout","family":"abstention","command":"search-high","query":"release-channel-unseen-991","expected_empty":true}, + {"id":"holdout-title-only","split":"holdout","family":"title_only","command":"search-high","query":"NebulaCache","relevant":["holdout-title-only"]}, + {"id":"holdout-date-scope","split":"holdout","family":"scope","command":"search-high","query":"date_scope_omega","date":"7d","relevant":["holdout-date-recent"],"max_age_days":7}, + {"id":"holdout-echo-clones","split":"holdout","family":"assistant_echo","command":"search-high","query":"Prefer reversible changes before broad refactors","relevant":["holdout-en-clean"],"top_role":"user","clone_session":"holdout-en-echo","clone_count":20}, + {"id":"holdout-artifact-clones","split":"holdout","family":"artifact_ranking","command":"search-high","query":"Playwright verify checkout flow","relevant":["holdout-mixed-clean"],"artifact_top1_forbidden":true,"clone_session":"holdout-mixed-artifact","clone_count":20}, {"id":"holdout-repeat-dependency","split":"holdout","family":"find_repeats","command":"find-repeats","query":"新增依赖前要评估复用和维护成本","relevant":["holdout-dependency-clean","holdout-dependency-related"],"expected_bucket":"strong_evidence"}, {"id":"holdout-repeat-schema","split":"holdout","family":"find_repeats","command":"find-repeats","query":"数据结构变更需要可执行回滚方案","relevant":["holdout-schema-clean-a","holdout-schema-clean-b"],"expected_bucket":"strong_evidence","artifact_top1_forbidden":true} ] diff --git a/scripts/run_retrieval_eval.py b/scripts/run_retrieval_eval.py index 5b7cd5e..5ba8b91 100644 --- a/scripts/run_retrieval_eval.py +++ b/scripts/run_retrieval_eval.py @@ -20,7 +20,7 @@ sys.path.insert(0, str(ROOT)) from chatview import db # noqa: E402 -from chatview.commands.retrieval import find_repeats_data, search_plus_data # noqa: E402 +from chatview.commands.retrieval import find_repeats_data, search_high_data # noqa: E402 from chatview.db import core as dbcore # noqa: E402 @@ -119,11 +119,15 @@ def args_for(case): before=None, after=None, line_number=False, + page=1, + evidence_only=False, + role="all", + max_chars=500, ) def search_case(case): - result = search_plus_data(case["query"], args_for(case)) + result = search_high_data(case["query"], args_for(case)) ids = unique_ids([row.get("sessionId", "") for row in result]) relevant = set(case.get("relevant", [])) expected_empty = bool(case.get("expected_empty")) diff --git a/skills/distill-yourself/SKILL.md b/skills/distill-yourself/SKILL.md index 509fe5c..7c36ca7 100644 --- a/skills/distill-yourself/SKILL.md +++ b/skills/distill-yourself/SKILL.md @@ -15,6 +15,7 @@ description: "本地 Claude Code/Codex/agent 对话历史的唯一检索入口 - `distill` 只负责取数、检索和暂存；结论由你基于证据生成。 - `profile-digest` / `aggregates` / `stats` 是地图，不是结论。写入 Memory/Profile/Twin 前必须用 `read-window`、`session-brief` 或原 session 内容核验。 - 不把 assistant echo、IDE/file context、task notification、agent/subagent prompt、工具输出噪声当作主证据。 +- 蒸馏长期偏好、Memory、Rules 或 Patterns 时，检索命令加 `--evidence-only`；普通历史定位不要加，以免隐藏诊断线索。 - Memory/Profile 的标准流程止于 `distill evolve-write`。告诉用户结果已暂存，并由用户在 UI 中预览、确认和同步；不要代替用户运行 `distill evolve-sync --execute`。 - 只有用户明确要求使用 CLI 同步时，才把 `distill evolve-sync` 作为后备入口，并仍须先展示完整预览/diff、再次取得明确确认。不要手搓 Claude/Codex 配置格式。 - Twin 写回仍走 `distill twin-sync`；任何对 `~/.claude/`、`~/.codex/AGENTS.md` 或 `~/.agents/skills/` 的写入都必须先展示完整预览/diff，并得到用户明确确认。 @@ -31,17 +32,17 @@ description: "本地 Claude Code/Codex/agent 对话历史的唯一检索入口 ```text 0. INDEX -> refresh local SQLite/cache 1. ORIENT -> load digest/aggregates/stats -2. EXPLORE -> route to search/search-plus/find-repeats/read-window +2. EXPLORE -> search, select candidates, then read-window 3. DISTILL -> produce evidence-backed insight 4. STAGE -> persist approved Memory/Profile results with evolve-write 5. HANDOFF -> user previews and confirms configuration sync in the UI ``` -Choose the scope before retrieval. For an exact fact, phrase, session, or project episode, use the fast path: refresh once, retrieve at most 10 candidates, then verify only the best 2-4 windows. Do not load digest/aggregates/stats for a simple lookup. +Choose the scope before retrieval. For an exact fact, phrase, session, or project episode, use the fast path: refresh once, retrieve one page of at most 20 candidates, then verify only the best 2-4 windows. Do not load digest/aggregates/stats for a simple lookup. ```bash distill refresh -distill search-plus "" --date 90d --limit 10 --json +distill search "" --recall normal --role user --evidence-only --format jsonl --max-chars 500 --limit 20 --page 1 --date 90d distill read-window --batch '[{"sessionId":"session_id","idx":123,"radius":2}]' ``` @@ -70,8 +71,8 @@ Expand to `all` only when evidence is thin. | Need | First tool | Follow-up | |---|---|---| -| Exact preference phrase or correction | `distill search "" --role user --json --limit 10` | `distill search-plus "" --limit 10 --json`, then `read-window --batch` for top direct user hits | -| Conceptual preference | Split into 2-4 concrete anchors, then `search` | `search-plus` if direct user evidence is thin | +| Exact preference phrase or correction | `distill search "" --recall normal --role user --evidence-only --format jsonl --limit 20 --page 1` | Select 2-4 direct user hits, then `read-window --batch` | +| Conceptual preference | Split into 2-4 concrete anchors, then run `search --recall normal` for each | If direct user evidence is thin, repeat with `--recall high`, then page | | Topic, project, or historical episode | `distill find-repeats "" --limit 5 --json` | `session-brief` / `read-window --batch` for top candidates | | Possible orchestration noise | `distill evidence-audit --json` | Treat `artifactReason` rows as diagnostic only | | Session-level context | `distill session-brief ` | `distill read-window --idx N` | @@ -83,20 +84,23 @@ Important commands: | `distill corrections --limit 100` | Correction anchors with stable `idx` and the nearest user/assistant pair in `conversation` | | `distill queries --limit 50` | User requests, including possible acceptance signals | | `distill highlights --limit 20` | High-signal sessions ranked by correction/decision density | -| `distill search "" -C 2 -n` | Grep-style context lines as `session:idx:role`, ready for `read-window` | -| `distill search-plus "" --limit 10 --json` | Hybrid retrieval with match reasons, artifact downranking, and `idx` | -| `distill search-plus "" -C 2 -n` | Grep-style hybrid results when manual triage is faster than JSON | +| `distill search "" --recall normal --format lines --limit 20 --page 1` | Compact grep-style exact results with `session:idx:role` locators | +| `distill search "" --recall high --format jsonl --limit 20 --page 1` | Hybrid high-recall results with match reasons, artifact downranking, and `idx` | | `distill find-repeats "" --limit 5 --json` | Evidence buckets: strong, related, weak, artifacts | | `distill read-window --idx N --radius 2` | Small context window around a hit | | `distill read-window --batch '[...]'` | Verify several windows at once | | `distill evidence-audit --json` | Estimate contamination from prompts/tasks/context noise | -`corrections` JSON exposes a stable `idx` and its nearest user/assistant pair. `search` and `search-plus` JSON expose both `idx` and `messageIndex`; prefer `idx` when building `read-window` commands. If an older retrieval result exposes only `messageIndex`, map it to `idx`: +`corrections` JSON exposes a stable `idx` and its nearest user/assistant pair. `search` JSON/JSONL exposes `sessionId`, `idx`, and `messageIndex`; use `sessionId + idx` as the read coordinate. `idx` is only ordered within one session, not a global message id. `read-window --radius N` reads the inclusive numeric range `idx-N ... idx+N`, so gaps may produce fewer than `2N+1` messages. ```bash distill read-window --batch '[{"sessionId":"session_id","idx":123,"radius":2}]' ``` +For manual triage, prefer `--format lines`; for scripts or incremental aggregation, prefer `--format jsonl`. `--limit N` is the page size and `--page 2`, `--page 3`, ... retrieves later pages. Keep the standard page size at 20 and page instead of requesting one oversized result. Mirrored copies of the same timestamped message are folded before paging, while `duplicateCount` / `duplicateSessionIds` preserve provenance in structured output. + +`-C/-B/-A` are optional inline previews for a human terminal, not the Agent verification path. Search without inline context, select a few locators, then use `read-window`. Human `read-window` output is bounded and marks truncation; `--json` returns the complete text stored in the local index, which may already be shorter than the original transcript because ingestion has its own limits. + Stop retrieval when you have enough direct user evidence. More searching after 2-4 strong, verified quotes usually adds noise. ## Required References diff --git a/skills/distill-yourself/evals/evals.json b/skills/distill-yourself/evals/evals.json new file mode 100644 index 0000000..fe8eb96 --- /dev/null +++ b/skills/distill-yourself/evals/evals.json @@ -0,0 +1,28 @@ +{ + "skill_name": "distill-yourself", + "evals": [ + { + "id": 1, + "prompt": "我要从历史对话里找出我明确说过‘先确认风险’的原话。现在只写出你会执行的检索和核验命令，不要真正运行。", + "expected_output": "先用 search --recall normal --role user --evidence-only --format jsonl --limit 20 --page 1 定位，再选少量 sessionId+idx 用 read-window 核验；不使用 search-plus 或默认 -C。", + "files": [], + "expectations": [ + "Uses search --recall normal with role=user, evidence-only, JSONL, limit 20, and page 1", + "Uses sessionId plus idx with read-window after selecting candidates", + "Does not use search-plus or inline -C as the default path" + ] + }, + { + "id": 2, + "prompt": "我想分析自己是否反复要求 Agent 不要过度设计、坚持最小改动。现在只给检索 SOP 和停止条件，不要真正检索。", + "expected_output": "把概念拆成具体 anchors，各自先 normal；直接用户证据不足再 high 并分页；选中候选后 read-window；至少两条、两个 session 的直接用户证据才形成长期结论。", + "files": [], + "expectations": [ + "Splits the concept into concrete anchors and starts with normal recall", + "Escalates to high recall only when direct evidence is thin and pages instead of inflating limit", + "Requires direct user evidence from at least two sessions and verifies selected windows", + "Does not use search-plus, --limit 10, or default inline context" + ] + } + ] +} diff --git a/skills/distill-yourself/references/rules-signals-patterns.md b/skills/distill-yourself/references/rules-signals-patterns.md index 83d3d4f..d313373 100644 --- a/skills/distill-yourself/references/rules-signals-patterns.md +++ b/skills/distill-yourself/references/rules-signals-patterns.md @@ -19,8 +19,8 @@ Use the same `--source`, `--date`, and `--project` filters the user requested. I ## How to Report 1. Run `distill refresh`. -2. Run the relevant deterministic command(s). -3. Inspect evidence references from the output. +2. Locate candidate corrections with `distill search "" --recall normal --role user --evidence-only --format jsonl --limit 20 --page 1`. +3. Run the relevant deterministic command(s), then inspect their evidence references. 4. For high-impact rules, verify 1-3 representative quotes with `read-window` before presenting as durable behavior. 5. Report findings with confidence and scope. @@ -32,6 +32,8 @@ Use the same `--source`, `--date`, and `--project` filters the user requested. I Do not overstate deterministic output as full AI analysis. These commands are useful aggregations; you still need judgment and evidence checks. +The correction extractor excludes known task notifications, continuation summaries, IDE context, agent/subagent prompts, retrieval work products, and explicit requests to research conversation history. Search remains inclusive by default for diagnosis; `--evidence-only` applies the stricter durable-evidence view. Split conceptual questions into concrete anchors, start with `--recall normal`, and use `--recall high` only when direct evidence remains thin. Page JSONL results instead of requesting one oversized JSON array. + ## When to Convert to Memory If the user wants to keep a rule, switch to `references/memory-profile.md` and follow the Memory writeback SOP. Do not write directly from rules output. diff --git a/skills/distill-yourself/references/twin-cognitive-model.md b/skills/distill-yourself/references/twin-cognitive-model.md index 56ac7a7..34f19d6 100644 --- a/skills/distill-yourself/references/twin-cognitive-model.md +++ b/skills/distill-yourself/references/twin-cognitive-model.md @@ -87,12 +87,12 @@ Required order: distill read-window --idx --radius 2 ``` - `highlights` and `queries` may still require session-level reading. Current `search` / `search-plus` JSON expose `idx` plus `messageIndex`; prefer `idx`. For quick manual triage, use grep-style context and copy the displayed `session:idx` pair: + `highlights` and `queries` may still require session-level reading. Search JSONL exposes `sessionId`, `idx`, and `messageIndex`; use `sessionId + idx` as the read coordinate. For quick manual triage, locate candidates first, then deep-read only survivors: ```bash distill session-brief distill read -s - distill search-plus "" -C 2 -n + distill search "" --recall high --role user --evidence-only --format jsonl --limit 20 --page 1 ``` Do not invent an idx. For older results that expose only `messageIndex`, map it to `idx`: diff --git a/tests/test_commands_analysis.py b/tests/test_commands_analysis.py index 8770d65..36a07cd 100644 --- a/tests/test_commands_analysis.py +++ b/tests/test_commands_analysis.py @@ -34,6 +34,11 @@ def _default_args(**kwargs): before=None, after=None, line_number=False, + format="standard", + page=1, + evidence_only=False, + recall="normal", + max_chars=500, ) defaults.update(kwargs) return SimpleNamespace(**defaults) @@ -349,6 +354,61 @@ def test_cmd_search_json_includes_idx_for_read_window(self): data = json.loads(out.getvalue()) self.assertEqual(data[0]["messageIndex"], 2) self.assertEqual(data[0]["idx"], 2) + self.assertEqual(data[0]["duplicateCount"], 1) + self.assertEqual(data[0]["duplicateSessionIds"], []) + + def test_query_centered_snippet_is_identical_in_lines_and_jsonl(self): + from chatview import db + from chatview.commands.analysis import cmd_search + + db.upsert_session( + meta={ + "id": "long-match-session", + "title": "Long match", + "date": "2026-06-02T10:00:00Z", + "lastDate": "2026-06-02T10:00:00Z", + "filePath": "/tmp/long-match.jsonl", + "fileSize": 1024, + "_mtime": 0, + "userMessageCount": 1, + "preview": "long", + "project": "grep-proj", + "projectName": "grep-proj", + "source": "claude", + }, + user_texts=[{ + "idx": 0, + "text": "前" * 800 + "中段关键词" + "后" * 800, + "ts": "2026-06-02T10:00:00Z", + }], + assistant_snippets=[], + ) + + jsonl_out = io.StringIO() + with contextlib.redirect_stdout(jsonl_out): + cmd_search(_default_args(query="中段关键词", format="jsonl")) + record = json.loads(jsonl_out.getvalue()) + + lines_out = io.StringIO() + with contextlib.redirect_stdout(lines_out): + cmd_search(_default_args(query="中段关键词", format="lines")) + line_snippet = lines_out.getvalue().split(": ", 1)[1].strip() + + self.assertEqual(line_snippet, record["snippet"]) + self.assertIn("中段关键词", record["snippet"]) + self.assertEqual(record["matchStart"], 800) + self.assertLessEqual(len(record["snippet"]), 500) + + def test_jsonl_context_is_nested_under_candidate(self): + from chatview.commands.analysis import cmd_search + + out = io.StringIO() + with contextlib.redirect_stdout(out): + cmd_search(_default_args(query="needle-token", format="jsonl", context=1)) + + record = json.loads(out.getvalue()) + self.assertEqual([item["idx"] for item in record["context"]], [1, 2]) + self.assertTrue(all("snippetTruncated" in item for item in record["context"])) def test_cmd_search_context_prints_grep_style_window(self): from chatview.commands.analysis import cmd_search @@ -369,6 +429,207 @@ def test_cmd_search_context_prints_grep_style_window(self): self.assertIn("Assistant context line", text) self.assertIn("needle-token", text) + def test_cmd_search_lines_format_is_compact_grep_output(self): + from chatview.commands.analysis import cmd_search + + out = io.StringIO() + with contextlib.redirect_stdout(out): + cmd_search( + _default_args( + query="needle-token", + format="lines", + context=1, + line_number=True, + ) + ) + + text = out.getvalue() + self.assertIn("search-sess-001:1:assistant", text) + self.assertIn("search-sess-001:2:user", text) + self.assertNotIn("Found ", text) + self.assertNotIn("read-window:", text) + + def test_cmd_search_evidence_only_filters_agent_artifacts(self): + from chatview import db + from chatview.commands.analysis import cmd_search + + db.upsert_session( + meta={ + "id": "search-agent-artifact", + "title": "You are Agent 1 reviewing history", + "date": "2026-06-01T10:00:00Z", + "lastDate": "2026-06-01T10:00:00Z", + "filePath": "/tmp/search-agent-artifact.jsonl", + "fileSize": 10, + "_mtime": 1, + "userMessageCount": 1, + "preview": "needle-token", + "project": "grep-proj", + "projectName": "grep-proj", + "source": "claude", + }, + user_texts=[{ + "idx": 0, + "text": "needle-token should not become preference evidence", + "ts": "2026-06-01T10:00:00Z", + }], + assistant_snippets=[], + ) + + out = io.StringIO() + with contextlib.redirect_stdout(out): + cmd_search(_default_args(query="needle-token", json=True)) + default_ids = [row["sessionId"] for row in json.loads(out.getvalue())] + self.assertIn("search-agent-artifact", default_ids) + + out = io.StringIO() + with contextlib.redirect_stdout(out): + cmd_search(_default_args(query="needle-token", json=True, evidence_only=True)) + + data = json.loads(out.getvalue()) + self.assertEqual([row["sessionId"] for row in data], ["search-sess-001"]) + + def test_cmd_search_normal_stops_after_first_progressive_batch(self): + from chatview.commands.analysis import cmd_search + + rows = [ + { + "session_id": f"progressive-clean-{index}", + "title": "Clean search result", + "project_name": "same-project", + "project_display": "same-project", + "source": "codex", + "idx": index, + "role": "user", + "text": f"event clean evidence {index}", + "ts": f"2026-07-01T10:{index:02d}:00Z", + } + for index in range(30) + ] + + with patch( + "chatview.db.search_fts", + side_effect=lambda _query, limit, **_kwargs: rows[:limit], + ) as search_fts: + with contextlib.redirect_stdout(io.StringIO()): + cmd_search( + _default_args( + query="event", json=True, evidence_only=True, limit=20 + ) + ) + + self.assertEqual(search_fts.call_count, 1) + self.assertEqual(search_fts.call_args.kwargs["limit"], 64) + + def test_cmd_search_normal_expands_only_when_clean_results_are_insufficient(self): + from chatview.commands.analysis import cmd_search + + artifacts = [ + { + "session_id": f"progressive-artifact-{index}", + "title": "You are Agent 1 reviewing history", + "project_name": "same-project", + "project_display": "same-project", + "source": "codex", + "idx": index, + "role": "user", + "text": f"event artifact {index}", + "ts": f"2026-07-01T09:{index:02d}:00Z", + } + for index in range(50) + ] + clean = [ + { + "session_id": f"progressive-clean-{index}", + "title": "Clean search result", + "project_name": "same-project", + "project_display": "same-project", + "source": "codex", + "idx": index, + "role": "user", + "text": f"event clean evidence {index}", + "ts": f"2026-07-01T10:{index:02d}:00Z", + } + for index in range(30) + ] + rows = artifacts + clean + + out = io.StringIO() + with patch( + "chatview.db.search_fts", + side_effect=lambda _query, limit, **_kwargs: rows[:limit], + ) as search_fts: + with contextlib.redirect_stdout(out): + cmd_search( + _default_args( + query="event", json=True, evidence_only=True, limit=20 + ) + ) + + self.assertEqual( + [call.kwargs["limit"] for call in search_fts.call_args_list], [64, 160] + ) + data = json.loads(out.getvalue()) + self.assertEqual(len(data), 20) + self.assertTrue(all(row["sessionId"].startswith("progressive-clean-") for row in data)) + + def test_cmd_search_deduplicates_mirrored_events_before_paging(self): + from chatview.commands.analysis import cmd_search + + rows = [ + { + "session_id": sid, + "title": f"Mirror {sid}", + "project_name": "same-project", + "project_display": "same-project", + "source": "codex", + "idx": 3, + "role": "user", + "text": "same mirrored event", + "ts": "2026-07-01T10:00:00Z", + } + for sid in ("mirror-a", "mirror-b") + ] + rows.append({ + "session_id": "later-event", + "title": "Later", + "project_name": "same-project", + "project_display": "same-project", + "source": "codex", + "idx": 4, + "role": "user", + "text": "a distinct event", + "ts": "2026-07-01T11:00:00Z", + }) + + out = io.StringIO() + with patch("chatview.db.search_fts", return_value=rows): + with contextlib.redirect_stdout(out): + cmd_search(_default_args(query="event", json=True, limit=1, page=2)) + + data = json.loads(out.getvalue()) + self.assertEqual(len(data), 1) + self.assertEqual(data[0]["sessionId"], "later-event") + + out = io.StringIO() + with patch("chatview.db.search_fts", return_value=rows): + with contextlib.redirect_stdout(out): + cmd_search(_default_args(query="event", json=True, limit=2, page=1)) + first_page = json.loads(out.getvalue()) + self.assertEqual(first_page[0]["duplicateCount"], 2) + self.assertEqual(first_page[0]["duplicateSessionIds"], ["mirror-b"]) + + def test_cmd_search_jsonl_emits_one_valid_object_per_line(self): + from chatview.commands.analysis import cmd_search + + out = io.StringIO() + with contextlib.redirect_stdout(out): + cmd_search(_default_args(query="needle-token", format="jsonl")) + + lines = out.getvalue().splitlines() + self.assertEqual(len(lines), 1) + self.assertEqual(json.loads(lines[0])["sessionId"], "search-sess-001") + def test_cmd_search_project_filter_matches_canonical_alias(self): from chatview import db from chatview.commands.analysis import cmd_search @@ -414,7 +675,7 @@ def test_cmd_search_project_filter_matches_canonical_alias(self): data = json.loads(out.getvalue()) self.assertEqual([item["sessionId"] for item in data], ["search-canonical-project"]) - def test_cmd_search_project_filter_triggers_identity_backfill(self): + def test_cmd_search_project_filter_does_not_trigger_identity_backfill(self): from chatview.commands import analysis from chatview.commands.analysis import cmd_search @@ -422,7 +683,8 @@ def test_cmd_search_project_filter_triggers_identity_backfill(self): with patch.object( analysis, "_init_index", side_effect=lambda force=False: calls.append(force) ), patch( - "chatview.db.project_identity_backfill_needed", return_value=True + "chatview.db.project_identity_backfill_needed", + side_effect=AssertionError("search must not backfill"), ), patch( "chatview.db.search_fts", return_value=[] ): @@ -434,7 +696,33 @@ def test_cmd_search_project_filter_triggers_identity_backfill(self): ) ) - self.assertEqual(calls, [True]) + self.assertEqual(calls, []) + + +class TestSearchCliContract(unittest.TestCase): + def test_search_defaults_to_normal_recall_and_twenty_results(self): + import chatview.cli as cli + + with patch.object(sys, "argv", ["distill", "search", "needle"]), patch.object( + cli, "cmd_search" + ) as command: + cli.main() + + args = command.call_args.args[0] + self.assertEqual(args.recall, "normal") + self.assertEqual(args.limit, 20) + self.assertEqual(args.max_chars, 500) + + def test_search_plus_is_an_illegal_command(self): + import chatview.cli as cli + + stderr = io.StringIO() + with patch.object(sys, "argv", ["distill", "search-plus", "needle"]), contextlib.redirect_stderr(stderr): + with self.assertRaises(SystemExit) as raised: + cli.main() + + self.assertEqual(raised.exception.code, 2) + self.assertIn("invalid choice", stderr.getvalue()) if __name__ == "__main__": diff --git a/tests/test_commands_evolve.py b/tests/test_commands_evolve.py index 16d0d69..ffb0c74 100644 --- a/tests/test_commands_evolve.py +++ b/tests/test_commands_evolve.py @@ -74,6 +74,80 @@ def test_bucket_smaller_than_floor_is_capped_and_output_is_descending(self): self.assertEqual([item["date"] for item in result], sorted([item["date"] for item in result], reverse=True)) +class TestCmdEvolvePatterns(unittest.TestCase): + def test_patterns_command_uses_error_data_without_name_error(self): + from chatview.commands.evolve import cmd_evolve_patterns + + args = SimpleNamespace(json=True) + out = io.StringIO() + with patch("chatview.commands.evolve._data_corrections", return_value=[]), patch( + "chatview.commands.evolve._data_errors", return_value=[] + ), patch( + "chatview.commands.evolve._write_evolve_cache", return_value="/tmp/patterns.json" + ): + with contextlib.redirect_stdout(out): + cmd_evolve_patterns(args) + + self.assertEqual(json.loads(out.getvalue()), {"bubbles": [], "cards": []}) + + +class TestProfileDigestEvidenceFilter(unittest.TestCase): + def test_scan_excludes_agent_sessions_and_meta_research_rows(self): + from chatview.commands.evolve import _profile_digest_scan_messages + + sessions = [ + { + "id": "agent", + "title": "You are Agent 1 extracting history", + "project_name": "proj", + "date": "2026-07-01", + "source": "codex", + "user_message_count": 1, + }, + { + "id": "meta", + "title": "Normal conversation", + "project_name": "proj", + "date": "2026-07-01", + "source": "codex", + "user_message_count": 1, + }, + ] + rows = [ + { + "session_id": "agent", + "idx": 0, + "text": "我们最终选择方案A，不应该 commit。", + "ts": "2026-07-01T10:00:00Z", + }, + { + "session_id": "meta", + "idx": 0, + "text": "请搜索历史对话里有哪些场景，我们最终选择方案A，不应该 commit。", + "ts": "2026-07-01T11:00:00Z", + }, + ] + + decisions, highlights = _profile_digest_scan_messages(sessions, rows) + + self.assertEqual(decisions, []) + self.assertNotIn("agent", {row["id"] for row in highlights}) + meta = next(row for row in highlights if row["id"] == "meta") + self.assertEqual(meta["corrections"], 0) + self.assertEqual(meta["decisions"], 0) + + def test_profile_query_filter_excludes_continuation_summary(self): + from chatview.commands.evolve import _filter_profile_evidence_rows + + sessions = [{"id": "summary", "title": "Normal conversation"}] + rows = [{ + "session_id": "summary", + "text": "This session is being continued from a previous conversation. Summary:\nOld context", + }] + + self.assertEqual(_filter_profile_evidence_rows(sessions, rows), []) + + class TestCmdProfileDigest(_AnalyzeDBTestCase): def _insert_signal_session(self, sid, date, project): from chatview import db diff --git a/tests/test_correction_events.py b/tests/test_correction_events.py index 61ee212..6b4b946 100644 --- a/tests/test_correction_events.py +++ b/tests/test_correction_events.py @@ -36,11 +36,11 @@ def tearDown(self): def _args(self): return SimpleNamespace(date="all", source="all", project="", json=True, limit=10) - def _insert_session(self, sid, mtime, user_text, assistant_text): + def _insert_session(self, sid, mtime, user_text, assistant_text, title=None): db.upsert_session( { "id": sid, - "title": f"Session {sid}", + "title": title or f"Session {sid}", "date": "2026-06-01T10:00:00", "lastDate": "2026-06-01T10:05:00", "filePath": f"/tmp/{sid}.jsonl", @@ -96,6 +96,56 @@ def test_cached_corrections_match_uncached_results(self): self.assertEqual(actual, expected) + def test_agent_prompt_does_not_become_correction_evidence(self): + self._insert_session( + "agent-prompt", + 1.0, + "不应该直接 commit，这是历史分析中的一个候选结论。", + "收到。", + title="You are Agent 1 extracting correction patterns", + ) + + self.assertEqual(corrections._data_corrections(self._args()), []) + + def test_meta_history_research_request_is_not_a_correction(self): + self._insert_session( + "meta-research", + 1.0, + "你搜索一下历史对话里有哪些场景不应该 commit，然后分析一下。", + "抱歉，我错了，之前不应该这样。", + ) + + self.assertEqual(corrections._data_corrections(self._args()), []) + + def test_v4_cache_is_reextracted_after_preceding_user_filter_change(self): + self._insert_session( + "old-cache", + 1.0, + "你搜索一下历史对话里有哪些场景不应该 commit，然后分析一下。", + "抱歉，我错了，之前不应该这样。", + ) + session = corrections._get_filtered_db(self._args())[0] + db.replace_correction_events( + session, + [{ + "message_idx": 2, + "source": "ai", + "kind": "correction", + "text": "stale polluted event", + "signals": ["抱歉"], + }], + "corrections-v4", + ) + + with patch( + "chatview.commands.corrections._extract_corrections_from_session", + wraps=corrections._extract_corrections_from_session, + ) as extract: + rows = corrections._data_corrections(self._args()) + + self.assertTrue(extract.called) + self.assertEqual(rows, []) + def test_fresh_cache_does_not_reextract_sessions(self): self._insert_session( "s1", diff --git a/tests/test_distill_skill_static.py b/tests/test_distill_skill_static.py index b61d59d..9f0ef49 100644 --- a/tests/test_distill_skill_static.py +++ b/tests/test_distill_skill_static.py @@ -76,16 +76,25 @@ def test_skill_md_is_router_and_declares_required_references(self): self.assertNotIn("§认知模型", text) self.assertIn("Choose the scope before ORIENT", text) self.assertIn("Do not load digest/aggregates/stats for a simple lookup", text) - self.assertIn('distill search "" --role user --json --limit 10', text) + self.assertIn('distill search "" --recall normal --role user --evidence-only --format jsonl', text) self.assertIn('distill find-repeats "" --limit 5 --json', text) self.assertIn("messageIndex", text) - self.assertIn("`search` and `search-plus` JSON expose both `idx` and `messageIndex`", text) - self.assertIn('distill search "" -C 2 -n', text) - self.assertIn('distill search-plus "" -C 2 -n', text) + self.assertIn("use `sessionId + idx` as the read coordinate", text) + self.assertIn('distill search "" --recall normal --format lines', text) + self.assertIn('distill search "" --recall high --format jsonl', text) + self.assertIn("--format lines", text) + self.assertIn("--format jsonl", text) + self.assertIn("--evidence-only", text) + self.assertIn("--page", text) + self.assertIn("--max-chars 500", text) self.assertIn('"sessionId":"session_id","idx":123,"radius":2', text) self.assertIn("~/.agents/skills/distill-yourself", text) self.assertNotIn("~/.codex/skills/distill-yourself", text) self.assertIn("写回 ~/.claude", text) + self.assertNotIn("search-plus", text) + self.assertNotIn("-C 1", text) + self.assertNotIn("-C 2", text) + self.assertNotRegex(text, re.compile(r"--limit 10(?:\D|$)")) for rel, h1 in ( ("references/memory-profile.md", "# Memory and Profile SOP"), @@ -164,8 +173,8 @@ def test_twin_reference_documents_current_six_stage_batch_workflow(self): self.assertIn("Show the candidate operations to the user", text) self.assertIn("Do not invent an idx", text) - self.assertIn('distill search-plus "" -C 2 -n', text) - self.assertIn("prefer `idx`", text) + self.assertIn('distill search "" --recall high', text) + self.assertIn("use `sessionId + idx` as the read coordinate", text) self.assertIn('"sessionId":"session_id","idx":123,"radius":2', text) diff --git a/tests/test_retrieval_eval_runner.py b/tests/test_retrieval_eval_runner.py index 7be1688..cbd2ce9 100644 --- a/tests/test_retrieval_eval_runner.py +++ b/tests/test_retrieval_eval_runner.py @@ -99,13 +99,13 @@ def test_search_case_enforces_max_age_days(): }, ] case = { - "command": "search-plus", + "command": "search-high", "query": "query", "relevant": ["recent"], "max_age_days": 7, } - with patch("scripts.run_retrieval_eval.search_plus_data", return_value=result): + with patch("scripts.run_retrieval_eval.search_high_data", return_value=result): row = search_case(case) assert not row["checks"]["date_scope"] diff --git a/tests/test_retrieval_tools.py b/tests/test_retrieval_tools.py index 241d2be..c81388d 100644 --- a/tests/test_retrieval_tools.py +++ b/tests/test_retrieval_tools.py @@ -20,7 +20,7 @@ is_noise_text, read_window_data, read_windows_data, - search_plus_data, + search_high_data, session_brief_data, ) from chatview.db import core as dbcore @@ -53,6 +53,11 @@ def _args(self, **overrides): "before": None, "after": None, "line_number": False, + "format": "standard", + "page": 1, + "evidence_only": False, + "role": "all", + "max_chars": 500, } defaults.update(overrides) return SimpleNamespace(**defaults) @@ -92,7 +97,7 @@ def test_project_scoped_retrieval_triggers_identity_backfill(self): self.assertEqual(calls, [True]) -class TestSearchPlusData(RetrievalToolTestCase): +class TestSearchHighData(RetrievalToolTestCase): def test_structured_identifier_does_not_fall_back_to_generic_subtoken(self): self._insert_session( "identifier-positive", @@ -115,8 +120,8 @@ def test_structured_identifier_does_not_fall_back_to_generic_subtoken(self): }], ) - positive = search_plus_data("build-target_nonexistent_xyz", self._args()) - unknown = search_plus_data("zzzz_nonexistent_xyz", self._args()) + positive = search_high_data("build-target_nonexistent_xyz", self._args()) + unknown = search_high_data("zzzz_nonexistent_xyz", self._args()) self.assertIn("identifier-positive", [row["sessionId"] for row in positive]) self.assertNotIn("identifier-distractor", [row["sessionId"] for row in positive]) @@ -155,7 +160,7 @@ def test_title_candidate_does_not_merge_with_message_index_zero(self): ) rows = [ - row for row in search_plus_data("needle", self._args()) + row for row in search_high_data("needle", self._args()) if row["sessionId"] == "title-and-message" ] @@ -182,7 +187,7 @@ def test_title_only_match_does_not_flood_results_with_unrelated_messages(self): rows = [ row - for row in search_plus_data("UniqueProductTheta", self._args(limit=10)) + for row in search_high_data("UniqueProductTheta", self._args(limit=10)) if row["sessionId"] == "title-only-many-messages" ] @@ -212,7 +217,7 @@ def test_clean_candidate_ranks_before_higher_scoring_artifact(self): }], ) - results = search_plus_data("记忆偏好", self._args()) + results = search_high_data("记忆偏好", self._args()) ids = [row["sessionId"] for row in results] self.assertLess(ids.index("lower-score-clean"), ids.index("high-score-artifact")) @@ -229,7 +234,7 @@ def test_fts_or_orders_candidates_by_rank_before_limiting(self): _fts_or_rows("ranking evidence", limit=10) sql = connection.execute.call_args.args[0] - self.assertRegex(sql, r"ORDER BY\s+rank\s+LIMIT") + self.assertRegex(sql, r"ORDER BY\s+rank[\s\S]+LIMIT") def test_fts_candidate_paths_apply_eligible_scope_before_limit(self): for sid, project in (("scope-target", "eval/target"), ("scope-noise", "eval/noise")): @@ -258,6 +263,55 @@ def test_fts_candidate_paths_apply_eligible_scope_before_limit(self): self.assertEqual([row["session_id"] for row in exact], ["scope-target"]) self.assertEqual([row["session_id"] for row in terms], ["scope-target"]) + def test_high_recall_pages_do_not_overlap_when_page_two_requests_more_rows(self): + self._insert_session( + "stable-pages", + "Stable pagination", + "distill-yourself", + [ + { + "idx": index, + "text": f"stable_page_needle evidence {index}", + "ts": f"2026-07-01T10:{index:02d}:00Z", + } + for index in range(45) + ], + ) + + page_one = search_high_data( + "stable_page_needle", self._args(role="user", limit=20, page=1) + ) + page_two = search_high_data( + "stable_page_needle", self._args(role="user", limit=20, page=2) + ) + + keys_one = {(row["sessionId"], row["idx"]) for row in page_one} + keys_two = {(row["sessionId"], row["idx"]) for row in page_two} + self.assertEqual(len(page_one), 20) + self.assertEqual(len(page_two), 20) + self.assertFalse(keys_one & keys_two) + + def test_evidence_only_filters_chat_view_evolve_subagent_prompt(self): + self._insert_session( + "evolve-subagent-prompt", + "Analysis request", + "distill-yourself", + [{ + "idx": 0, + "text": ( + "你是 Chat Viewer Evolve 的只读分析 sub-agent。" + "背景：我们正在分析用户过去 7 天的 AI 对话历史。search" + ), + "ts": "2026-07-01T10:00:00Z", + }], + ) + + rows = search_high_data( + "search", self._args(role="user", evidence_only=True, limit=20) + ) + + self.assertNotIn("evolve-subagent-prompt", [row["sessionId"] for row in rows]) + def test_finds_chinese_synonym_query_that_plain_fts_misses(self): self._insert_session( "memory-scope", @@ -287,7 +341,7 @@ def test_finds_chinese_synonym_query_that_plain_fts_misses(self): plain = db.search_fts("记忆偏好", limit=10) self.assertEqual(plain, []) - results = search_plus_data("记忆偏好", self._args()) + results = search_high_data("记忆偏好", self._args()) self.assertGreaterEqual(len(results), 1) self.assertEqual(results[0]["sessionId"], "memory-scope") @@ -303,22 +357,22 @@ def test_title_project_matches_are_returned_when_content_does_not_match(self): [{"idx": 0, "text": "No matching acronym in this message", "ts": "2026-07-01T10:00:00Z"}], ) - results = search_plus_data("PR", self._args()) + results = search_high_data("PR", self._args()) self.assertTrue(any(r["sessionId"] == "title-only" for r in results)) row = next(r for r in results if r["sessionId"] == "title-only") self.assertIn("title_project", row["reasons"]) - def test_cmd_search_plus_context_prints_grep_style_window(self): - from chatview.commands.retrieval import cmd_search_plus + def test_cmd_search_high_context_prints_grep_style_window(self): + from chatview.commands.analysis import cmd_search self._insert_session( - "search-plus-context", - "Search plus context", + "high-recall-context", + "High recall context", "grep-proj", [ - {"idx": 0, "text": "Before search-plus context.", "ts": "2026-07-01T10:00:00Z"}, - {"idx": 2, "text": "Please find search-plus-needle here.", "ts": "2026-07-01T10:02:00Z"}, + {"idx": 0, "text": "Before high recall context.", "ts": "2026-07-01T10:00:00Z"}, + {"idx": 2, "text": "Please find high-recall-needle here.", "ts": "2026-07-01T10:02:00Z"}, ], [ {"idx": 1, "text": "Assistant context for search plus.", "ts": "2026-07-01T10:01:00Z"}, @@ -330,19 +384,21 @@ def test_cmd_search_plus_context_prints_grep_style_window(self): out = io.StringIO() with contextlib.redirect_stdout(out): - cmd_search_plus( + cmd_search( self._args( - query="search-plus-needle", + query="high-recall-needle", + recall="high", + format="lines", context=1, line_number=True, ) ) text = out.getvalue() - self.assertIn("search-plus-context:1:assistant", text) - self.assertIn("search-plus-context:2:user", text) + self.assertIn("high-recall-context:1:assistant", text) + self.assertIn("high-recall-context:2:user", text) self.assertIn("Assistant context for search plus.", text) - self.assertIn("search-plus-needle", text) + self.assertIn("high-recall-needle", text) def test_marks_agent_artifacts_without_hard_filtering(self): self._insert_session( @@ -370,13 +426,42 @@ def test_marks_agent_artifacts_without_hard_filtering(self): ], ) - results = search_plus_data("不要 过度设计", self._args()) + results = search_high_data("不要 过度设计", self._args()) ids = [r["sessionId"] for r in results] self.assertLess(ids.index("real-user-signal"), ids.index("agent-artifact")) artifact = next(r for r in results if r["sessionId"] == "agent-artifact") self.assertEqual(artifact["artifactReason"], "agent_prompt") + def test_evidence_only_hard_filters_agent_artifacts(self): + self._insert_session( + "agent-artifact-only", + "You are Agent 2 reviewing history", + "claude_chat_view", + [{ + "idx": 0, + "text": "不要过度设计，这是子 agent 汇总出的观点。", + "ts": "2026-07-01T10:00:00Z", + }], + ) + self._insert_session( + "human-evidence", + "Normal user conversation", + "claude_chat_view", + [{ + "idx": 0, + "text": "不要过度设计，做最小改动。", + "ts": "2026-07-01T11:00:00Z", + }], + ) + + results = search_high_data( + "不要 过度设计", + self._args(evidence_only=True), + ) + + self.assertEqual([row["sessionId"] for row in results], ["human-evidence"]) + def test_skips_token_scan_when_exact_clean_user_matches_are_enough(self): self._insert_session( "exact-a", @@ -403,7 +488,7 @@ def test_skips_token_scan_when_exact_clean_user_matches_are_enough(self): "chatview.commands.retrieval._token_scan_rows", side_effect=AssertionError("token scan should be skipped"), ): - results = search_plus_data("不要过度设计", self._args(limit=20)) + results = search_high_data("不要过度设计", self._args(limit=20)) self.assertEqual({r["sessionId"] for r in results[:2]}, {"exact-a", "exact-b"}) @@ -430,11 +515,25 @@ def test_exact_user_evidence_ranks_above_assistant_echo(self): }], ) - results = search_plus_data("不要过度设计", self._args(limit=10)) + results = search_high_data("不要过度设计", self._args(limit=10)) self.assertEqual(results[0]["sessionId"], "user-evidence") self.assertEqual(results[0]["role"], "user") + def test_high_recall_honors_user_role_filter(self): + self._insert_session( + "mixed-roles", + "Mixed roles", + "distill-yourself", + [{"idx": 0, "text": "role-filter-needle user", "ts": "2026-07-01T10:00:00Z"}], + [{"idx": 1, "text": "role-filter-needle assistant", "ts": "2026-07-01T10:01:00Z"}], + ) + + results = search_high_data("role-filter-needle", self._args(role="user")) + + self.assertTrue(results) + self.assertEqual({row["role"] for row in results}, {"user"}) + class TestFindRepeatsData(RetrievalToolTestCase): @staticmethod @@ -872,6 +971,46 @@ def test_batch_returns_multiple_windows(self): self.assertEqual([m["idx"] for m in data["windows"][0]["messages"]], [0]) self.assertEqual([m["idx"] for m in data["windows"][1]["messages"]], [1, 2, 3]) + def test_radius_uses_idx_range_and_does_not_fill_gaps(self): + self._insert_session( + "sparse-window", + "Sparse", + "distill-yourself", + [ + {"idx": 0, "text": "outside left", "ts": "2026-07-01T10:00:00Z"}, + {"idx": 2, "text": "target", "ts": "2026-07-01T10:02:00Z"}, + {"idx": 5, "text": "outside right", "ts": "2026-07-01T10:05:00Z"}, + ], + ) + + data = read_window_data("sparse-window", idx=2, radius=1) + + self.assertEqual([msg["idx"] for msg in data["messages"]], [2]) + + def test_human_output_marks_truncation_but_json_keeps_indexed_text(self): + import contextlib + import io + import json + from chatview.commands.retrieval import cmd_read_window + + long_text = "长" * 1500 + self._insert_session( + "long-window", + "Long", + "distill-yourself", + [{"idx": 0, "text": long_text, "ts": "2026-07-01T10:00:00Z"}], + ) + + human = io.StringIO() + with contextlib.redirect_stdout(human): + cmd_read_window(self._args(session="long-window", idx=0, radius=0, batch="")) + self.assertIn("truncated from 1500 chars", human.getvalue()) + + structured = io.StringIO() + with contextlib.redirect_stdout(structured): + cmd_read_window(self._args(session="long-window", idx=0, radius=0, batch="", json=True)) + self.assertEqual(json.loads(structured.getvalue())["messages"][0]["text"], long_text) + class TestSessionBriefData(RetrievalToolTestCase): def test_returns_compact_session_summary_and_noise_counts(self): @@ -908,6 +1047,13 @@ def test_flags_system_and_tool_noise(self): self.assertTrue(is_noise_text("你是一个严格、独立的标注员。任务：判断每条用户消息是否在纠正 AI。")) self.assertTrue(is_noise_text("x" * 2000 + "Pre-collected Data (do NOT re-run these)")) self.assertTrue(is_noise_text("pre-computed project distribution + daily activity as JSON\n=== STATS ===")) + self.assertTrue(is_noise_text("你是独立审阅员。子代理任务：回测这些历史样例。")) + self.assertTrue(is_noise_text("tool_search for the capability NOW and call it exactly once")) + self.assertTrue(is_noise_text("Acceptance self-test: FLAG_RETRIEVAL_OK")) + self.assertTrue(is_noise_text("# 独立 Review 请求（请先读源码核对，再下结论）")) + self.assertTrue(is_noise_text("# 设计评审请求（请完全独立判断，不要揣测提交者）")) + self.assertTrue(is_noise_text("现在只说明你会调用哪个 skill 以及为什么，然后停止，不要真正检索。")) + self.assertTrue(is_noise_text('# Judgment Cards (input data) [{"strength": 0.86, "status": "confirmed"}]')) def test_keeps_real_user_preference_signal(self): self.assertFalse(is_noise_text("不要过度设计，也别新增没有要求的抽象；最小改动即可。")) diff --git a/tests/test_search_robustness.py b/tests/test_search_robustness.py new file mode 100644 index 0000000..0a899bb --- /dev/null +++ b/tests/test_search_robustness.py @@ -0,0 +1,502 @@ +"""Regression tests for search depth, evidence hygiene, and read-only caches.""" + +import contextlib +import hashlib +import io +import json +import os +import shutil +import sqlite3 +import sys +import tempfile +import threading +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from chatview import db +from chatview.commands.analysis import cmd_search +from chatview.commands.evidence import artifact_reason +from chatview.db import core as dbcore + + +def _args(**overrides): + defaults = { + "query": "needle", + "source": "all", + "project": "", + "date": "all", + "role": "all", + "limit": 20, + "page": 1, + "json": True, + "context": 0, + "before": None, + "after": None, + "line_number": False, + "format": "standard", + "evidence_only": True, + "recall": "normal", + "max_chars": 500, + } + defaults.update(overrides) + return SimpleNamespace(**defaults) + + +def _row(index, artifact=False): + text = ( + f"needle generated artifact {index}" + if artifact + else f"needle durable user evidence {index}" + ) + return { + "session_id": f"session-{index}", + "title": "Search fixture", + "project_name": "test-project", + "project_key": "test-project", + "project_display": "test-project", + "source": "codex", + "idx": index, + "role": "user", + "text": text, + "ts": f"2026-07-01T10:{index % 60:02d}:00Z", + } + + +class TestProgressiveNormalSearch(unittest.TestCase): + def _search(self, rows, **overrides): + limits = [] + + def fake_search(_query, limit, **_kwargs): + limits.append(limit) + return rows[:limit] + + output = io.StringIO() + with patch("chatview.commands.analysis._prepare_search_db"), patch( + "chatview.db.search_fts", side_effect=fake_search + ), contextlib.redirect_stdout(output): + cmd_search(_args(**overrides)) + return json.loads(output.getvalue()), limits + + def test_expands_past_500_until_page_is_full(self): + rows = [_row(i, artifact=True) for i in range(600)] + rows += [_row(i) for i in range(600, 630)] + + results, limits = self._search(rows) + + self.assertEqual(limits, [64, 160, 500, 1000]) + self.assertEqual(len(results), 20) + + def test_expands_past_1000_without_a_query_specific_ceiling(self): + rows = [_row(i, artifact=True) for i in range(1100)] + rows += [_row(i) for i in range(1100, 1130)] + + results, limits = self._search(rows) + + self.assertEqual(limits, [64, 160, 500, 1000, 2000]) + self.assertEqual(len(results), 20) + + def test_stops_at_true_database_exhaustion(self): + rows = [_row(i, artifact=True) for i in range(700)] + rows += [_row(i) for i in range(700, 705)] + + results, limits = self._search(rows) + + self.assertEqual(limits, [64, 160, 500, 1000]) + self.assertEqual(len(results), 5) + + def test_stops_after_first_prefix_when_it_is_already_full(self): + results, limits = self._search([_row(i) for i in range(80)]) + + self.assertEqual(limits, [64]) + self.assertEqual(len(results), 20) + + def test_page_two_matches_the_second_half_of_a_larger_first_page(self): + rows = [_row(i) for i in range(80)] + + page_one, _ = self._search(rows, page=1, limit=20) + page_two, _ = self._search(rows, page=2, limit=20) + combined, _ = self._search(rows, page=1, limit=40) + + keys = lambda data: [(row["sessionId"], row["idx"]) for row in data] + self.assertEqual(keys(page_one) + keys(page_two), keys(combined)) + + +class TestStructuredOrchestrationEvidence(unittest.TestCase): + def test_filters_long_multi_signal_agent_work_orders(self): + positives = [ + """# 第2轮 QA/测试 review +背景：修复已提交，需要独立复核。 +任务：只读检查检索结果。 +- 跑三个 holdout +- 记录失败样例 +输出格式：只返回 PASS/FAIL 和证据，不要修改代码。""", + """# 研究任务 B：搜索质量 +你是检索研究员，请独立完成这一轮。 +目标：分析历史命中污染。 +要求： +1. 阅读现有实现 +2. 给出反例 +完成后返回证据表；全程只读。""", + """# Visual review +You are a visual design reviewer working independently. +Task: inspect the result presentation. +Requirements: +- check hierarchy and spacing +- cite exact screenshots +Output format: report only findings and do not modify files.""", + """Resume /goal search validation after the implementation complete notice. +Context: the worker reports that all changes are ready. +Acceptance criteria: +1. verify normal recall +2. review high recall +Respond with exact failures only; this is read-only.""", + """# 任务：浏览器测试 Digital Twin +背景：功能已经由另一个 worker 完成。 +请打开页面检查以下路径，并截图记录关键状态。 +输出格式：只给出复现步骤、实际结果和 verdict，不要修改任何文件。""", + """第二轮 review：上一轮问题声称已经修复。 +你是独立评审，请只读审查相关 diff。 +重点：检查分页稳定性和异常路径。 +Return: Verdict, evidence, and any remaining risk. Do not edit files.""", + """你是 AI Coding Agent 设计专家，请评估这个交互方案。 +任务：找出用户可能误操作的地方。 +- 检查入口和反馈 +- 检查失败恢复 +输出格式：三条结论和一个 verdict；不要修改代码。""", + """Focus on pagination regressions and schema compatibility across both recall modes. +Inspect the implementation and its tests independently. Do not edit files. +Return: Verdict, exact evidence, and the smallest remaining risk.""", + """Fallback: check the current worktree after the previous reviewer timed out. +Scope: retrieval and evidence classification only. +Do not edit files or create fixtures. +Return: verdict and exact failing commands.""", + """你是独立评审，对一项管线改动做对抗性 review。先读完整 diff 和测试。任务：专挑失败模式。必答：1) 给出 false positive 反例；2) 检查并发语义；3) 标注 file:line。中文回答，结论先行并给明确 verdict。""", + """三条 finding 已落实到同一 plan 文件。请复核这些修订，只验 finding，不重开全面 review。1) 检查函数定义；2) 检查回归命令；3) 检查文档更新。给最终 verdict：通过 / 不通过。""", + """Background: You are analyzing AI conversation history for reusable preferences. +CLI tool: run the supplied history command only. +Focus area: testing, verification, and output artifacts. +Output requirements: +- return four candidate memories +- cite exact evidence +Do not edit files.""", + """Review this diff for correctness, edge cases, and consistency with the codebase. +Focus on: +1. helper correctness +2. stdout schema compatibility +3. file flush behavior +Report findings by severity with file:line references. Do not edit files.""", + """你是 Chat Viewer Evolve 的子分析代理。你的任务是分析用户画像的一部分，不能写入 evolve，只返回证据充分的中文洞察。 +固定范围：所有 CLI 调用必须使用给定日期。 +必须流程：1. 先读 digest；2. 搜索候选；3. 返回证据。不要修改文件。""", + """Read-only verification task in the current repo. Do not modify files. +Scope: CLI and DB migration behavior. +Verify these claims: +1. normal search returns a stable page +2. high recall stays read-only +Return: verdict, evidence, and missing tests.""", + """Act as the release sheriff for this search change. Inspect the diff and tests independently. +Do not edit files or create commits. +Return: verdict, exact evidence, and any release-blocking risk.""", + """作为红队验证负责人，请独立检查这个检索修复的失败模式。 +要求：只读验证，不要修改代码。 +输出格式：明确 verdict、证据位置和剩余风险。""", + """Take the chair as launch gatekeeper and independently check this candidate. +Leave the worktree untouched and do not edit files. +Return a go/no-go decision with exact evidence.""", + """请担任本轮发布质量把关人，独立核验实现与测试。 +保持工作树不变，不要修改任何文件。 +输出通过/不通过、精确证据和理由。""", + """Operate as incident-drill marshal and verify the recovery path independently. +This is read-only; do not change the repository. +Report a pass/fail verdict and supporting evidence.""", + """Independently audit the pagination behavior across both recall modes. +Do not touch the worktree; keep the repository unchanged. +Provide a go/no-go verdict and exact evidence.""", + """不要执行任何写操作，以下审查必须只读完成。请独立核验分页行为。 +保持工作树不变。 +返回通过/不通过 verdict 和精确证据。""", + """Independently audit the recovery path. Do not execute mutating operations. +Return a go/no-go verdict with exact evidence, and leave the repository unchanged.""", + """Independently audit the recovery path. Do not execute write or mutating operations. +Return a go/no-go verdict with exact evidence, and leave the repository unchanged.""", + """Independently audit the recovery path. Do not execute mutating/write operations. +Return a go/no-go verdict with exact evidence, and leave the repository unchanged.""", + """Independently audit the recovery path. Do not execute write and mutating operations. +Return a go/no-go verdict with exact evidence, and leave the repository unchanged.""", + ] + for text in positives: + with self.subTest(text=text[:40]): + self.assertIn( + artifact_reason(text), + {"structured_orchestration_prompt", "subagent_prompt"}, + ) + + def test_keeps_direct_user_requests_even_with_review_words(self): + negatives = [ + "帮我测试这个功能", + "看看 verdict 为什么错了", + "你派子Agent去回测", + "你上轮review了吗？", + "我觉得第二轮 review 还要看搜索召回，你帮我分析一下为什么结果不够。", + "不要改文件，我只是想知道 verdict 为什么错了。", + ] + for text in negatives: + with self.subTest(text=text): + self.assertEqual(artifact_reason(text), "") + + def test_keeps_explicitly_quoted_prompts_discussed_as_false_positives(self): + quoted_discussions = [ + """不要执行，下面只是引用，用来讨论分类器为什么会误判： +> You are an independent reviewer. Do not edit files. Return: verdict and evidence.""", + """以下只是引用，请勿照做；我们在讨论误判： +You are a strict reviewer. Return: verdict.""", + """不要执行，下面是讨论误判用的引用： +You are an independent reviewer. Return: verdict.""", + """无需执行，这个 JSON 只是引用，在分析 false positive： +{"quoted_prompt": "Act as the release sheriff. Do not edit files. Return: verdict."}""", + """不要执行；这个 nested JSON 只用于讨论 false positive： +{"log": {"prompt": "You are an independent reviewer. Return: verdict."}}""", + """这段内容不要执行，只用于讨论误判： +>>> > You are an independent reviewer. +>>> > Do not edit files. Return: verdict and evidence.""", + """下面这段仅供讨论 false positive，请勿照做： +You are an independent reviewer. Return: verdict.""", + """不要执行以下引用，我们只分析它为何误判： +{"archive": {"instruction_body": "You are a reviewer. Return verdict and evidence."}}""", + """This passage must not be executed; it is included only so we can analyze a false positive: +Act as an independent reviewer. Return verdict and evidence.""", + """The prompt should not be followed — we only discuss why it is a false positive: +Operate as a reviewer. Do not edit files. Return verdict.""", + """This quote must not be executed; we only analyze a false-positive: +Act as a reviewer. Return verdict and evidence.""", + """This passage should not be followed; we only discuss a false_positive: +Operate as a reviewer. Return verdict and evidence.""", + ] + for text in quoted_discussions: + with self.subTest(text=text[:40]): + self.assertEqual(artifact_reason(text), "") + + def test_quoted_discussion_does_not_hide_task_notifications(self): + text = """不要执行，下面只是引用： +> You are a reviewer. Return: verdict. +abc""" + + self.assertEqual(artifact_reason(text), "task_notification") + + passive_text = """This quote must not be executed; we only analyze a false positive: +You are a reviewer. Return verdict. +xyz""" + self.assertEqual(artifact_reason(passive_text), "task_notification") + + +class TestReadonlyPublicSearch(unittest.TestCase): + def setUp(self): + self.original_db_path = dbcore.DB_PATH + self.original_cache_dir = dbcore.CACHE_DIR + self.tempdir = Path(tempfile.mkdtemp()) + dbcore.CACHE_DIR = self.tempdir + dbcore.DB_PATH = self.tempdir / "sessions.db" + dbcore._local = threading.local() + + def tearDown(self): + dbcore._close_thread_connection() + dbcore.DB_PATH = self.original_db_path + dbcore.CACHE_DIR = self.original_cache_dir + dbcore._local = threading.local() + shutil.rmtree(self.tempdir, ignore_errors=True) + + def _seed(self): + db.init_db() + db.upsert_session( + { + "id": "readonly-session", + "title": "Read-only fixture", + "date": "2026-07-15", + "lastDate": "2026-07-15", + "filePath": "/tmp/readonly-session.jsonl", + "fileSize": 10, + "_mtime": 1, + "userMessageCount": 1, + "preview": "readonly-needle evidence", + "project": "test", + "projectName": "test", + "source": "codex", + }, + [{"idx": 0, "text": "readonly-needle evidence", "ts": "2026-07-15"}], + [], + ) + dbcore._close_thread_connection() + + def _force_readonly_connection(self): + dbcore._open_readonly_connection() + + def _seed_live_wal(self): + self._seed() + writer = sqlite3.connect(dbcore.DB_PATH) + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + writer.execute( + "UPDATE sessions SET title=title || ' wal' WHERE id='readonly-session'" + ) + writer.commit() + self.assertGreater(Path(f"{dbcore.DB_PATH}-wal").stat().st_size, 0) + self.assertTrue(Path(f"{dbcore.DB_PATH}-shm").is_file()) + return writer + + def _snapshot(self): + return { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in self.tempdir.iterdir() + if path.is_file() + } + + def _run_cli_search(self, recall): + import chatview.cli as cli + + output = io.StringIO() + argv = ["distill", "search", "readonly-needle", "--json"] + if recall == "high": + argv.extend(["--recall", "high"]) + with patch.object(sys, "argv", argv), contextlib.redirect_stdout(output): + cli.main() + return json.loads(output.getvalue()) + + def test_normal_and_high_search_query_compatible_readonly_db_without_writes(self): + self._seed() + before = self._snapshot() + + self._force_readonly_connection() + normal = self._run_cli_search("normal") + dbcore._close_thread_connection() + self._force_readonly_connection() + high = self._run_cli_search("high") + dbcore._close_thread_connection() + + self.assertEqual(normal[0]["sessionId"], "readonly-session") + self.assertEqual(high[0]["sessionId"], "readonly-session") + self.assertEqual(self._snapshot(), before) + + def test_search_attempts_no_sqlite_writes_when_authorizer_rejects_them(self): + self._seed() + dbcore.get_conn() + write_actions = { + sqlite3.SQLITE_INSERT, + sqlite3.SQLITE_UPDATE, + sqlite3.SQLITE_DELETE, + sqlite3.SQLITE_CREATE_INDEX, + sqlite3.SQLITE_CREATE_TABLE, + sqlite3.SQLITE_CREATE_VTABLE, + sqlite3.SQLITE_DROP_INDEX, + sqlite3.SQLITE_DROP_TABLE, + sqlite3.SQLITE_DROP_VTABLE, + sqlite3.SQLITE_ALTER_TABLE, + } + attempts = [] + + def authorizer(action, arg1, arg2, database, trigger): + if action in write_actions: + attempts.append((action, arg1, arg2)) + return sqlite3.SQLITE_DENY + return sqlite3.SQLITE_OK + + dbcore._local.conn.set_authorizer(authorizer) + normal = self._run_cli_search("normal") + high = self._run_cli_search("high") + + self.assertTrue(normal) + self.assertTrue(high) + self.assertEqual(attempts, []) + + def test_nonempty_wal_with_shm_is_read_without_changing_source_files(self): + writer = self._seed_live_wal() + try: + before = self._snapshot() + + self._force_readonly_connection() + rows = self._run_cli_search("normal") + dbcore._close_thread_connection() + + self.assertEqual(rows[0]["sessionId"], "readonly-session") + self.assertEqual(rows[0]["title"], "Read-only fixture wal") + # SQLite may update transient lock bytes in the existing SHM file; + # the durable database and WAL must remain byte-identical. + after = self._snapshot() + self.assertEqual(set(after), set(before)) + self.assertEqual(after["sessions.db"], before["sessions.db"]) + self.assertEqual(after["sessions.db-wal"], before["sessions.db-wal"]) + finally: + writer.close() + + def test_nonempty_wal_without_shm_fails_before_touching_source_files(self): + writer = self._seed_live_wal() + shm_path = Path(f"{dbcore.DB_PATH}-shm") + shm_path.unlink() + before = self._snapshot() + try: + with self.assertRaisesRegex( + RuntimeError, "snapshot is incomplete.*checkpoint.*write access" + ): + db.prepare_search_db() + + self.assertEqual(self._snapshot(), before) + finally: + writer.close() + + def test_explicit_init_switches_a_search_connection_back_to_writable(self): + self._seed() + self._force_readonly_connection() + self.assertTrue(db.prepare_search_db()) + + db.init_db() + + self.assertFalse(dbcore._local.readonly) + self.assertEqual(db.get_conn().execute("PRAGMA query_only").fetchone()[0], 0) + + def test_incompatible_readonly_schema_fails_with_migration_instruction(self): + conn = sqlite3.connect(dbcore.DB_PATH) + conn.execute("CREATE TABLE sessions (id TEXT PRIMARY KEY)") + conn.commit() + conn.close() + self._force_readonly_connection() + + with self.assertRaisesRegex(RuntimeError, "schema is incompatible.*distill refresh"): + db.prepare_search_db() + + def test_plain_tables_named_like_fts_are_rejected(self): + conn = sqlite3.connect(dbcore.DB_PATH) + conn.executescript(""" + CREATE TABLE sessions ( + id TEXT, title TEXT, date TEXT, project_name TEXT, + project_key TEXT, project_display TEXT, + project_identity_version INTEGER, source TEXT + ); + CREATE TABLE messages ( + id INTEGER, session_id TEXT, idx INTEGER, role TEXT, + text TEXT, ts TEXT + ); + CREATE TABLE messages_fts (text TEXT); + CREATE TABLE sessions_fts (title TEXT, project_name TEXT); + """) + conn.commit() + conn.close() + self._force_readonly_connection() + + with self.assertRaisesRegex( + RuntimeError, "not FTS5 virtual tables: messages_fts, sessions_fts.*distill refresh" + ): + db.prepare_search_db() + + def test_missing_db_is_initialized_when_location_is_writable(self): + self.assertFalse(dbcore.DB_PATH.exists()) + + readonly = db.prepare_search_db() + + self.assertFalse(readonly) + self.assertTrue(dbcore.DB_PATH.exists()) + self.assertEqual(db.search_fts("nothing", limit=1), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search_snippets.py b/tests/test_search_snippets.py new file mode 100644 index 0000000..20ffac9 --- /dev/null +++ b/tests/test_search_snippets.py @@ -0,0 +1,31 @@ +import unittest + +from chatview.search import _make_snippet_data +from chatview.snippets import make_query_snippet + + +class TestCanonicalQuerySnippet(unittest.TestCase): + def test_centers_match_near_character_800_and_stays_bounded(self): + text = "前" * 800 + "关键命中" + "后" * 800 + + data = make_query_snippet(text, "关键命中", max_chars=500) + + self.assertIn("关键命中", data["snippet"]) + self.assertLessEqual(len(data["snippet"]), 500) + self.assertEqual(data["matchStart"], 800) + self.assertEqual(data["originalChars"], len(text)) + self.assertTrue(data["snippetTruncated"]) + self.assertTrue(data["snippet"].startswith("…")) + self.assertTrue(data["snippet"].endswith("…")) + + def test_web_search_uses_the_same_snippet_contract(self): + text = "x" * 800 + "needle" + "y" * 800 + + self.assertEqual( + _make_snippet_data(text, "needle"), + make_query_snippet(text, "needle", max_chars=500), + ) + + +if __name__ == "__main__": + unittest.main() From 434cd8150a5d5a282679a6950419ce9a172b26a9 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Fri, 17 Jul 2026 23:01:04 -0400 Subject: [PATCH 02/15] feat: add bounded tool call retrieval --- analyze.py | 6 + chatview/cli.py | 31 +++++ chatview/commands/tool_calls.py | 160 ++++++++++++++++++++++++ chatview/db/__init__.py | 10 ++ chatview/db/core.py | 58 ++++++++- chatview/db/sessions.py | 10 ++ chatview/db/tool_calls.py | 199 ++++++++++++++++++++++++++++++ chatview/index.py | 12 +- chatview/parsers/claude.py | 32 ++++- chatview/parsers/codex.py | 45 +++++-- chatview/tool_events.py | 128 +++++++++++++++++++ tests/test_tool_call_retrieval.py | 188 ++++++++++++++++++++++++++++ 12 files changed, 860 insertions(+), 19 deletions(-) create mode 100644 chatview/commands/tool_calls.py create mode 100644 chatview/db/tool_calls.py create mode 100644 chatview/tool_events.py create mode 100644 tests/test_tool_call_retrieval.py diff --git a/analyze.py b/analyze.py index 37e34d0..ced294c 100644 --- a/analyze.py +++ b/analyze.py @@ -52,6 +52,12 @@ cmd_refresh, cmd_install_skill, ) +from chatview.commands.tool_calls import ( # noqa: F401 + cmd_tool_search, + cmd_read_tool_event, + tool_search_data, + read_tool_event_data, +) from chatview.commands.twin import ( # noqa: F401 cmd_twin_stats, cmd_twin_budget, diff --git a/chatview/cli.py b/chatview/cli.py index 7df6271..96cdefa 100644 --- a/chatview/cli.py +++ b/chatview/cli.py @@ -56,6 +56,7 @@ cmd_twin_link, cmd_twin_batch, ) +from chatview.commands.tool_calls import cmd_tool_search, cmd_read_tool_event def _max_chars(value): @@ -182,6 +183,34 @@ def main(): help='JSON list of {"session": "...", "idx": N, "radius": N} items', ) + p_tool_search = sub.add_parser( + "tool-search", + parents=[shared], + help="Search bounded tool-call inputs and return raw JSONL locators", + ) + p_tool_search.set_defaults(limit=20) + p_tool_search.add_argument("query", help="Tool input, command, path, or patch target") + p_tool_search.add_argument("--page", type=int, default=1, help="1-based result page") + p_tool_search.add_argument( + "--format", choices=("standard", "lines", "jsonl"), default="standard" + ) + p_tool_search.add_argument( + "--max-chars", type=_max_chars, default=500, + help="Maximum characters per search result (default: 500)", + ) + + p_read_tool = sub.add_parser( + "read-tool-event", + parents=[shared], + help="Read one indexed tool call from its source JSONL with bounded output", + ) + p_read_tool.add_argument("session", help="Exact session ID from tool-search") + p_read_tool.add_argument("--event-idx", type=int, required=True) + p_read_tool.add_argument( + "--max-chars", type=_max_chars, default=2000, + help="Maximum tool-input characters to return (default/max: 2000)", + ) + p_find_repeats = sub.add_parser( "find-repeats", parents=[shared], @@ -444,6 +473,8 @@ def main(): "read": cmd_read, "search": cmd_search, "read-window": cmd_read_window, + "tool-search": cmd_tool_search, + "read-tool-event": cmd_read_tool_event, "find-repeats": cmd_find_repeats, "session-brief": cmd_session_brief, "queries": cmd_queries, diff --git a/chatview/commands/tool_calls.py b/chatview/commands/tool_calls.py new file mode 100644 index 0000000..ab06dfd --- /dev/null +++ b/chatview/commands/tool_calls.py @@ -0,0 +1,160 @@ +"""Bounded CLI retrieval for indexed tool-call provenance.""" + +from __future__ import annotations + +import json +import os + +from chatview.snippets import make_query_snippet +from chatview.tool_events import read_jsonl_record, tool_call_from_record + + +def tool_search_data(query: str, args) -> list: + from chatview import db as _db + + _db.prepare_search_db() + rows = _db.search_tool_calls( + query, + source=getattr(args, "source", "all"), + project=getattr(args, "project", ""), + date_range=getattr(args, "date", "all"), + limit=max(getattr(args, "limit", 20), 1), + page=max(getattr(args, "page", 1), 1), + ) + results = [] + for row in rows: + snippet = make_query_snippet( + row.get("input_text", ""), + query=query, + max_chars=getattr(args, "max_chars", 500), + ) + results.append( + { + "sessionId": row.get("session_id", ""), + "eventIdx": row.get("event_idx"), + "lineNumber": row.get("line_number"), + "byteOffset": row.get("byte_offset"), + "byteLength": row.get("byte_length"), + "blockIndex": row.get("block_index"), + "toolName": row.get("tool_name", ""), + "rawName": row.get("raw_name", ""), + "callId": row.get("call_id", ""), + "timestamp": row.get("ts", ""), + "title": row.get("title", ""), + "project": row.get("project_display") + or row.get("project_name", ""), + "source": row.get("source", ""), + "filePath": row.get("file_path", ""), + "snippet": snippet["snippet"], + "snippetTruncated": snippet["snippetTruncated"], + "indexedChars": snippet["originalChars"], + "originalBytes": row.get("original_bytes", 0), + "inputTruncatedAtIndex": bool(row.get("input_truncated")), + } + ) + return results + + +def cmd_tool_search(args) -> None: + results = tool_search_data(args.query, args) + output_format = getattr(args, "format", "standard") + if getattr(args, "json", False): + print(json.dumps(results, ensure_ascii=False, indent=2)) + return + if output_format == "jsonl": + for row in results: + print(json.dumps(row, ensure_ascii=False)) + return + if output_format == "lines": + for row in results: + snippet = row["snippet"].replace("\n", " ") + print( + f"{row['sessionId']}:{row['eventIdx']}:{row['toolName']}: {snippet}" + ) + return + + print(f"Found {len(results)} tool-call matches for '{args.query}':\n") + for row in results: + print( + f" [{row['source']}] {row['timestamp'][:19]} " + f"{row['toolName']} · {row['title'][:60]}" + ) + print(f" > {row['snippet']}") + print(f" session: {row['sessionId']} · event-idx: {row['eventIdx']}") + print( + " read-tool-event: " + f"distill read-tool-event {row['sessionId']} --event-idx {row['eventIdx']}" + ) + print() + + +def read_tool_event_data(session_id: str, event_idx: int, max_chars: int = 2000) -> dict: + from chatview import db as _db + + _db.prepare_search_db() + locator = _db.get_tool_call(session_id, int(event_idx)) + if not locator: + raise KeyError(f"Tool call not found: {session_id}:{event_idx}") + filepath = locator.get("file_path", "") + if not filepath or not os.path.isfile(filepath): + raise FileNotFoundError(f"Session JSONL not found: {filepath}") + + raw_record = read_jsonl_record( + filepath, + int(locator.get("byte_offset", -1)), + int(locator.get("byte_length", 0)), + ) + event = tool_call_from_record( + raw_record, + locator.get("source", ""), + int(locator.get("block_index", 0)), + ) + indexed_call_id = locator.get("call_id", "") + if indexed_call_id and event.get("callId") != indexed_call_id: + raise RuntimeError("Indexed tool call changed; run `distill refresh`") + + full_input = event.pop("input", "") + max_chars = max(int(max_chars), 1) + event["input"] = full_input[:max_chars] + event.update( + { + "sessionId": locator["session_id"], + "eventIdx": locator["event_idx"], + "lineNumber": locator["line_number"], + "byteOffset": locator["byte_offset"], + "byteLength": locator["byte_length"], + "blockIndex": locator["block_index"], + "title": locator.get("title", ""), + "project": locator.get("project_name", ""), + "source": locator.get("source", ""), + "filePath": filepath, + "originalChars": len(full_input), + "outputTruncated": len(full_input) > max_chars, + } + ) + return event + + +def cmd_read_tool_event(args) -> None: + try: + result = read_tool_event_data( + args.session, + args.event_idx, + max_chars=getattr(args, "max_chars", 2000), + ) + except (KeyError, FileNotFoundError, RuntimeError, ValueError) as exc: + raise SystemExit(str(exc)) from exc + + if getattr(args, "json", False): + print(json.dumps(result, ensure_ascii=False, indent=2)) + return + print(f"# {result['toolName']} · {result['title']}") + print( + f"# session:{result['sessionId']} event-idx:{result['eventIdx']} " + f"line:{result['lineNumber']}" + ) + print(f"# source:{result['source']} file:{result['filePath']}") + print(f"\n--- TOOL INPUT ({result['originalChars']} chars) ---") + print(result["input"]) + if result["outputTruncated"]: + print("\n[TRUNCATED — increase --max-chars up to 2000 only if needed]") diff --git a/chatview/db/__init__.py b/chatview/db/__init__.py index f5b60a7..25be581 100644 --- a/chatview/db/__init__.py +++ b/chatview/db/__init__.py @@ -32,6 +32,12 @@ get_message_window, verify_fts_integrity, ) +from .tool_calls import ( + replace_tool_calls, + search_tool_calls, + get_tool_call, + delete_tool_calls_for_sessions, +) from .insights import ( get_aggregate, set_aggregate, @@ -137,6 +143,10 @@ "get_session_messages", "get_message_window", "verify_fts_integrity", + "replace_tool_calls", + "search_tool_calls", + "get_tool_call", + "delete_tool_calls_for_sessions", "get_aggregate", "set_aggregate", "refresh_aggregates", diff --git a/chatview/db/core.py b/chatview/db/core.py index d09a6cf..0333a5b 100644 --- a/chatview/db/core.py +++ b/chatview/db/core.py @@ -67,14 +67,17 @@ def _open_readonly_connection() -> sqlite3.Connection: def _validate_search_schema(conn: sqlite3.Connection) -> None: - required_tables = {"sessions", "messages", "messages_fts", "sessions_fts"} + required_tables = { + "sessions", "messages", "messages_fts", "sessions_fts", + "tool_calls", "tool_calls_fts", + } rows = conn.execute( "SELECT name, type, sql FROM sqlite_master WHERE type IN ('table', 'view')" ).fetchall() schema = {row["name"]: row for row in rows} missing_tables = sorted(required_tables - set(schema)) invalid_fts = [] - for table in ("messages_fts", "sessions_fts"): + for table in ("messages_fts", "sessions_fts", "tool_calls_fts"): row = schema.get(table) if row is None: continue @@ -90,6 +93,12 @@ def _validate_search_schema(conn: sqlite3.Connection) -> None: "messages": {"id", "session_id", "idx", "role", "text", "ts"}, "messages_fts": {"text"}, "sessions_fts": {"title", "project_name"}, + "tool_calls": { + "id", "session_id", "event_idx", "line_number", "byte_offset", + "byte_length", "block_index", "ts", "tool_name", "raw_name", + "call_id", "input_text", "original_bytes", "input_truncated", + }, + "tool_calls_fts": {"input_text"}, } missing_columns = [] for table, expected in required_columns.items(): @@ -185,6 +194,7 @@ def query_in_chunks( { "sessions", "messages", + "tool_calls", "evidence_events", "judgment_cards", "card_relations", @@ -200,6 +210,7 @@ def query_in_chunks( "correction_session_state", "messages_fts", "messages_fts_trigram", + "tool_calls_fts", "twin_checkpoints", "twin_runs", "twin_run_events", @@ -228,6 +239,18 @@ def query_in_chunks( "project_identity_version", "source", "starred", + # tool_calls + "event_idx", + "line_number", + "byte_offset", + "byte_length", + "block_index", + "tool_name", + "raw_name", + "call_id", + "input_text", + "original_bytes", + "input_truncated", # evidence_events "run_id", "session_id", @@ -402,6 +425,37 @@ def init_db(): content_rowid=rowid ); + CREATE TABLE IF NOT EXISTS tool_calls ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + event_idx INTEGER NOT NULL, + line_number INTEGER NOT NULL, + byte_offset INTEGER NOT NULL, + byte_length INTEGER NOT NULL, + block_index INTEGER NOT NULL DEFAULT 0, + ts TEXT, + tool_name TEXT, + raw_name TEXT, + call_id TEXT, + input_text TEXT NOT NULL, + original_bytes INTEGER NOT NULL DEFAULT 0, + input_truncated INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (session_id) REFERENCES sessions(id), + UNIQUE(session_id, event_idx) + ); + CREATE INDEX IF NOT EXISTS idx_tool_calls_session_idx + ON tool_calls(session_id, event_idx); + CREATE INDEX IF NOT EXISTS idx_tool_calls_name + ON tool_calls(tool_name); + CREATE INDEX IF NOT EXISTS idx_tool_calls_call_id + ON tool_calls(call_id); + + CREATE VIRTUAL TABLE IF NOT EXISTS tool_calls_fts USING fts5( + input_text, + content=tool_calls, + content_rowid=id + ); + CREATE TABLE IF NOT EXISTS aggregates ( key TEXT PRIMARY KEY, value TEXT, diff --git a/chatview/db/sessions.py b/chatview/db/sessions.py index 07e741f..ec8e608 100644 --- a/chatview/db/sessions.py +++ b/chatview/db/sessions.py @@ -140,6 +140,7 @@ def rebuild_fts(): "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')" ) conn.execute("INSERT INTO sessions_fts(sessions_fts) VALUES('rebuild')") + conn.execute("INSERT INTO tool_calls_fts(tool_calls_fts) VALUES('rebuild')") maybe_commit(conn) @@ -157,9 +158,14 @@ def verify_fts_integrity() -> bool: ).fetchone()[0] sessions = conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] sessions_fts = conn.execute("SELECT COUNT(*) FROM sessions_fts_docsize").fetchone()[0] + tool_calls = conn.execute("SELECT COUNT(*) FROM tool_calls").fetchone()[0] + tool_calls_fts = conn.execute( + "SELECT COUNT(*) FROM tool_calls_fts_docsize" + ).fetchone()[0] return ( messages == messages_fts == messages_fts_trigram and sessions == sessions_fts + and tool_calls == tool_calls_fts ) @@ -172,6 +178,10 @@ def prune_stale_sessions(valid_file_paths) -> int: if not stale_ids: return 0 + from .tool_calls import delete_tool_calls_for_sessions + + delete_tool_calls_for_sessions(stale_ids) + # 分批规避 SQLite 宿主参数上限：先批量取出待删消息的 rowid 删 FTS，再分批删各表。 msg_ids = [ r["id"] diff --git a/chatview/db/tool_calls.py b/chatview/db/tool_calls.py new file mode 100644 index 0000000..62bacb6 --- /dev/null +++ b/chatview/db/tool_calls.py @@ -0,0 +1,199 @@ +"""Tool-call index CRUD and bounded search helpers.""" + +from __future__ import annotations + +import re +import sqlite3 +from datetime import datetime, timedelta + +from .core import get_conn, maybe_commit, query_in_chunks + + +def replace_tool_calls(session_id: str, calls: list) -> None: + conn = get_conn() + old_ids = [ + row[0] + for row in conn.execute( + "SELECT id FROM tool_calls WHERE session_id=?", (session_id,) + ).fetchall() + ] + for start in range(0, len(old_ids), 900): + chunk = old_ids[start : start + 900] + placeholders = ",".join("?" * len(chunk)) + conn.execute( + f"DELETE FROM tool_calls_fts WHERE rowid IN ({placeholders})", chunk + ) + conn.execute("DELETE FROM tool_calls WHERE session_id=?", (session_id,)) + + if calls: + conn.executemany( + """INSERT INTO tool_calls + (session_id, event_idx, line_number, byte_offset, byte_length, + block_index, ts, tool_name, raw_name, call_id, input_text, + original_bytes, input_truncated) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""", + [ + ( + session_id, + call.get("event_idx", index), + call.get("line_number", 0), + call.get("byte_offset", 0), + call.get("byte_length", 0), + call.get("block_index", 0), + call.get("ts", ""), + call.get("tool_name", "unknown"), + call.get("raw_name", "unknown"), + call.get("call_id", ""), + call.get("input_text", ""), + call.get("original_bytes", 0), + int(bool(call.get("input_truncated", False))), + ) + for index, call in enumerate(calls) + ], + ) + rows = conn.execute( + "SELECT id, input_text FROM tool_calls WHERE session_id=? ORDER BY id", + (session_id,), + ).fetchall() + conn.executemany( + "INSERT INTO tool_calls_fts(rowid, input_text) VALUES (?,?)", + [(row["id"], row["input_text"]) for row in rows], + ) + maybe_commit(conn) + + +def _sanitize_fts_query(query: str) -> str: + tokens = re.split(r"[\s,;]+", (query or "").strip()) + sanitized = [] + for token in tokens: + clean = re.sub(r'["\*\(\)\{\}\[\]\^~:]', "", token) + if clean: + sanitized.append(f'"{clean}"') + return " ".join(sanitized) + + +def _min_date(date_range: str) -> str: + days = {"1d": 1, "7d": 7, "30d": 30, "90d": 90}.get(date_range) + if days is None: + return "" + return (datetime.utcnow() - timedelta(days=days)).strftime("%Y-%m-%d") + + +def search_tool_calls( + query: str, + *, + source: str = "all", + project: str = "", + date_range: str = "all", + limit: int = 20, + page: int = 1, +) -> list: + conn = get_conn() + filters = [] + params = [] + if source and source != "all": + filters.append("s.source=?") + params.append(source) + if project: + escaped = ( + project.casefold() + .replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + ) + needle = f"%{escaped}%" + filters.append( + """( + LOWER(COALESCE(s.project_key, '')) LIKE ? ESCAPE '\\' + OR LOWER(COALESCE(s.project_display, '')) LIKE ? ESCAPE '\\' + OR LOWER(COALESCE(s.project_name, '')) LIKE ? ESCAPE '\\' + )""" + ) + params.extend([needle, needle, needle]) + min_date = _min_date(date_range) + if min_date: + filters.append("COALESCE(NULLIF(tc.ts, ''), s.date, '') >= ?") + params.append(min_date) + filter_sql = (" AND " + " AND ".join(filters)) if filters else "" + page_size = max(int(limit), 1) + offset = (max(int(page), 1) - 1) * page_size + columns = """ + tc.id, tc.session_id, tc.event_idx, tc.line_number, tc.byte_offset, + tc.byte_length, tc.block_index, tc.ts, tc.tool_name, tc.raw_name, + tc.call_id, tc.input_text, tc.original_bytes, tc.input_truncated, + s.title, s.project_name, s.project_key, s.project_display, s.source, + s.file_path + """ + safe_query = _sanitize_fts_query(query) + if safe_query: + sql = f""" + SELECT {columns} + FROM tool_calls_fts fts + JOIN tool_calls tc ON fts.rowid=tc.id + JOIN sessions s ON tc.session_id=s.id + WHERE tool_calls_fts MATCH ?{filter_sql} + ORDER BY rank, + COALESCE(NULLIF(tc.ts, ''), s.date, '') DESC, + tc.session_id, tc.event_idx, tc.id + LIMIT ? OFFSET ? + """ + try: + rows = conn.execute( + sql, [safe_query, *params, page_size, offset] + ).fetchall() + if rows: + return [dict(row) for row in rows] + except sqlite3.OperationalError: + pass + + sql = f""" + SELECT {columns} + FROM tool_calls tc + JOIN sessions s ON tc.session_id=s.id + WHERE tc.input_text LIKE ?{filter_sql} + ORDER BY COALESCE(NULLIF(tc.ts, ''), s.date, '') DESC, + tc.session_id, tc.event_idx, tc.id + LIMIT ? OFFSET ? + """ + rows = conn.execute( + sql, [f"%{query}%", *params, page_size, offset] + ).fetchall() + return [dict(row) for row in rows] + + +def get_tool_call(session_id: str, event_idx: int) -> dict | None: + row = get_conn().execute( + """SELECT tc.*, s.title, s.project_name, s.source, s.file_path, + s.file_size, s.file_mtime + FROM tool_calls tc + JOIN sessions s ON tc.session_id=s.id + WHERE tc.session_id=? AND tc.event_idx=?""", + (session_id, event_idx), + ).fetchone() + return dict(row) if row else None + + +def delete_tool_calls_for_sessions(session_ids: list) -> None: + if not session_ids: + return + conn = get_conn() + ids = [ + row["id"] + for row in query_in_chunks( + conn, + "SELECT id FROM tool_calls WHERE session_id IN ({placeholders})", + session_ids, + ) + ] + for start in range(0, len(ids), 900): + chunk = ids[start : start + 900] + placeholders = ",".join("?" * len(chunk)) + conn.execute( + f"DELETE FROM tool_calls_fts WHERE rowid IN ({placeholders})", chunk + ) + for start in range(0, len(session_ids), 900): + chunk = session_ids[start : start + 900] + placeholders = ",".join("?" * len(chunk)) + conn.execute( + f"DELETE FROM tool_calls WHERE session_id IN ({placeholders})", chunk + ) diff --git a/chatview/index.py b/chatview/index.py index 53e0145..7d5e7e9 100644 --- a/chatview/index.py +++ b/chatview/index.py @@ -19,7 +19,7 @@ PROJECTS_DIR = Path.home() / ".claude" / "projects" CACHE_DIR = Path(__file__).resolve().parent.parent / ".cache" INDEX_CACHE = CACHE_DIR / "index.json" -INDEX_SCHEMA_VERSION = PROJECT_IDENTITY_VERSION +INDEX_SCHEMA_VERSION = f"{PROJECT_IDENTITY_VERSION}:tool-calls-v1" MAX_SEARCH_WORKERS = 8 # Codex CLI paths @@ -279,6 +279,9 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: meta.get("userTexts", []), meta.get("assistantSnippets", []), ) + _db.replace_tool_calls( + meta["id"], meta.get("_tool_calls", []) + ) _store_session_insights(meta) _db.upsert_insight_state(meta["id"], current_files.get(fp, 0)) bulk_n += 1 @@ -358,6 +361,9 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: meta.get("userTexts", []), meta.get("assistantSnippets", []), ) + _db.replace_tool_calls( + meta["id"], meta.get("_tool_calls", []) + ) _store_session_insights(meta) _db.upsert_insight_state(meta["id"], current_files.get(fp, 0)) bulk_n += 1 @@ -429,6 +435,9 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: _db.upsert_session( meta, meta.get("userTexts", []), meta.get("assistantSnippets", []) ) + _db.replace_tool_calls( + meta["id"], meta.get("_tool_calls", []) + ) backfill_count += 1 if backfill_count % 50 == 0: _db.bulk_commit() @@ -511,6 +520,7 @@ def _parse_for_backfill(args): "_insight_files", "_insight_errors", "_insight_snippets", + "_tool_calls", ): meta.pop(k, None) diff --git a/chatview/parsers/claude.py b/chatview/parsers/claude.py index c24ba99..b5cfff0 100644 --- a/chatview/parsers/claude.py +++ b/chatview/parsers/claude.py @@ -10,6 +10,7 @@ from chatview.utils.constants import MAX_TOOL_RESULT_LEN from chatview.utils.text import normalize_error as _normalize_error +from chatview.tool_events import make_tool_call_record # --------------------------------------------------------------------------- @@ -43,6 +44,7 @@ def extract_metadata(filepath: str): last_ts = None user_texts = [] # (message_index, text, timestamp) assistant_snippets = [] # (message_index, assistant text for search) + tool_calls = [] msg_index = 0 # Insight extraction accumulators _tool_daily = {} # (day, tool_name) -> count @@ -62,10 +64,17 @@ def extract_metadata(filepath: str): _prev_user_msg = "" try: - with open(filepath, "r", encoding="utf-8", errors="replace") as f: - for line in f: + with open(filepath, "rb") as f: + line_number = 0 + while True: + byte_offset = f.tell() + raw_line = f.readline() + if not raw_line: + break + line_number += 1 + byte_length = len(raw_line) try: - obj = json.loads(line) + obj = json.loads(raw_line.decode("utf-8", errors="replace")) except json.JSONDecodeError: continue @@ -113,7 +122,7 @@ def extract_metadata(filepath: str): _code_blocks = [] _tool_writes = [] if isinstance(a_content, list): - for blk in a_content: + for block_index, blk in enumerate(a_content): if isinstance(blk, dict) and blk.get("type") == "text": t = blk.get("text", "").strip() if t: @@ -134,6 +143,20 @@ def extract_metadata(filepath: str): ): # Insight: tool usage + file refs tool_name = blk.get("name", "unknown") + tool_calls.append( + make_tool_call_record( + event_idx=len(tool_calls), + line_number=line_number, + byte_offset=byte_offset, + byte_length=byte_length, + block_index=block_index, + ts=ts, + tool_name=tool_name, + raw_name=tool_name, + call_id=blk.get("id", ""), + tool_input=blk.get("input", {}), + ) + ) day = (first_ts or "")[:10] if day: key = (day, tool_name) @@ -220,6 +243,7 @@ def extract_metadata(filepath: str): "_insight_files": _file_refs, "_insight_errors": _error_list, "_insight_snippets": _snippet_list, + "_tool_calls": tool_calls, } diff --git a/chatview/parsers/codex.py b/chatview/parsers/codex.py index efc8288..bd5b401 100644 --- a/chatview/parsers/codex.py +++ b/chatview/parsers/codex.py @@ -10,6 +10,7 @@ from chatview.parsers.claude import _truncate_tool_output, _strip_tags from chatview.utils.text import normalize_error as _normalize_error +from chatview.tool_events import CODEX_TOOL_NAMES, make_tool_call_record CODEX_DIR = Path.home() / ".codex" CODEX_SESSIONS_DIR = CODEX_DIR / "sessions" @@ -19,15 +20,7 @@ # Module-level state _codex_titles = {} # session_id -> thread_name -_CODEX_TOOL_NAMES = { - "shell": "Bash", - "exec_command": "Bash", - "write_stdin": "Bash", - "apply_patch": "Edit", - "read_file": "Read", - "write_file": "Write", - "list_directory": "Glob", -} +_CODEX_TOOL_NAMES = CODEX_TOOL_NAMES def _normalize_codex_tool_input(raw_name: str, args: dict) -> dict: @@ -129,6 +122,7 @@ def extract_codex_metadata(filepath: str): cwd = None user_texts = [] assistant_snippets = [] + tool_calls = [] msg_index = 0 # Insight extraction accumulators _tool_daily = {} @@ -145,10 +139,17 @@ def extract_codex_metadata(filepath: str): ) try: - with open(filepath, "r", encoding="utf-8", errors="replace") as f: - for line in f: + with open(filepath, "rb") as f: + line_number = 0 + while True: + byte_offset = f.tell() + raw_line = f.readline() + if not raw_line: + break + line_number += 1 + byte_length = len(raw_line) try: - obj = json.loads(line) + obj = json.loads(raw_line.decode("utf-8", errors="replace")) except json.JSONDecodeError: continue @@ -193,6 +194,25 @@ def extract_codex_metadata(filepath: str): # Insight: tool usage + file refs raw_name = payload.get("name", "unknown") tool_name = _CODEX_TOOL_NAMES.get(raw_name, raw_name) + tool_input = ( + payload.get("arguments", "") + if p_type == "function_call" + else payload.get("input", "") + ) + tool_calls.append( + make_tool_call_record( + event_idx=len(tool_calls), + line_number=line_number, + byte_offset=byte_offset, + byte_length=byte_length, + block_index=0, + ts=ts, + tool_name=tool_name, + raw_name=raw_name, + call_id=payload.get("call_id", ""), + tool_input=tool_input, + ) + ) day = (first_ts or "")[:10] if day: key = (day, tool_name) @@ -265,6 +285,7 @@ def extract_codex_metadata(filepath: str): "_insight_files": _file_refs, "_insight_errors": _error_list, "_insight_snippets": [], + "_tool_calls": tool_calls, } diff --git a/chatview/tool_events.py b/chatview/tool_events.py new file mode 100644 index 0000000..33f4974 --- /dev/null +++ b/chatview/tool_events.py @@ -0,0 +1,128 @@ +"""Bounded tool-call indexing and precise JSONL event extraction helpers.""" + +from __future__ import annotations + +import json + + +TOOL_INPUT_MAX_BYTES = 2000 + +CODEX_TOOL_NAMES = { + "shell": "Bash", + "exec_command": "Bash", + "write_stdin": "Bash", + "apply_patch": "Edit", + "read_file": "Read", + "write_file": "Write", + "list_directory": "Glob", +} + + +def stringify_tool_input(value) -> str: + """Return stable searchable text for a tool input without tool results.""" + if isinstance(value, str): + return value + if value is None: + return "" + try: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + except (TypeError, ValueError): + return str(value) + + +def truncate_utf8(text: str, max_bytes: int = TOOL_INPUT_MAX_BYTES) -> tuple[str, int, bool]: + """Clip text to a UTF-8 byte budget without leaving a partial codepoint.""" + raw = (text or "").encode("utf-8", errors="replace") + if len(raw) <= max_bytes: + return text or "", len(raw), False + clipped = raw[:max_bytes].decode("utf-8", errors="ignore") + return clipped, len(raw), True + + +def make_tool_call_record( + *, + event_idx: int, + line_number: int, + byte_offset: int, + byte_length: int, + block_index: int, + ts: str, + tool_name: str, + raw_name: str, + call_id: str, + tool_input, + max_bytes: int = TOOL_INPUT_MAX_BYTES, +) -> dict: + text = stringify_tool_input(tool_input) + clipped, original_bytes, truncated = truncate_utf8(text, max_bytes=max_bytes) + return { + "event_idx": event_idx, + "line_number": line_number, + "byte_offset": byte_offset, + "byte_length": byte_length, + "block_index": block_index, + "ts": ts or "", + "tool_name": tool_name or raw_name or "unknown", + "raw_name": raw_name or tool_name or "unknown", + "call_id": call_id or "", + "input_text": clipped, + "original_bytes": original_bytes, + "input_truncated": truncated, + } + + +def read_jsonl_record(filepath: str, byte_offset: int, byte_length: int) -> dict: + """Read exactly one indexed JSONL record and parse it locally.""" + if byte_offset < 0 or byte_length <= 0: + raise ValueError("Invalid JSONL byte locator") + with open(filepath, "rb") as handle: + handle.seek(byte_offset) + raw = handle.read(byte_length) + if len(raw) != byte_length: + raise RuntimeError("Indexed JSONL locator is stale; run `distill refresh`") + try: + return json.loads(raw.decode("utf-8", errors="replace")) + except json.JSONDecodeError as exc: + raise RuntimeError( + "Indexed JSONL event no longer parses; run `distill refresh`" + ) from exc + + +def tool_call_from_record(obj: dict, source: str, block_index: int = 0) -> dict: + """Project one tool call from an indexed raw record without returning the line.""" + if source == "codex": + if obj.get("type") != "response_item": + raise RuntimeError("Indexed record is no longer a Codex tool call") + payload = obj.get("payload") or {} + payload_type = payload.get("type") + if payload_type == "function_call": + tool_input = payload.get("arguments", "") + elif payload_type == "custom_tool_call": + tool_input = payload.get("input", "") + else: + raise RuntimeError("Indexed record is no longer a Codex tool call") + raw_name = payload.get("name", "unknown") + return { + "timestamp": obj.get("timestamp", ""), + "toolName": CODEX_TOOL_NAMES.get(raw_name, raw_name), + "rawName": raw_name, + "callId": payload.get("call_id", ""), + "input": stringify_tool_input(tool_input), + } + + if obj.get("type") != "assistant": + raise RuntimeError("Indexed record is no longer a Claude tool call") + content = (obj.get("message") or {}).get("content", []) + if not isinstance(content, list) or not 0 <= block_index < len(content): + raise RuntimeError("Indexed Claude tool block no longer exists") + block = content[block_index] + if not isinstance(block, dict) or block.get("type") != "tool_use": + raise RuntimeError("Indexed Claude block is no longer a tool call") + raw_name = block.get("name", "unknown") + return { + "timestamp": obj.get("timestamp", ""), + "toolName": raw_name, + "rawName": raw_name, + "callId": block.get("id", ""), + "input": stringify_tool_input(block.get("input", {})), + } diff --git a/tests/test_tool_call_retrieval.py b/tests/test_tool_call_retrieval.py new file mode 100644 index 0000000..cb0b331 --- /dev/null +++ b/tests/test_tool_call_retrieval.py @@ -0,0 +1,188 @@ +import json +import shutil +import tempfile +import threading +import unittest +from pathlib import Path +from types import SimpleNamespace + +from chatview import db +from chatview.commands.tool_calls import read_tool_event_data, tool_search_data +from chatview.db import core as dbcore +from chatview.parsers.claude import extract_metadata +from chatview.parsers.codex import extract_codex_metadata + + +class ToolCallRetrievalTestCase(unittest.TestCase): + def setUp(self): + self._orig_db_path = dbcore.DB_PATH + self._orig_cache_dir = dbcore.CACHE_DIR + self._tmpdir = Path(tempfile.mkdtemp()) + dbcore.CACHE_DIR = self._tmpdir + dbcore.DB_PATH = self._tmpdir / "sessions.db" + dbcore._local = threading.local() + db.init_db() + + def tearDown(self): + dbcore.DB_PATH = self._orig_db_path + dbcore.CACHE_DIR = self._orig_cache_dir + dbcore._local = threading.local() + shutil.rmtree(self._tmpdir, ignore_errors=True) + + def _args(self, **overrides): + values = { + "source": "all", + "project": "", + "date": "all", + "limit": 20, + "page": 1, + "max_chars": 500, + } + values.update(overrides) + return SimpleNamespace(**values) + + def _store_meta(self, meta): + meta["project"] = "fixture" + meta["projectName"] = "fixture" + meta["source"] = meta.get("source", "claude") + meta["_mtime"] = Path(meta["filePath"]).stat().st_mtime + db.upsert_session( + meta, + meta.get("userTexts", []), + meta.get("assistantSnippets", []), + ) + db.replace_tool_calls(meta["id"], meta.get("_tool_calls", [])) + + def test_codex_indexes_calls_only_and_reads_one_raw_event(self): + target = ".verify-search-fix-result-v2.json" + call_input = ( + "*** Begin Patch\n*** Add File: /workspace/" + target + "\n" + + "x" * 3000 + ) + path = self._tmpdir / "codex.jsonl" + records = [ + { + "timestamp": "2026-07-17T20:00:00Z", + "type": "session_meta", + "payload": {"id": "codex-tool-fixture", "cwd": "/workspace"}, + }, + { + "timestamp": "2026-07-17T20:01:00Z", + "type": "response_item", + "payload": { + "type": "custom_tool_call", + "name": "apply_patch", + "call_id": "call-create-result", + "input": call_input, + }, + }, + { + "timestamp": "2026-07-17T20:01:01Z", + "type": "response_item", + "payload": { + "type": "custom_tool_call_output", + "call_id": "call-create-result", + "output": "result_only_secret must never be indexed", + }, + }, + ] + path.write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in records), + encoding="utf-8", + ) + + meta = extract_codex_metadata(str(path)) + self.assertEqual(len(meta["_tool_calls"]), 1) + call = meta["_tool_calls"][0] + self.assertLessEqual(len(call["input_text"].encode("utf-8")), 2000) + self.assertTrue(call["input_truncated"]) + self.assertGreater(call["original_bytes"], 2000) + self.assertGreater(call["byte_offset"], 0) + self.assertGreater(call["byte_length"], 0) + + self._store_meta(meta) + hits = tool_search_data(target, self._args()) + self.assertEqual(len(hits), 1) + self.assertEqual(hits[0]["sessionId"], "codex-codex-tool-fixture") + self.assertEqual(hits[0]["eventIdx"], 0) + self.assertEqual(hits[0]["rawName"], "apply_patch") + self.assertEqual( + tool_search_data("result_only_secret", self._args()), [] + ) + + event = read_tool_event_data(hits[0]["sessionId"], 0, max_chars=80) + self.assertEqual(event["callId"], "call-create-result") + self.assertEqual(event["rawName"], "apply_patch") + self.assertIn(target, event["input"]) + self.assertEqual(len(event["input"]), 80) + self.assertTrue(event["outputTruncated"]) + self.assertNotIn("result_only_secret", json.dumps(event)) + + def test_claude_locator_selects_the_exact_tool_block(self): + path = self._tmpdir / "claude.jsonl" + records = [ + { + "type": "user", + "sessionId": "claude-tool-fixture", + "timestamp": "2026-07-17T20:00:00Z", + "message": {"content": [{"type": "text", "text": "inspect tools"}]}, + }, + { + "type": "assistant", + "timestamp": "2026-07-17T20:01:00Z", + "message": { + "content": [ + {"type": "text", "text": "Checking."}, + { + "type": "tool_use", + "id": "tool-first", + "name": "Read", + "input": {"file_path": "/workspace/first.json"}, + }, + { + "type": "tool_use", + "id": "tool-second", + "name": "Edit", + "input": {"file_path": "/workspace/second.json"}, + }, + ] + }, + }, + { + "type": "user", + "toolUseResult": {"status": "ok"}, + "timestamp": "2026-07-17T20:01:01Z", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "tool-second", + "content": "claude_result_only_secret", + } + ] + }, + }, + ] + path.write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in records), + encoding="utf-8", + ) + + meta = extract_metadata(str(path)) + self.assertEqual(len(meta["_tool_calls"]), 2) + self.assertEqual([row["block_index"] for row in meta["_tool_calls"]], [1, 2]) + self._store_meta(meta) + + hits = tool_search_data("second.json", self._args()) + self.assertEqual(len(hits), 1) + event = read_tool_event_data(meta["id"], hits[0]["eventIdx"], max_chars=2000) + self.assertEqual(event["callId"], "tool-second") + self.assertIn("second.json", event["input"]) + self.assertNotIn("first.json", event["input"]) + self.assertEqual( + tool_search_data("claude_result_only_secret", self._args()), [] + ) + + +if __name__ == "__main__": + unittest.main() From b4caae78bd71041a003fb294b093d7941826b067 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Fri, 17 Jul 2026 23:18:48 -0400 Subject: [PATCH 03/15] feat: document precise tool provenance workflow --- chatview/cli.py | 7 +++- chatview/commands/tool_calls.py | 36 ++++++++++++++++++-- chatview/tool_events.py | 43 ++++++++++++++++++++++++ skills/distill-yourself/SKILL.md | 37 +++++++++++++++++++- skills/distill-yourself/evals/evals.json | 23 +++++++++++++ tests/test_distill_skill_static.py | 12 +++++++ tests/test_tool_call_retrieval.py | 22 ++++++++++-- 7 files changed, 172 insertions(+), 8 deletions(-) diff --git a/chatview/cli.py b/chatview/cli.py index 96cdefa..f9b25ee 100644 --- a/chatview/cli.py +++ b/chatview/cli.py @@ -206,9 +206,14 @@ def main(): ) p_read_tool.add_argument("session", help="Exact session ID from tool-search") p_read_tool.add_argument("--event-idx", type=int, required=True) + p_read_tool.add_argument( + "--include-result", + action="store_true", + help="Read the matching result from source JSONL without indexing it", + ) p_read_tool.add_argument( "--max-chars", type=_max_chars, default=2000, - help="Maximum tool-input characters to return (default/max: 2000)", + help="Maximum input/result characters to return (default/max: 2000)", ) p_find_repeats = sub.add_parser( diff --git a/chatview/commands/tool_calls.py b/chatview/commands/tool_calls.py index ab06dfd..a9bf5a5 100644 --- a/chatview/commands/tool_calls.py +++ b/chatview/commands/tool_calls.py @@ -6,7 +6,11 @@ import os from chatview.snippets import make_query_snippet -from chatview.tool_events import read_jsonl_record, tool_call_from_record +from chatview.tool_events import ( + find_tool_result, + read_jsonl_record, + tool_call_from_record, +) def tool_search_data(query: str, args) -> list: @@ -88,7 +92,12 @@ def cmd_tool_search(args) -> None: print() -def read_tool_event_data(session_id: str, event_idx: int, max_chars: int = 2000) -> dict: +def read_tool_event_data( + session_id: str, + event_idx: int, + max_chars: int = 2000, + include_result: bool = False, +) -> dict: from chatview import db as _db _db.prepare_search_db() @@ -114,7 +123,7 @@ def read_tool_event_data(session_id: str, event_idx: int, max_chars: int = 2000) raise RuntimeError("Indexed tool call changed; run `distill refresh`") full_input = event.pop("input", "") - max_chars = max(int(max_chars), 1) + max_chars = min(max(int(max_chars), 1), 2000) event["input"] = full_input[:max_chars] event.update( { @@ -132,6 +141,21 @@ def read_tool_event_data(session_id: str, event_idx: int, max_chars: int = 2000) "outputTruncated": len(full_input) > max_chars, } ) + if include_result: + result_text = find_tool_result( + filepath, + int(locator.get("byte_offset", 0)) + int(locator.get("byte_length", 0)), + locator.get("source", ""), + event.get("callId", ""), + ) + if result_text is None: + event["result"] = None + event["resultOriginalChars"] = 0 + event["resultTruncated"] = False + else: + event["result"] = result_text[:max_chars] + event["resultOriginalChars"] = len(result_text) + event["resultTruncated"] = len(result_text) > max_chars return event @@ -141,6 +165,7 @@ def cmd_read_tool_event(args) -> None: args.session, args.event_idx, max_chars=getattr(args, "max_chars", 2000), + include_result=getattr(args, "include_result", False), ) except (KeyError, FileNotFoundError, RuntimeError, ValueError) as exc: raise SystemExit(str(exc)) from exc @@ -158,3 +183,8 @@ def cmd_read_tool_event(args) -> None: print(result["input"]) if result["outputTruncated"]: print("\n[TRUNCATED — increase --max-chars up to 2000 only if needed]") + if "result" in result: + print(f"\n--- TOOL RESULT ({result['resultOriginalChars']} chars) ---") + print(result["result"] if result["result"] is not None else "[not found]") + if result["resultTruncated"]: + print("\n[TRUNCATED — tool results are never stored in the index]") diff --git a/chatview/tool_events.py b/chatview/tool_events.py index 33f4974..01dab09 100644 --- a/chatview/tool_events.py +++ b/chatview/tool_events.py @@ -126,3 +126,46 @@ def tool_call_from_record(obj: dict, source: str, block_index: int = 0) -> dict: "callId": block.get("id", ""), "input": stringify_tool_input(block.get("input", {})), } + + +def find_tool_result( + filepath: str, + start_offset: int, + source: str, + call_id: str, +) -> str | None: + """Find a call's result after its indexed record without returning raw lines.""" + if not call_id: + return None + with open(filepath, "rb") as handle: + handle.seek(max(int(start_offset), 0)) + for raw in handle: + try: + obj = json.loads(raw.decode("utf-8", errors="replace")) + except json.JSONDecodeError: + continue + if source == "codex": + if obj.get("type") != "response_item": + continue + payload = obj.get("payload") or {} + if ( + payload.get("type") + in ("function_call_output", "custom_tool_call_output") + and payload.get("call_id", "") == call_id + ): + return stringify_tool_input(payload.get("output", "")) + continue + + if obj.get("type") != "user": + continue + content = (obj.get("message") or {}).get("content", []) + if not isinstance(content, list): + continue + for block in content: + if ( + isinstance(block, dict) + and block.get("type") == "tool_result" + and block.get("tool_use_id", "") == call_id + ): + return stringify_tool_input(block.get("content", "")) + return None diff --git a/skills/distill-yourself/SKILL.md b/skills/distill-yourself/SKILL.md index 7c36ca7..e7c425f 100644 --- a/skills/distill-yourself/SKILL.md +++ b/skills/distill-yourself/SKILL.md @@ -10,7 +10,10 @@ description: "本地 Claude Code/Codex/agent 对话历史的唯一检索入口 ## Core Rules - 检索本地 Claude Code、Codex 或 agent 对话历史时，先用 `distill` 定位并核验证据，不先扫描原始 transcript。 -- 只有当 `distill` 无法提供所需的工具调用、压缩边界或原始事件细节时，才定向读取原始 session JSONL；使用后备路径时说明原因，并继续遵守证据纪律。 +- 追查“哪个 Agent 用什么工具、命令或 patch 产生了某个文件”时，先用 `tool-search` 定位，再用 `read-tool-event` 定点核验；不要从消息搜索直接跳到原始 JSONL。 +- 工具索引只保存每条调用参数最多 2 KB 的搜索副本和原 JSONL 坐标，不保存工具结果。需要结果时给 `read-tool-event` 加 `--include-result`，由它从原 JSONL 定点读取并截断。 +- 只有当 `distill` 的消息窗口和工具事件读取都无法提供压缩边界或特殊旧格式细节时，才定向读取原始 session JSONL；说明原因，结构化投影必要字段并设置硬输出上限。 +- 不要对 session JSONL 运行会回显整行的 `rg -n`，也不要用 `head` 或 `sed` 把“若干行”当作小窗口。JSONL 单行可能包含数万 token；`rg -l` 只能用于定位文件，不能作为证据读取器。 - 若任务还匹配其他工作流 skill，先完成历史取证，再把核验后的上下文交给后续流程。 - `distill` 只负责取数、检索和暂存；结论由你基于证据生成。 - `profile-digest` / `aggregates` / `stats` 是地图，不是结论。写入 Memory/Profile/Twin 前必须用 `read-window`、`session-brief` 或原 session 内容核验。 @@ -46,6 +49,32 @@ distill search "" --recall normal --role user --evidence-only --format js distill read-window --batch '[{"sessionId":"session_id","idx":123,"radius":2}]' ``` +For exact tool provenance, use a separate bounded ladder. If the user already gives a filename, path, command fragment, patch target, or call ID, start at step 2: + +```text +1. EPISODE -> use message search only when you still need to identify the episode +2. LOCATE -> tool-search the narrowest stable anchor +3. SELECT -> keep the best 1-3 sessionId + eventIdx locators +4. VERIFY -> read-tool-event for the exact call input +5. RESULT -> add --include-result only when the result is necessary +6. STOP -> answer as soon as the provenance is established +``` + +```bash +distill refresh +distill tool-search "" --format jsonl --max-chars 500 --limit 20 --page 1 --date 90d +distill read-tool-event --event-idx --max-chars 2000 +distill read-tool-event --event-idx --include-result --max-chars 2000 +``` + +`tool-search` searches only bounded call inputs, never tool results. Its `filePath` is provenance, not permission to print that JSONL. `read-tool-event` seeks to the indexed byte range, projects the selected call, and caps model-visible output. If the result is requested, it follows the indexed `callId` locally and applies the same cap without persisting the result. + +Whenever you recommend `--include-result`, say explicitly that the result is read on demand from the source JSONL and is not stored in the SQLite search index. This distinction explains why the command is safe and why `tool-search` cannot search result bodies. + +`--max-chars 2000` is a hard ceiling for tool-event reads, not a first step in progressive expansion. If the needed fact is outside the returned projection, report the bounded-reader limitation or refine the anchor; never propose 4000, 10000, or an unbounded raw read. + +If `tool-search` misses, refresh once, retry with a shorter stable basename or command fragment, then use message search to identify a better anchor. Do not compensate by increasing `--limit`, globally grepping transcripts, or reading the beginning of a rollout file. + Choose the scope before ORIENT, and use ORIENT only for broad profile, trend, correction-pattern, or cognitive-model analysis. If the user asks about a recent/project-specific topic, carry that `--date` / `--project` into the orienting commands; otherwise use `--date all --source all`. For broad global analysis, start with: @@ -76,6 +105,7 @@ Expand to `all` only when evidence is thin. | Topic, project, or historical episode | `distill find-repeats "" --limit 5 --json` | `session-brief` / `read-window --batch` for top candidates | | Possible orchestration noise | `distill evidence-audit --json` | Treat `artifactReason` rows as diagnostic only | | Session-level context | `distill session-brief ` | `distill read-window --idx N` | +| Tool call, command, patch, or file provenance | `distill tool-search "" --format jsonl --limit 20 --page 1` | `distill read-tool-event --event-idx N`; add `--include-result` only if needed | Important commands: @@ -90,6 +120,9 @@ Important commands: | `distill read-window --idx N --radius 2` | Small context window around a hit | | `distill read-window --batch '[...]'` | Verify several windows at once | | `distill evidence-audit --json` | Estimate contamination from prompts/tasks/context noise | +| `distill tool-search "" --format jsonl --limit 20 --page 1` | Bounded search over tool-call inputs with JSONL locators | +| `distill read-tool-event --event-idx N --max-chars 2000` | Read exactly one indexed call from its source JSONL | +| `distill read-tool-event --event-idx N --include-result --max-chars 2000` | Also read the matching result locally without indexing it | `corrections` JSON exposes a stable `idx` and its nearest user/assistant pair. `search` JSON/JSONL exposes `sessionId`, `idx`, and `messageIndex`; use `sessionId + idx` as the read coordinate. `idx` is only ordered within one session, not a global message id. `read-window --radius N` reads the inclusive numeric range `idx-N ... idx+N`, so gaps may produce fewer than `2N+1` messages. @@ -103,6 +136,8 @@ For manual triage, prefer `--format lines`; for scripts or incremental aggregati Stop retrieval when you have enough direct user evidence. More searching after 2-4 strong, verified quotes usually adds noise. +For tool provenance, stop after 1-3 exact verified events. Tool outputs are routing or execution evidence, not durable evidence of a user preference. + ## Required References Before doing any task below, read the matching reference. Do not rely on memory for product-specific SOPs. diff --git a/skills/distill-yourself/evals/evals.json b/skills/distill-yourself/evals/evals.json index fe8eb96..67f995b 100644 --- a/skills/distill-yourself/evals/evals.json +++ b/skills/distill-yourself/evals/evals.json @@ -23,6 +23,29 @@ "Requires direct user evidence from at least two sessions and verifies selected windows", "Does not use search-plus, --limit 10, or default inline context" ] + }, + { + "id": 3, + "prompt": "我看到仓库根目录有一个 .verify-search-fix-result-v2.json。请只写出你会如何从历史对话追查是哪个 Agent、通过什么工具调用创建它的命令序列，不要真正执行。", + "expected_output": "直接用 tool-search 搜索精确文件名，选择少量 sessionId+eventIdx 后用 read-tool-event 定点核验；不使用 rg -n、head、sed 或整份 session read。", + "files": [], + "expectations": [ + "Starts with tool-search on the exact filename using JSONL, limit 20, and page 1", + "Uses sessionId plus eventIdx with read-tool-event and max-chars 2000", + "Does not propose rg -n, head, sed, or printing raw rollout JSONL" + ] + }, + { + "id": 4, + "prompt": "我已经通过工具调用定位到一次失败的测试，现在需要核对这个调用对应的工具结果。只给安全的取证命令和停止条件，不要执行。", + "expected_output": "调用已经定位，因此直接用 read-tool-event --include-result --max-chars 2000 读取对应结果；说明结果不入索引，2000 是硬上限，并在一个精确事件足够时停止。", + "files": [], + "expectations": [ + "Uses read-tool-event with --include-result and max-chars 2000", + "States that tool results are read from source JSONL but are not stored in the index", + "Never raises max-chars above 2000 and never proposes raw JSONL shell reads", + "Stops after the exact event establishes the needed fact" + ] } ] } diff --git a/tests/test_distill_skill_static.py b/tests/test_distill_skill_static.py index 9f0ef49..ca210cf 100644 --- a/tests/test_distill_skill_static.py +++ b/tests/test_distill_skill_static.py @@ -45,6 +45,10 @@ def test_skill_description_routes_conversation_history_without_run_history_false self.assertIn("先用 `distill` 定位并核验证据", text) self.assertIn("不先扫描原始 transcript", text) self.assertIn("才定向读取原始 session JSONL", text) + self.assertIn("不要从消息搜索直接跳到原始 JSONL", text) + self.assertIn("不保存工具结果", text) + self.assertIn("不要对 session JSONL 运行会回显整行的 `rg -n`", text) + self.assertIn("不要用 `head` 或 `sed`", text) self.assertIn("先完成历史取证", text) for coupled_name in ( "read_session.py", @@ -82,6 +86,14 @@ def test_skill_md_is_router_and_declares_required_references(self): self.assertIn("use `sessionId + idx` as the read coordinate", text) self.assertIn('distill search "" --recall normal --format lines', text) self.assertIn('distill search "" --recall high --format jsonl', text) + self.assertIn('distill tool-search ""', text) + self.assertIn("distill read-tool-event --event-idx ", text) + self.assertIn("--include-result --max-chars 2000", text) + self.assertIn("`--max-chars 2000` is a hard ceiling", text) + self.assertIn("never propose 4000, 10000, or an unbounded raw read", text) + self.assertIn("filePath` is provenance, not permission", text) + self.assertIn("result is read on demand from the source JSONL", text) + self.assertIn("is not stored in the SQLite search index", text) self.assertIn("--format lines", text) self.assertIn("--format jsonl", text) self.assertIn("--evidence-only", text) diff --git a/tests/test_tool_call_retrieval.py b/tests/test_tool_call_retrieval.py index cb0b331..d1db39e 100644 --- a/tests/test_tool_call_retrieval.py +++ b/tests/test_tool_call_retrieval.py @@ -110,13 +110,23 @@ def test_codex_indexes_calls_only_and_reads_one_raw_event(self): tool_search_data("result_only_secret", self._args()), [] ) - event = read_tool_event_data(hits[0]["sessionId"], 0, max_chars=80) + event = read_tool_event_data( + hits[0]["sessionId"], 0, max_chars=80, include_result=True + ) self.assertEqual(event["callId"], "call-create-result") self.assertEqual(event["rawName"], "apply_patch") self.assertIn(target, event["input"]) self.assertEqual(len(event["input"]), 80) self.assertTrue(event["outputTruncated"]) - self.assertNotIn("result_only_secret", json.dumps(event)) + self.assertIn("result_only_secret", event["result"]) + self.assertLessEqual(len(event["result"]), 80) + self.assertNotIn("result_only_secret", hits[0]["snippet"]) + + hard_capped = read_tool_event_data( + hits[0]["sessionId"], 0, max_chars=10_000 + ) + self.assertEqual(len(hard_capped["input"]), 2000) + self.assertTrue(hard_capped["outputTruncated"]) def test_claude_locator_selects_the_exact_tool_block(self): path = self._tmpdir / "claude.jsonl" @@ -175,10 +185,16 @@ def test_claude_locator_selects_the_exact_tool_block(self): hits = tool_search_data("second.json", self._args()) self.assertEqual(len(hits), 1) - event = read_tool_event_data(meta["id"], hits[0]["eventIdx"], max_chars=2000) + event = read_tool_event_data( + meta["id"], + hits[0]["eventIdx"], + max_chars=2000, + include_result=True, + ) self.assertEqual(event["callId"], "tool-second") self.assertIn("second.json", event["input"]) self.assertNotIn("first.json", event["input"]) + self.assertEqual(event["result"], "claude_result_only_secret") self.assertEqual( tool_search_data("claude_result_only_secret", self._args()), [] ) From 83baa280b581abf23e73327ce7f4b07e27f88259 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sat, 18 Jul 2026 01:04:25 -0400 Subject: [PATCH 04/15] feat: harden bounded history retrieval --- chatview/cli.py | 5 + chatview/commands/retrieval.py | 256 ++++++++++++--- chatview/commands/tool_calls.py | 71 +++- chatview/db/core.py | 55 +++- chatview/db/tool_calls.py | 36 +- chatview/index.py | 224 +++++++++---- chatview/parsers/claude.py | 97 ++++-- chatview/parsers/codex.py | 72 ++-- chatview/tool_events.py | 215 +++++++++--- chatview/utils/constants.py | 23 ++ skills/distill-yourself/SKILL.md | 30 +- skills/distill-yourself/evals/evals.json | 32 +- tests/test_db.py | 399 +++++++++++++++++++++++ tests/test_distill_skill_static.py | 39 ++- tests/test_retrieval_tools.py | 158 ++++++++- tests/test_tool_call_retrieval.py | 330 ++++++++++++++++++- 16 files changed, 1782 insertions(+), 260 deletions(-) diff --git a/chatview/cli.py b/chatview/cli.py index f9b25ee..a3e310b 100644 --- a/chatview/cli.py +++ b/chatview/cli.py @@ -211,6 +211,11 @@ def main(): action="store_true", help="Read the matching result from source JSONL without indexing it", ) + p_read_tool.add_argument( + "--around", + default="", + help="Center the bounded tool input projection around an exact anchor", + ) p_read_tool.add_argument( "--max-chars", type=_max_chars, default=2000, help="Maximum input/result characters to return (default/max: 2000)", diff --git a/chatview/commands/retrieval.py b/chatview/commands/retrieval.py index adc83d3..bd21308 100644 --- a/chatview/commands/retrieval.py +++ b/chatview/commands/retrieval.py @@ -965,28 +965,90 @@ def find_repeats_data(query: str, args) -> dict: } -def read_window_data(session_id: str, idx: int, radius: int = 2) -> dict: - """Return DB-backed message context around a message index.""" +_READ_WINDOW_MAX_RADIUS = 5 +_READ_WINDOW_MAX_BATCH = 5 +_READ_WINDOW_MESSAGE_CHARS = 1200 +_READ_WINDOW_OUTPUT_BYTES = 20000 + + +def _validated_window_radius(radius: int) -> int: + if isinstance(radius, bool) or isinstance(radius, float): + raise ValueError("radius must be between 0 and 5") + if isinstance(radius, int): + value = radius + elif isinstance(radius, str) and re.fullmatch(r"[+-]?\d+", radius.strip()): + value = int(radius) + else: + raise ValueError("radius must be between 0 and 5") + if not 0 <= value <= _READ_WINDOW_MAX_RADIUS: + raise ValueError("radius must be between 0 and 5") + return value + + +def _truncate_window_text( + text: str, max_chars: int = _READ_WINDOW_MESSAGE_CHARS +) -> tuple[str, bool]: + value = text or "" + if len(value) <= max_chars: + return value, False + marker = f"\n[… truncated from {len(value)} chars]" + return value[:max(max_chars - len(marker), 0)] + marker[:max_chars], True + + +def _window_json(data: dict) -> str: + """Serialize bounded read-window output and keep outputBytes exact.""" + for _ in range(8): + rendered = json.dumps(data, ensure_ascii=False, separators=(",", ":")) + "\n" + size = len(rendered.encode("utf-8")) + if data.get("outputBytes") == size: + return rendered + data["outputBytes"] = size + return json.dumps(data, ensure_ascii=False, separators=(",", ":")) + "\n" + + +def _refresh_window_budget_metadata(data: dict, windows: list[dict]): + for window in windows: + window["outputTruncated"] = bool( + window["omittedMessages"] + or any(message["outputTruncated"] for message in window["messages"]) + ) + data["omittedMessages"] = sum(window["omittedMessages"] for window in windows) + data["outputTruncated"] = bool( + data["omittedMessages"] or any(window["outputTruncated"] for window in windows) + ) + _window_json(data) + + +def _read_window_unbudgeted( + session_id: str, idx: int, radius: int +) -> tuple[dict, list[tuple[dict, str]]]: + """Load one DB window and return public message projections plus source text.""" from chatview import db as _db + radius = _validated_window_radius(radius) _db.init_db() meta = _db.get_session_meta(session_id) if not meta: raise KeyError(f"Session not found: {session_id}") sid = meta["id"] - radius = max(0, int(radius)) start = idx - radius end = idx + radius - messages = [ - { - "idx": m.get("idx"), - "role": m.get("role", ""), - "ts": m.get("ts", ""), - "text": m.get("text", ""), + messages = [] + candidates = [] + for message in _db.get_message_window(sid, start, end): + source_text = message.get("text", "") or "" + text, truncated = _truncate_window_text(source_text) + public = { + "idx": message.get("idx"), + "role": message.get("role", ""), + "ts": message.get("ts", ""), + "text": text, + "originalChars": len(source_text), + "outputTruncated": truncated, } - for m in _db.get_message_window(sid, start, end) - ] - return { + messages.append(public) + candidates.append((public, source_text)) + window = { "sessionId": sid, "title": meta.get("title") or "Untitled", "project": meta.get("project_name") or "", @@ -995,21 +1057,110 @@ def read_window_data(session_id: str, idx: int, radius: int = 2) -> dict: "targetIndex": idx, "radius": radius, "messages": messages, + "omittedMessages": 0, + "outputTruncated": any(message["outputTruncated"] for message in messages), } + return window, candidates + + +def _budget_read_windows( + raw_windows: list[tuple[dict, list[tuple[dict, str]]]], batch: bool +) -> dict: + """Apply one output budget, preserving every target before surrounding context.""" + windows = [window for window, _ in raw_windows] + targets = [] + contexts = [] + for window_index, (window, window_candidates) in enumerate(raw_windows): + window["messages"] = [] + window["omittedMessages"] = len(window_candidates) + for message, source_text in window_candidates: + distance = abs(int(message["idx"]) - int(window["targetIndex"])) + candidate = (distance, window_index, message, source_text) + (targets if distance == 0 else contexts).append(candidate) + + if batch: + data = { + "windows": windows, + "outputBytes": 0, + "outputTruncated": False, + "omittedMessages": 0, + } + else: + data = windows[0] + data["outputBytes"] = 0 + + def place_all_targets(max_chars: int) -> bool: + for window_index, (_window, window_candidates) in enumerate(raw_windows): + windows[window_index]["messages"] = [] + windows[window_index]["omittedMessages"] = len(window_candidates) + for _distance, window_index, message, source_text in targets: + candidate = dict(message) + candidate["text"], candidate["outputTruncated"] = _truncate_window_text( + source_text, max_chars + ) + windows[window_index]["messages"].append(candidate) + windows[window_index]["omittedMessages"] -= 1 + _refresh_window_budget_metadata(data, windows) + return data["outputBytes"] <= _READ_WINDOW_OUTPUT_BYTES + + # Find one common cap so no earlier window can consume another target's + # share of the budget. Short targets remain complete; long targets are fair. + low = 0 + high = _READ_WINDOW_MESSAGE_CHARS + target_cap = None + while low <= high: + middle = (low + high) // 2 + if place_all_targets(middle): + target_cap = middle + low = middle + 1 + else: + high = middle - 1 + if target_cap is None: + raise ValueError("read-window metadata exceeds the 20000-byte output budget") + place_all_targets(target_cap) + + for _distance, window_index, message, _source_text in sorted( + contexts, + key=lambda item: (item[0], item[1], item[2]["idx"]), + ): + window = windows[window_index] + window["messages"].append(message) + window["omittedMessages"] -= 1 + _refresh_window_budget_metadata(data, windows) + if data["outputBytes"] > _READ_WINDOW_OUTPUT_BYTES: + window["messages"].pop() + window["omittedMessages"] += 1 + _refresh_window_budget_metadata(data, windows) + # A later context item may have a shorter projection and still fit. + continue + + for window in windows: + window["messages"].sort(key=lambda message: message["idx"]) + _refresh_window_budget_metadata(data, windows) + return data + + +def read_window_data(session_id: str, idx: int, radius: int = 2) -> dict: + """Return bounded DB-backed message context around a message index.""" + raw = _read_window_unbudgeted(session_id, idx, radius) + return _budget_read_windows([raw], batch=False) def read_windows_data(requests: list) -> dict: - """Return multiple read-window results in one call.""" + """Return up to five message windows under one shared output budget.""" + requests = requests or [] + if len(requests) > _READ_WINDOW_MAX_BATCH: + raise ValueError("--batch accepts at most 5 items") windows = [] - for i, request in enumerate(requests or []): + for i, request in enumerate(requests): session = request.get("session") or request.get("sessionId") if not session: raise ValueError(f"Batch item {i} missing session") if "idx" not in request: raise ValueError(f"Batch item {i} missing idx") radius = request.get("radius", 2) - windows.append(read_window_data(session, int(request["idx"]), int(radius))) - return {"windows": windows} + windows.append(_read_window_unbudgeted(session, int(request["idx"]), radius)) + return _budget_read_windows(windows, batch=True) def _brief_message(msg: dict, title: str, max_chars: int = 220) -> dict: @@ -1113,52 +1264,59 @@ def evidence_audit_data(args, kind: str = "all") -> dict: return summary -def _human_window_text(text: str, max_chars: int = 1200) -> str: - value = (text or "").strip() - if len(value) <= max_chars: - return value - marker = f"\n[… truncated from {len(value)} chars]" - return value[:max(max_chars - len(marker), 0)] + marker[:max_chars] - - -def _print_human_window(window: dict): - print(f"# {window['title']}") - print(f"# {window['project']} | target idx:{window['targetIndex']} radius:{window['radius']}\n") - for msg in window["messages"]: - print(f"--- {msg.get('role', '').upper()} idx:{msg.get('idx')} {msg.get('ts', '')[:16]} ---") - print(_human_window_text(msg.get("text") or "")) - print() +def _human_windows_output(data: dict) -> str: + windows = data.get("windows") if "windows" in data else [data] + parts = [] + for window in windows: + parts.append(f"# {window['title']}\n") + parts.append( + f"# {window['project']} | target idx:{window['targetIndex']} " + f"radius:{window['radius']}\n\n" + ) + for message in window["messages"]: + parts.append( + f"--- {message.get('role', '').upper()} idx:{message.get('idx')} " + f"{message.get('ts', '')[:16]} ---\n" + ) + parts.append((message.get("text") or "") + "\n\n") + body = "".join(parts) + output_bytes = 0 + for _ in range(8): + summary = ( + f"# outputBytes={output_bytes} " + f"outputTruncated={str(bool(data['outputTruncated'])).lower()} " + f"omittedMessages={data['omittedMessages']}\n" + ) + rendered = body + summary + size = len(rendered.encode("utf-8")) + if size == output_bytes: + if size > _READ_WINDOW_OUTPUT_BYTES: + raise ValueError("read-window human output exceeds the 20000-byte output budget") + return rendered + output_bytes = size + raise ValueError("Could not calculate read-window output size") def cmd_read_window(args): if getattr(args, "batch", ""): - try: - requests = json.loads(args.batch) - if not isinstance(requests, list): - raise ValueError("--batch must be a JSON list") - data = read_windows_data(requests) - except (json.JSONDecodeError, TypeError, ValueError, KeyError) as exc: - print(str(exc)) - return + requests = json.loads(args.batch) + if not isinstance(requests, list): + raise ValueError("--batch must be a JSON list") + data = read_windows_data(requests) if args.json: - print(json.dumps(data, ensure_ascii=False, indent=2)) + print(_window_json(data), end="") return - for window in data["windows"]: - _print_human_window(window) + print(_human_windows_output(data), end="") return if not args.session or args.idx is None: print("read-window requires SESSION and --idx, or --batch JSON") return - try: - data = read_window_data(args.session, args.idx, args.radius) - except KeyError as exc: - print(str(exc)) - return + data = read_window_data(args.session, args.idx, args.radius) if args.json: - print(json.dumps(data, ensure_ascii=False, indent=2)) + print(_window_json(data), end="") return - _print_human_window(data) + print(_human_windows_output(data), end="") def cmd_find_repeats(args): diff --git a/chatview/commands/tool_calls.py b/chatview/commands/tool_calls.py index a9bf5a5..a612845 100644 --- a/chatview/commands/tool_calls.py +++ b/chatview/commands/tool_calls.py @@ -7,12 +7,31 @@ from chatview.snippets import make_query_snippet from chatview.tool_events import ( - find_tool_result, read_jsonl_record, tool_call_from_record, + tool_result_from_record, ) +def _bounded_projection(text: str, max_chars: int, around: str = "") -> tuple[str, int]: + if not around: + return text[:max_chars], 0 + if len(around) > max_chars: + raise ValueError("Anchor is longer than the requested --max-chars budget") + position = text.find(around) + if position < 0: + raise ValueError(f"Anchor not found in tool input: {around}") + anchor_end = min(position + len(around), len(text)) + start = max(position - max((max_chars - len(around)) // 2, 0), 0) + end = min(start + max_chars, len(text)) + if anchor_end > end: + end = anchor_end + start = max(end - max_chars, 0) + if end - start < max_chars: + start = max(end - max_chars, 0) + return text[start:end], start + + def tool_search_data(query: str, args) -> list: from chatview import db as _db @@ -22,7 +41,7 @@ def tool_search_data(query: str, args) -> list: source=getattr(args, "source", "all"), project=getattr(args, "project", ""), date_range=getattr(args, "date", "all"), - limit=max(getattr(args, "limit", 20), 1), + limit=min(max(getattr(args, "limit", 20), 1), 20), page=max(getattr(args, "page", 1), 1), ) results = [] @@ -30,7 +49,7 @@ def tool_search_data(query: str, args) -> list: snippet = make_query_snippet( row.get("input_text", ""), query=query, - max_chars=getattr(args, "max_chars", 500), + max_chars=min(max(getattr(args, "max_chars", 500), 1), 500), ) results.append( { @@ -97,6 +116,7 @@ def read_tool_event_data( event_idx: int, max_chars: int = 2000, include_result: bool = False, + around: str = "", ) -> dict: from chatview import db as _db @@ -112,6 +132,7 @@ def read_tool_event_data( filepath, int(locator.get("byte_offset", -1)), int(locator.get("byte_length", 0)), + locator.get("call_record_digest"), ) event = tool_call_from_record( raw_record, @@ -124,7 +145,8 @@ def read_tool_event_data( full_input = event.pop("input", "") max_chars = min(max(int(max_chars), 1), 2000) - event["input"] = full_input[:max_chars] + input_projection, input_start = _bounded_projection(full_input, max_chars, around) + event["input"] = input_projection event.update( { "sessionId": locator["session_id"], @@ -138,24 +160,42 @@ def read_tool_event_data( "source": locator.get("source", ""), "filePath": filepath, "originalChars": len(full_input), - "outputTruncated": len(full_input) > max_chars, + "outputTruncated": len(input_projection) < len(full_input), + "inputStartChar": input_start, + "inputEndChar": input_start + len(input_projection), + "around": around, } ) if include_result: - result_text = find_tool_result( - filepath, - int(locator.get("byte_offset", 0)) + int(locator.get("byte_length", 0)), - locator.get("source", ""), - event.get("callId", ""), - ) - if result_text is None: + result_offset = int(locator.get("result_byte_offset", -1)) + result_length = int(locator.get("result_byte_length", 0)) + if result_offset < 0 or result_length <= 0: event["result"] = None event["resultOriginalChars"] = 0 event["resultTruncated"] = False + event["resultStatus"] = "not_indexed" + event["resultMessage"] = ( + "Tool result locator is not indexed; run `distill refresh` " + "if the source session is complete" + ) else: + result_record = read_jsonl_record( + filepath, + result_offset, + result_length, + locator.get("result_record_digest"), + ) + result_text = tool_result_from_record( + result_record, + locator.get("source", ""), + int(locator.get("result_block_index", 0)), + event.get("callId", ""), + ) event["result"] = result_text[:max_chars] event["resultOriginalChars"] = len(result_text) event["resultTruncated"] = len(result_text) > max_chars + event["resultStatus"] = "found" + event["resultMessage"] = "Read on demand from the indexed source locator" return event @@ -166,6 +206,7 @@ def cmd_read_tool_event(args) -> None: args.event_idx, max_chars=getattr(args, "max_chars", 2000), include_result=getattr(args, "include_result", False), + around=getattr(args, "around", ""), ) except (KeyError, FileNotFoundError, RuntimeError, ValueError) as exc: raise SystemExit(str(exc)) from exc @@ -185,6 +226,10 @@ def cmd_read_tool_event(args) -> None: print("\n[TRUNCATED — increase --max-chars up to 2000 only if needed]") if "result" in result: print(f"\n--- TOOL RESULT ({result['resultOriginalChars']} chars) ---") - print(result["result"] if result["result"] is not None else "[not found]") + print( + result["result"] + if result["result"] is not None + else f"[{result['resultStatus']}: {result['resultMessage']}]" + ) if result["resultTruncated"]: print("\n[TRUNCATED — tool results are never stored in the index]") diff --git a/chatview/db/core.py b/chatview/db/core.py index 0333a5b..7c09c02 100644 --- a/chatview/db/core.py +++ b/chatview/db/core.py @@ -1,11 +1,14 @@ """Database connection, schema initialization, and migrations.""" +import os import re import sqlite3 import threading from pathlib import Path from urllib.parse import quote +from chatview.utils.constants import TOOL_RESULT_ERROR_CLASSES + # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- @@ -15,6 +18,24 @@ _local = threading.local() +def _secure_cache_paths(*, precreate_db: bool = False) -> None: + """Keep cached session material private even under a permissive umask.""" + CACHE_DIR.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(CACHE_DIR, 0o700) + if precreate_db and not DB_PATH.exists(): + try: + fd = os.open(DB_PATH, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) + except FileExistsError: + pass + else: + os.close(fd) + for path in (DB_PATH, Path(f"{DB_PATH}-wal"), Path(f"{DB_PATH}-shm")): + try: + os.chmod(path, 0o600) + except FileNotFoundError: + continue + + # --------------------------------------------------------------------------- # Connection # --------------------------------------------------------------------------- @@ -22,12 +43,14 @@ def get_conn() -> sqlite3.Connection: """Return a thread-local sqlite3.Connection with WAL mode and Row factory.""" conn = getattr(_local, "conn", None) if conn is None: - CACHE_DIR.mkdir(parents=True, exist_ok=True) + _secure_cache_paths(precreate_db=True) conn = sqlite3.connect(str(DB_PATH), check_same_thread=False) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("PRAGMA secure_delete=ON") conn.execute("PRAGMA busy_timeout=30000") + _secure_cache_paths() _local.conn = conn _local.readonly = False return conn @@ -97,6 +120,8 @@ def _validate_search_schema(conn: sqlite3.Connection) -> None: "id", "session_id", "event_idx", "line_number", "byte_offset", "byte_length", "block_index", "ts", "tool_name", "raw_name", "call_id", "input_text", "original_bytes", "input_truncated", + "result_line_number", "result_byte_offset", "result_byte_length", + "result_block_index", "call_record_digest", "result_record_digest", }, "tool_calls_fts": {"input_text"}, } @@ -251,6 +276,12 @@ def query_in_chunks( "input_text", "original_bytes", "input_truncated", + "result_line_number", + "result_byte_offset", + "result_byte_length", + "result_block_index", + "call_record_digest", + "result_record_digest", # evidence_events "run_id", "session_id", @@ -335,6 +366,7 @@ def query_in_chunks( "TEXT DEFAULT ''", "TEXT NOT NULL DEFAULT ''", "INTEGER DEFAULT 0", + "INTEGER DEFAULT -1", "REAL DEFAULT 0.0", } ) @@ -440,6 +472,12 @@ def init_db(): input_text TEXT NOT NULL, original_bytes INTEGER NOT NULL DEFAULT 0, input_truncated INTEGER NOT NULL DEFAULT 0, + result_line_number INTEGER NOT NULL DEFAULT 0, + result_byte_offset INTEGER NOT NULL DEFAULT -1, + result_byte_length INTEGER NOT NULL DEFAULT 0, + result_block_index INTEGER NOT NULL DEFAULT 0, + call_record_digest BLOB, + result_record_digest BLOB, FOREIGN KEY (session_id) REFERENCES sessions(id), UNIQUE(session_id, event_idx) ); @@ -749,6 +787,12 @@ def init_db(): created_at TEXT NOT NULL ); """) + _ensure_column(conn, "tool_calls", "result_line_number", "INTEGER DEFAULT 0") + _ensure_column(conn, "tool_calls", "result_byte_offset", "INTEGER DEFAULT -1") + _ensure_column(conn, "tool_calls", "result_byte_length", "INTEGER DEFAULT 0") + _ensure_column(conn, "tool_calls", "result_block_index", "INTEGER DEFAULT 0") + _ensure_column(conn, "tool_calls", "call_record_digest", "BLOB") + _ensure_column(conn, "tool_calls", "result_record_digest", "BLOB") _ensure_column(conn, "evidence_events", "run_id", "TEXT") _ensure_column(conn, "judgment_cards", "run_id", "TEXT") _ensure_column(conn, "cognitive_traits", "run_id", "TEXT") @@ -799,6 +843,15 @@ def init_db(): _ensure_column(conn, "evolve_runs", "completed_at", "TEXT") _ensure_sessions_fts(conn) _ensure_messages_trigram_fts(conn) + # Enforce the fixed-class invariant independently of source parsing. This + # runs with secure_delete enabled by get_conn(); refresh performs the final + # WAL truncate checkpoint after all index writes complete. + conn.execute("PRAGMA secure_delete=ON") + placeholders = ",".join("?" for _ in TOOL_RESULT_ERROR_CLASSES) + conn.execute( + f"DELETE FROM insight_errors WHERE error_key IS NULL OR error_key NOT IN ({placeholders})", + TOOL_RESULT_ERROR_CLASSES, + ) conn.commit() try: from . import evolve as _evolve diff --git a/chatview/db/tool_calls.py b/chatview/db/tool_calls.py index 62bacb6..38f491c 100644 --- a/chatview/db/tool_calls.py +++ b/chatview/db/tool_calls.py @@ -30,8 +30,10 @@ def replace_tool_calls(session_id: str, calls: list) -> None: """INSERT INTO tool_calls (session_id, event_idx, line_number, byte_offset, byte_length, block_index, ts, tool_name, raw_name, call_id, input_text, - original_bytes, input_truncated) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""", + original_bytes, input_truncated, result_line_number, + result_byte_offset, result_byte_length, result_block_index, + call_record_digest, result_record_digest) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", [ ( session_id, @@ -47,6 +49,12 @@ def replace_tool_calls(session_id: str, calls: list) -> None: call.get("input_text", ""), call.get("original_bytes", 0), int(bool(call.get("input_truncated", False))), + call.get("result_line_number", 0), + call.get("result_byte_offset", -1), + call.get("result_byte_length", 0), + call.get("result_block_index", 0), + call.get("call_record_digest", b""), + call.get("result_record_digest", b""), ) for index, call in enumerate(calls) ], @@ -115,15 +123,37 @@ def search_tool_calls( filters.append("COALESCE(NULLIF(tc.ts, ''), s.date, '') >= ?") params.append(min_date) filter_sql = (" AND " + " AND ".join(filters)) if filters else "" - page_size = max(int(limit), 1) + page_size = min(max(int(limit), 1), 20) offset = (max(int(page), 1) - 1) * page_size columns = """ tc.id, tc.session_id, tc.event_idx, tc.line_number, tc.byte_offset, tc.byte_length, tc.block_index, tc.ts, tc.tool_name, tc.raw_name, tc.call_id, tc.input_text, tc.original_bytes, tc.input_truncated, + tc.call_record_digest, tc.result_record_digest, s.title, s.project_name, s.project_key, s.project_display, s.source, s.file_path """ + if query: + exact_sql = f""" + SELECT {columns} + FROM tool_calls tc + JOIN sessions s ON tc.session_id=s.id + WHERE tc.call_id=?{filter_sql} + ORDER BY COALESCE(NULLIF(tc.ts, ''), s.date, '') DESC, + tc.session_id, tc.event_idx, tc.id + LIMIT ? OFFSET ? + """ + exact_rows = conn.execute( + exact_sql, [query, *params, page_size, offset] + ).fetchall() + exact_exists = exact_rows or conn.execute( + f"""SELECT 1 FROM tool_calls tc + JOIN sessions s ON tc.session_id=s.id + WHERE tc.call_id=?{filter_sql} LIMIT 1""", + [query, *params], + ).fetchone() + if exact_exists: + return [dict(row) for row in exact_rows] safe_query = _sanitize_fts_query(query) if safe_query: sql = f""" diff --git a/chatview/index.py b/chatview/index.py index 7d5e7e9..fd53855 100644 --- a/chatview/index.py +++ b/chatview/index.py @@ -19,7 +19,7 @@ PROJECTS_DIR = Path.home() / ".claude" / "projects" CACHE_DIR = Path(__file__).resolve().parent.parent / ".cache" INDEX_CACHE = CACHE_DIR / "index.json" -INDEX_SCHEMA_VERSION = f"{PROJECT_IDENTITY_VERSION}:tool-calls-v1" +INDEX_SCHEMA_VERSION = f"{PROJECT_IDENTITY_VERSION}:tool-calls-v4-private-result-errors" MAX_SEARCH_WORKERS = 8 # Codex CLI paths @@ -46,6 +46,20 @@ INDEX_STALE_CHECK_INTERVAL = float(os.environ.get("INDEX_STALE_CHECK_INTERVAL", "10")) +def _secure_index_cache(*, precreate_file: bool = False) -> None: + CACHE_DIR.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(CACHE_DIR, 0o700) + if precreate_file and not INDEX_CACHE.exists(): + try: + fd = os.open(INDEX_CACHE, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) + except FileExistsError: + pass + else: + os.close(fd) + if INDEX_CACHE.exists(): + os.chmod(INDEX_CACHE, 0o600) + + def _cached(key, compute_fn): """Return cached result if index hasn't changed, else compute and cache. @@ -172,18 +186,88 @@ def _post_process_db_refresh(_db, force: bool, changed: bool): _db.refresh_aggregates() +def _write_session_bundle(_db, meta: dict, file_mtime: float, store_insights) -> None: + """Atomically replace one session and all of its derived indexed state.""" + conn = _db.get_conn() + conn.execute("SAVEPOINT refresh_one_session") + try: + _db.upsert_session( + meta, + meta.get("userTexts", []), + meta.get("assistantSnippets", []), + ) + _db.replace_tool_calls(meta["id"], meta.get("_tool_calls", [])) + store_insights(meta) + _db.upsert_insight_state(meta["id"], file_mtime) + except Exception: + conn.execute("ROLLBACK TO SAVEPOINT refresh_one_session") + conn.execute("RELEASE SAVEPOINT refresh_one_session") + raise + conn.execute("RELEASE SAVEPOINT refresh_one_session") + + +def _db_session_coverage(_db) -> dict[str, dict]: + """Return compact coverage facts used to distrust a skinny JSON cache.""" + conn = _db.get_conn() + coverage = { + row["id"]: { + "user_count": 0, + "message_count": 0, + "tool_count": 0, + "insight_mtime": None, + } + for row in conn.execute("SELECT id FROM sessions").fetchall() + } + for row in conn.execute( + """SELECT session_id, count(*) AS n, + sum(CASE WHEN role='user' THEN 1 ELSE 0 END) AS user_n + FROM messages GROUP BY session_id""" + ).fetchall(): + if row["session_id"] in coverage: + coverage[row["session_id"]]["message_count"] = row["n"] + coverage[row["session_id"]]["user_count"] = row["user_n"] + for row in conn.execute( + "SELECT session_id, count(*) AS n FROM tool_calls GROUP BY session_id" + ).fetchall(): + if row["session_id"] in coverage: + coverage[row["session_id"]]["tool_count"] = row["n"] + for row in conn.execute( + "SELECT session_id, file_mtime FROM insight_state" + ).fetchall(): + if row["session_id"] in coverage: + coverage[row["session_id"]]["insight_mtime"] = row["file_mtime"] + return coverage + + +def _has_db_session_coverage(meta: dict, coverage: dict | None, mtime: float) -> bool: + if not coverage or coverage.get("insight_mtime") is None: + return False + if float(coverage["insight_mtime"] or 0) != float(mtime or 0): + return False + if coverage["user_count"] != int(meta.get("userMessageCount", 0) or 0): + return False + expected_messages = meta.get("messageCount") + if ( + expected_messages is not None + and coverage["message_count"] != int(expected_messages) + ): + return False + expected_tools = meta.get("toolCallCount") + return expected_tools is None or coverage["tool_count"] == int(expected_tools) + + def build_index(force: bool = False, known_files: dict = None) -> dict: """Scan all JSONL files and build/update the metadata index + SQLite DB. known_files: optional {path: mtime} dict pre-computed by the caller (e.g. from _session_source_mtimes()); reused to skip a redundant scan. - At most one build runs concurrently (_build_lock). A non-force waiter - that finds the index already up-to-date after acquiring the lock returns - the current index immediately without re-parsing. + At most one build runs concurrently (_build_lock). Cached metadata is reused + only after SQLite coverage is independently verified. """ global _index, _index_gen + _secure_index_cache() with _build_lock: return _build_index_locked(force=force, known_files=known_files) @@ -192,14 +276,6 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: """Inner implementation — must be called while _build_lock is held.""" global _index, _index_gen - # Non-force path: if a preceding build already captured all changes, return early. - if not force and known_files is None: - current = _session_source_mtimes() - with _index_lock: - indexed = dict(_index.get("_file_mtimes", {})) - if current == indexed and indexed: - return dict(_index) - from chatview.parsers.claude import extract_metadata, pretty_project_name from chatview.parsers.codex import ( _load_codex_titles, @@ -235,6 +311,12 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: cached_mtimes = cached.get("_file_mtimes", {}) cached_sessions = cached.get("sessions", {}) + cached_sid_by_path = { + meta.get("filePath"): sid + for sid, meta in cached_sessions.items() + if meta.get("filePath") + } + db_coverage = _db_session_coverage(_db) # Determine which files need (re)parsing. # Reuse known_files mtimes if supplied (avoids a second filesystem walk). @@ -248,7 +330,16 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: except OSError: continue current_files[fpath] = mtime - if fpath not in cached_mtimes or cached_mtimes[fpath] != mtime: + cached_sid = cached_sid_by_path.get(fpath) + if ( + fpath not in cached_mtimes + or cached_mtimes[fpath] != mtime + or not _has_db_session_coverage( + cached_sessions.get(cached_sid, {}), + db_coverage.get(cached_sid), + mtime, + ) + ): to_parse.append((fpath, proj_name)) print(f"Index: {len(jsonl_files)} files, {len(to_parse)} need parsing") @@ -273,17 +364,17 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: meta["source"] = "claude" enrich_project_identity(meta) meta["_mtime"] = current_files.get(fp, 0) - new_sessions[meta["id"]] = meta - _db.upsert_session( - meta, - meta.get("userTexts", []), - meta.get("assistantSnippets", []), + meta["toolCallCount"] = len(meta.get("_tool_calls", [])) + meta["messageCount"] = len(meta.get("userTexts", [])) + len( + meta.get("assistantSnippets", []) ) - _db.replace_tool_calls( - meta["id"], meta.get("_tool_calls", []) + _write_session_bundle( + _db, + meta, + current_files.get(fp, 0), + _store_session_insights, ) - _store_session_insights(meta) - _db.upsert_insight_state(meta["id"], current_files.get(fp, 0)) + new_sessions[meta["id"]] = meta bulk_n += 1 if bulk_n % 50 == 0: _db.bulk_commit() @@ -332,7 +423,16 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: except OSError: continue current_files[fpath] = mtime - if fpath not in cached_mtimes or cached_mtimes[fpath] != mtime: + cached_sid = cached_sid_by_path.get(fpath) + if ( + fpath not in cached_mtimes + or cached_mtimes[fpath] != mtime + or not _has_db_session_coverage( + cached_sessions.get(cached_sid, {}), + db_coverage.get(cached_sid), + mtime, + ) + ): codex_to_parse.append(fpath) print(f"Codex: {len(codex_files)} files, {len(codex_to_parse)} need parsing") @@ -355,17 +455,17 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: meta["project"] = "codex" enrich_project_identity(meta) meta["_mtime"] = current_files.get(fp, 0) - codex_new[meta["id"]] = meta - _db.upsert_session( - meta, - meta.get("userTexts", []), - meta.get("assistantSnippets", []), + meta["toolCallCount"] = len(meta.get("_tool_calls", [])) + meta["messageCount"] = len(meta.get("userTexts", [])) + len( + meta.get("assistantSnippets", []) ) - _db.replace_tool_calls( - meta["id"], meta.get("_tool_calls", []) + _write_session_bundle( + _db, + meta, + current_files.get(fp, 0), + _store_session_insights, ) - _store_session_insights(meta) - _db.upsert_insight_state(meta["id"], current_files.get(fp, 0)) + codex_new[meta["id"]] = meta bulk_n += 1 if bulk_n % 50 == 0: _db.bulk_commit() @@ -419,33 +519,6 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: if pruned_count: print(f"DB prune: {pruned_count} stale sessions") - # Backfill DB from cached sessions (only if DB is missing entries) - db_count = _db.get_conn().execute("SELECT count(*) FROM sessions").fetchone()[0] - if db_count < len(sessions): - _db.begin_bulk() - backfill_count = 0 - try: - for sid, meta in sessions.items(): - exists = ( - _db.get_conn() - .execute("SELECT 1 FROM sessions WHERE id=?", (sid,)) - .fetchone() - ) - if not exists: - _db.upsert_session( - meta, meta.get("userTexts", []), meta.get("assistantSnippets", []) - ) - _db.replace_tool_calls( - meta["id"], meta.get("_tool_calls", []) - ) - backfill_count += 1 - if backfill_count % 50 == 0: - _db.bulk_commit() - finally: - _db.end_bulk() - if backfill_count: - print(f"DB backfill: {backfill_count} sessions") - # Backfill insight tables for sessions not yet tracked in insight_state. # Uses precise per-session state (replaces the fragile 50% heuristic). session_mtimes = {sid: meta.get("_mtime", 0) for sid, meta in sessions.items()} @@ -477,9 +550,16 @@ def _parse_for_backfill(args): ) if fresh: fresh["id"] = sid + fresh["source"] = source + fresh["project"] = meta.get("project", "") fresh["projectName"] = meta.get("projectName", "") fresh["date"] = meta.get("date", "") fresh["_mtime"] = meta.get("_mtime", 0) + fresh["toolCallCount"] = len(fresh.get("_tool_calls", [])) + fresh["messageCount"] = len(fresh.get("userTexts", [])) + len( + fresh.get("assistantSnippets", []) + ) + enrich_project_identity(fresh) return fresh futures = {pool.submit(_parse_for_backfill, item): item for item in to_backfill} @@ -497,8 +577,12 @@ def _parse_for_backfill(args): try: for fresh in parsed_results: try: - _store_session_insights(fresh) - _db.upsert_insight_state(fresh["id"], fresh.get("_mtime", 0)) + _write_session_bundle( + _db, + fresh, + fresh.get("_mtime", 0), + _store_session_insights, + ) backfill_n += 1 if backfill_n % 50 == 0: _db.bulk_commit() @@ -513,6 +597,12 @@ def _parse_for_backfill(args): # Strip message bodies and non-serializable insight data before cache write. # Messages live in the SQLite DB (messages table); no need to duplicate in JSON. for sid, meta in sessions.items(): + if "_tool_calls" in meta: + meta["toolCallCount"] = len(meta["_tool_calls"]) + if "userTexts" in meta or "assistantSnippets" in meta: + meta["messageCount"] = len(meta.get("userTexts", [])) + len( + meta.get("assistantSnippets", []) + ) for k in ( "userTexts", "assistantSnippets", @@ -525,11 +615,12 @@ def _parse_for_backfill(args): meta.pop(k, None) # Save to cache - CACHE_DIR.mkdir(parents=True, exist_ok=True) + _secure_index_cache(precreate_file=True) cache_written = False try: with open(INDEX_CACHE, "w") as f: json.dump(index, f, ensure_ascii=False) + os.chmod(INDEX_CACHE, 0o600) cache_written = True except Exception as e: print(f"Cache write error: {e}") @@ -566,4 +657,13 @@ def _parse_for_backfill(args): if cache_written: _db.mark_project_identity_backfill_complete() + # A prior extractor could have persisted free-form tool-result errors. The + # replacement rows are safe, but old WAL frames still retain their bytes + # until an explicit truncate checkpoint completes. + conn = _db.get_conn() + conn.commit() + checkpoint = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + if checkpoint and checkpoint[0]: + raise RuntimeError("Could not truncate the SQLite WAL after refresh") + return index diff --git a/chatview/parsers/claude.py b/chatview/parsers/claude.py index b5cfff0..24a08d7 100644 --- a/chatview/parsers/claude.py +++ b/chatview/parsers/claude.py @@ -8,14 +8,36 @@ import re from pathlib import Path -from chatview.utils.constants import MAX_TOOL_RESULT_LEN -from chatview.utils.text import normalize_error as _normalize_error -from chatview.tool_events import make_tool_call_record +from chatview.utils.constants import MAX_TOOL_RESULT_LEN, TOOL_RESULT_ERROR_CLASSES +from chatview.tool_events import digest_jsonl_record, make_tool_call_record # --------------------------------------------------------------------------- # Constants (mirrored from server.py config) # --------------------------------------------------------------------------- +_TOOL_RESULT_ERROR_CLASS_BY_FOLD = { + name.casefold(): name for name in TOOL_RESULT_ERROR_CLASSES +} +_TOOL_RESULT_ERROR_CLASS_RE = re.compile( + r"(? list[str]: + """Return only fixed allowlisted classes; never retain result message text.""" + if not isinstance(text, str): + return [] + return list( + dict.fromkeys( + _TOOL_RESULT_ERROR_CLASS_BY_FOLD[match.group(0).casefold()] + for match in _TOOL_RESULT_ERROR_CLASS_RE.finditer(text) + ) + ) + + def pretty_project_name(dirname: str) -> str: """Convert encoded dir name like '-Users-foo-Desktop-proj-bar' to readable name.""" home_encoded = str(Path.home()).replace("/", "-").lstrip("-") @@ -45,22 +67,14 @@ def extract_metadata(filepath: str): user_texts = [] # (message_index, text, timestamp) assistant_snippets = [] # (message_index, assistant text for search) tool_calls = [] + tool_calls_by_id = {} msg_index = 0 # Insight extraction accumulators _tool_daily = {} # (day, tool_name) -> count _file_refs = {} # file_path -> count - _error_list = [] # [(normalized_error, day)] + _error_list = [] # [(fixed_allowlisted_error_class, day)] _snippet_list = [] # [(lang, code, context, applied)] _code_re = re.compile(r"```(\w*)\n([\s\S]*?)```") - _err_re = re.compile( - r"((?:Traceback.*?:\s*)?" - r"(?:(?:Error|Exception|TypeError|ValueError|KeyError|AttributeError|" - r"ImportError|ModuleNotFoundError|NameError|IndexError|RuntimeError|" - r"SyntaxError|FileNotFoundError|PermissionError|OSError|IOError|" - r"ConnectionError|TimeoutError)" - r"[:\s].{0,120}))", - re.IGNORECASE, - ) _prev_user_msg = "" try: @@ -143,20 +157,22 @@ def extract_metadata(filepath: str): ): # Insight: tool usage + file refs tool_name = blk.get("name", "unknown") - tool_calls.append( - make_tool_call_record( - event_idx=len(tool_calls), - line_number=line_number, - byte_offset=byte_offset, - byte_length=byte_length, - block_index=block_index, - ts=ts, - tool_name=tool_name, - raw_name=tool_name, - call_id=blk.get("id", ""), - tool_input=blk.get("input", {}), - ) + call = make_tool_call_record( + event_idx=len(tool_calls), + line_number=line_number, + byte_offset=byte_offset, + byte_length=byte_length, + block_index=block_index, + ts=ts, + tool_name=tool_name, + raw_name=tool_name, + call_id=blk.get("id", ""), + tool_input=blk.get("input", {}), + call_record_digest=digest_jsonl_record(raw_line), ) + tool_calls.append(call) + if call["call_id"]: + tool_calls_by_id[call["call_id"]] = call day = (first_ts or "")[:10] if day: key = (day, tool_name) @@ -196,23 +212,40 @@ def extract_metadata(filepath: str): msg_index += 1 elif msg_type == "user" and obj.get("toolUseResult"): - # Insight: extract errors from tool results + # Insight: retain only fixed error classes, never result text. content = obj.get("message", {}).get("content", []) if isinstance(content, list): - for blk in content: + for block_index, blk in enumerate(content): if ( isinstance(blk, dict) and blk.get("type") == "tool_result" ): + result_call = tool_calls_by_id.get( + blk.get("tool_use_id", "") + ) + if result_call is not None: + result_call.update( + { + "result_line_number": line_number, + "result_byte_offset": byte_offset, + "result_byte_length": byte_length, + "result_block_index": block_index, + "result_record_digest": digest_jsonl_record( + raw_line + ), + } + ) result_text = blk.get("content", "") if isinstance(result_text, list): result_text = json.dumps(result_text) if isinstance(result_text, str): day = (first_ts or "")[:10] - for m in _err_re.finditer(result_text[:5000]): - norm = _normalize_error(m.group(1)) - if len(norm) >= 10: - _error_list.append((norm, day)) + _error_list.extend( + (error_class, day) + for error_class in _tool_result_error_classes( + result_text + ) + ) msg_index += 1 except Exception: diff --git a/chatview/parsers/codex.py b/chatview/parsers/codex.py index bd5b401..ef3ce91 100644 --- a/chatview/parsers/codex.py +++ b/chatview/parsers/codex.py @@ -8,9 +8,16 @@ import re from pathlib import Path -from chatview.parsers.claude import _truncate_tool_output, _strip_tags -from chatview.utils.text import normalize_error as _normalize_error -from chatview.tool_events import CODEX_TOOL_NAMES, make_tool_call_record +from chatview.parsers.claude import ( + _strip_tags, + _tool_result_error_classes, + _truncate_tool_output, +) +from chatview.tool_events import ( + CODEX_TOOL_NAMES, + digest_jsonl_record, + make_tool_call_record, +) CODEX_DIR = Path.home() / ".codex" CODEX_SESSIONS_DIR = CODEX_DIR / "sessions" @@ -123,20 +130,12 @@ def extract_codex_metadata(filepath: str): user_texts = [] assistant_snippets = [] tool_calls = [] + tool_calls_by_id = {} msg_index = 0 # Insight extraction accumulators _tool_daily = {} _file_refs = {} _error_list = [] - _err_re = re.compile( - r"((?:Traceback.*?:\s*)?" - r"(?:(?:Error|Exception|TypeError|ValueError|KeyError|AttributeError|" - r"ImportError|ModuleNotFoundError|NameError|IndexError|RuntimeError|" - r"SyntaxError|FileNotFoundError|PermissionError|OSError|IOError|" - r"ConnectionError|TimeoutError)" - r"[:\s].{0,120}))", - re.IGNORECASE, - ) try: with open(filepath, "rb") as f: @@ -199,20 +198,22 @@ def extract_codex_metadata(filepath: str): if p_type == "function_call" else payload.get("input", "") ) - tool_calls.append( - make_tool_call_record( - event_idx=len(tool_calls), - line_number=line_number, - byte_offset=byte_offset, - byte_length=byte_length, - block_index=0, - ts=ts, - tool_name=tool_name, - raw_name=raw_name, - call_id=payload.get("call_id", ""), - tool_input=tool_input, - ) + call = make_tool_call_record( + event_idx=len(tool_calls), + line_number=line_number, + byte_offset=byte_offset, + byte_length=byte_length, + block_index=0, + ts=ts, + tool_name=tool_name, + raw_name=raw_name, + call_id=payload.get("call_id", ""), + tool_input=tool_input, + call_record_digest=digest_jsonl_record(raw_line), ) + tool_calls.append(call) + if call["call_id"]: + tool_calls_by_id[call["call_id"]] = call day = (first_ts or "")[:10] if day: key = (day, tool_name) @@ -247,14 +248,25 @@ def extract_codex_metadata(filepath: str): _file_refs[fp] = _file_refs.get(fp, 0) + 1 msg_index += 1 elif p_type in ("function_call_output", "custom_tool_call_output"): - # Insight: errors from tool output + result_call = tool_calls_by_id.get(payload.get("call_id", "")) + if result_call is not None: + result_call.update( + { + "result_line_number": line_number, + "result_byte_offset": byte_offset, + "result_byte_length": byte_length, + "result_block_index": 0, + "result_record_digest": digest_jsonl_record(raw_line), + } + ) + # Insight: retain only fixed error classes, never result text. output = payload.get("output", "") if isinstance(output, str): day = (first_ts or "")[:10] - for m in _err_re.finditer(output[:5000]): - norm = _normalize_error(m.group(1)) - if len(norm) >= 10: - _error_list.append((norm, day)) + _error_list.extend( + (error_class, day) + for error_class in _tool_result_error_classes(output) + ) msg_index += 1 except Exception: return None diff --git a/chatview/tool_events.py b/chatview/tool_events.py index 01dab09..281075b 100644 --- a/chatview/tool_events.py +++ b/chatview/tool_events.py @@ -2,10 +2,17 @@ from __future__ import annotations +import hashlib import json +import os +import re TOOL_INPUT_MAX_BYTES = 2000 +TOOL_ANCHOR_BYTES = 798 +TOOL_HEAD_BYTES = 600 +TOOL_TAIL_BYTES = 600 +RECORD_DIGEST_BYTES = 16 CODEX_TOOL_NAMES = { "shell": "Bash", @@ -39,6 +46,124 @@ def truncate_utf8(text: str, max_bytes: int = TOOL_INPUT_MAX_BYTES) -> tuple[str return clipped, len(raw), True +def digest_jsonl_record(raw: bytes) -> bytes: + """Return a compact digest for stale-locator detection without storing content.""" + return hashlib.blake2b(raw, digest_size=RECORD_DIGEST_BYTES).digest() + + +def _head_utf8(raw: bytes, max_bytes: int) -> str: + return raw[:max_bytes].decode("utf-8", errors="ignore") + + +def _tail_utf8(raw: bytes, max_bytes: int) -> str: + return raw[-max_bytes:].decode("utf-8", errors="ignore") + + +_PATH_RE = re.compile( + r"(?:(?:/|\.\.?/)[^\s\"'`<>{}\[\](),;]+|" + r"[A-Za-z0-9_.-]+\.(?:jsonl?|ya?ml|toml|md|txt|py|js|ts|tsx|jsx|sh|sql|csv))", + re.IGNORECASE, +) +_PATCH_HEADER_RE = re.compile( + r"^\*\*\*\s+(?:Add|Update|Delete|Move to)\s+File:\s*(.+)$", + re.MULTILINE, +) +_NESTED_TOOL_RE = re.compile(r"\btools\.([A-Za-z_][A-Za-z0-9_]*)\b") +_PATH_KEYS = frozenset( + {"file_path", "filepath", "path", "cwd", "workdir", "directory"} +) +_COMMAND_KEYS = frozenset({"cmd", "command", "script"}) + + +def _append_unique(items: list[str], seen: set[str], label: str, value) -> None: + text = stringify_tool_input(value).strip() + if not text: + return + item = f"{label}:{text}" + if item not in seen: + seen.add(item) + items.append(item) + + +def _structured_anchors(value, raw_text: str) -> list[str]: + """Extract high-value search anchors without retaining a second raw payload.""" + paths: list[str] = [] + patches: list[str] = [] + tools: list[str] = [] + commands: list[str] = [] + seen: set[str] = set() + + parsed = value + if isinstance(value, str): + try: + parsed = json.loads(value) + except (json.JSONDecodeError, TypeError): + parsed = value + + def walk(node, key: str = "") -> None: + normalized_key = key.casefold() + if normalized_key in _PATH_KEYS and isinstance(node, (str, int, float)): + path_text = str(node) + _append_unique(paths, seen, "path", path_text) + basename = os.path.basename(path_text.rstrip("/")) + if basename and basename != path_text: + _append_unique(paths, seen, "basename", basename) + elif normalized_key in _COMMAND_KEYS and isinstance(node, (str, list)): + command_text = stringify_tool_input(node) + if len(command_text) > 600: + command_text = command_text[:300] + " … " + command_text[-300:] + _append_unique(commands, seen, "command", command_text) + if isinstance(node, dict): + for child_key, child in node.items(): + walk(child, str(child_key)) + elif isinstance(node, list): + for child in node: + walk(child, key) + + walk(parsed) + for match in _PATH_RE.finditer(raw_text): + path_text = match.group(0).rstrip(".:)") + _append_unique(paths, seen, "path", path_text) + basename = os.path.basename(path_text.rstrip("/")) + if basename and basename != path_text: + _append_unique(paths, seen, "basename", basename) + for match in _PATCH_HEADER_RE.finditer(raw_text): + _append_unique(patches, seen, "patch", match.group(0).strip()) + for match in _NESTED_TOOL_RE.finditer(raw_text): + _append_unique(tools, seen, "nested-tool", f"tools.{match.group(1)}") + return paths + patches + tools + commands + + +def make_bounded_index_text(value, max_bytes: int = TOOL_INPUT_MAX_BYTES) -> tuple[str, int, bool]: + """Build a searchable bounded projection: anchors first, then raw head/tail.""" + max_bytes = min(max(int(max_bytes), 0), TOOL_INPUT_MAX_BYTES) + raw_text = stringify_tool_input(value) + raw_bytes = raw_text.encode("utf-8", errors="replace") + if len(raw_bytes) <= max_bytes: + return raw_text, len(raw_bytes), False + + # Fixed quotas prevent a long anchor list from consuming the raw head/tail. + # The anchor quota itself samples both ends so late unique paths survive. + anchor_raw = "\n".join(_structured_anchors(value, raw_text)).encode( + "utf-8", errors="replace" + ) + if len(anchor_raw) <= TOOL_ANCHOR_BYTES: + anchor_projection = anchor_raw.decode("utf-8", errors="ignore") + else: + anchor_head_bytes = (TOOL_ANCHOR_BYTES - 1) // 2 + anchor_tail_bytes = TOOL_ANCHOR_BYTES - anchor_head_bytes - 1 + anchor_projection = ( + _head_utf8(anchor_raw, anchor_head_bytes) + + "\n" + + _tail_utf8(anchor_raw, anchor_tail_bytes) + ) + head = _head_utf8(raw_bytes, TOOL_HEAD_BYTES) + tail = _tail_utf8(raw_bytes, TOOL_TAIL_BYTES) + projection = "\n".join((anchor_projection, head, tail)) + projection, _, _ = truncate_utf8(projection, max_bytes=max_bytes) + return projection, len(raw_bytes), True + + def make_tool_call_record( *, event_idx: int, @@ -51,10 +176,12 @@ def make_tool_call_record( raw_name: str, call_id: str, tool_input, + call_record_digest: bytes = b"", max_bytes: int = TOOL_INPUT_MAX_BYTES, ) -> dict: - text = stringify_tool_input(tool_input) - clipped, original_bytes, truncated = truncate_utf8(text, max_bytes=max_bytes) + clipped, original_bytes, truncated = make_bounded_index_text( + tool_input, max_bytes=max_bytes + ) return { "event_idx": event_idx, "line_number": line_number, @@ -68,10 +195,21 @@ def make_tool_call_record( "input_text": clipped, "original_bytes": original_bytes, "input_truncated": truncated, + "call_record_digest": call_record_digest, + "result_line_number": 0, + "result_byte_offset": -1, + "result_byte_length": 0, + "result_block_index": 0, + "result_record_digest": b"", } -def read_jsonl_record(filepath: str, byte_offset: int, byte_length: int) -> dict: +def read_jsonl_record( + filepath: str, + byte_offset: int, + byte_length: int, + expected_digest: bytes | None = None, +) -> dict: """Read exactly one indexed JSONL record and parse it locally.""" if byte_offset < 0 or byte_length <= 0: raise ValueError("Invalid JSONL byte locator") @@ -80,6 +218,8 @@ def read_jsonl_record(filepath: str, byte_offset: int, byte_length: int) -> dict raw = handle.read(byte_length) if len(raw) != byte_length: raise RuntimeError("Indexed JSONL locator is stale; run `distill refresh`") + if expected_digest and digest_jsonl_record(raw) != bytes(expected_digest): + raise RuntimeError("Indexed JSONL locator digest is stale; run `distill refresh`") try: return json.loads(raw.decode("utf-8", errors="replace")) except json.JSONDecodeError as exc: @@ -128,44 +268,31 @@ def tool_call_from_record(obj: dict, source: str, block_index: int = 0) -> dict: } -def find_tool_result( - filepath: str, - start_offset: int, - source: str, - call_id: str, -) -> str | None: - """Find a call's result after its indexed record without returning raw lines.""" - if not call_id: - return None - with open(filepath, "rb") as handle: - handle.seek(max(int(start_offset), 0)) - for raw in handle: - try: - obj = json.loads(raw.decode("utf-8", errors="replace")) - except json.JSONDecodeError: - continue - if source == "codex": - if obj.get("type") != "response_item": - continue - payload = obj.get("payload") or {} - if ( - payload.get("type") - in ("function_call_output", "custom_tool_call_output") - and payload.get("call_id", "") == call_id - ): - return stringify_tool_input(payload.get("output", "")) - continue - - if obj.get("type") != "user": - continue - content = (obj.get("message") or {}).get("content", []) - if not isinstance(content, list): - continue - for block in content: - if ( - isinstance(block, dict) - and block.get("type") == "tool_result" - and block.get("tool_use_id", "") == call_id - ): - return stringify_tool_input(block.get("content", "")) - return None +def tool_result_from_record( + obj: dict, source: str, block_index: int, call_id: str +) -> str: + """Project the one result selected by an indexed record/block locator.""" + if source == "codex": + if obj.get("type") != "response_item": + raise RuntimeError("Indexed tool result record is no longer a Codex result") + payload = obj.get("payload") or {} + if payload.get("type") not in ( + "function_call_output", + "custom_tool_call_output", + ) or payload.get("call_id", "") != call_id: + raise RuntimeError("Indexed tool result changed; run `distill refresh`") + return stringify_tool_input(payload.get("output", "")) + + if obj.get("type") != "user": + raise RuntimeError("Indexed tool result record is no longer a Claude result") + content = (obj.get("message") or {}).get("content", []) + if not isinstance(content, list) or not 0 <= block_index < len(content): + raise RuntimeError("Indexed Claude tool result block no longer exists") + block = content[block_index] + if ( + not isinstance(block, dict) + or block.get("type") != "tool_result" + or block.get("tool_use_id", "") != call_id + ): + raise RuntimeError("Indexed tool result changed; run `distill refresh`") + return stringify_tool_input(block.get("content", "")) diff --git a/chatview/utils/constants.py b/chatview/utils/constants.py index d284a7c..8fdeda1 100644 --- a/chatview/utils/constants.py +++ b/chatview/utils/constants.py @@ -5,3 +5,26 @@ # Thinking block truncation limit (characters) MAX_THINKING_LEN = 800 + +# Fixed classes that may be derived from tool-result bodies. Result messages and +# arbitrary suffixes must never be persisted in SQLite. +TOOL_RESULT_ERROR_CLASSES = ( + "ModuleNotFoundError", + "FileNotFoundError", + "PermissionError", + "ConnectionError", + "AttributeError", + "RuntimeError", + "TimeoutError", + "ImportError", + "IndexError", + "SyntaxError", + "TypeError", + "ValueError", + "KeyError", + "NameError", + "OSError", + "IOError", + "Exception", + "Error", +) diff --git a/skills/distill-yourself/SKILL.md b/skills/distill-yourself/SKILL.md index e7c425f..e5a3f7c 100644 --- a/skills/distill-yourself/SKILL.md +++ b/skills/distill-yourself/SKILL.md @@ -9,14 +9,14 @@ description: "本地 Claude Code/Codex/agent 对话历史的唯一检索入口 ## Core Rules -- 检索本地 Claude Code、Codex 或 agent 对话历史时，先用 `distill` 定位并核验证据，不先扫描原始 transcript。 +- 检索本地 Claude Code、Codex 或 agent 对话历史时，先用 `distill` 定位并核验证据，不扫描原始 transcript。 - 追查“哪个 Agent 用什么工具、命令或 patch 产生了某个文件”时，先用 `tool-search` 定位，再用 `read-tool-event` 定点核验；不要从消息搜索直接跳到原始 JSONL。 -- 工具索引只保存每条调用参数最多 2 KB 的搜索副本和原 JSONL 坐标，不保存工具结果。需要结果时给 `read-tool-event` 加 `--include-result`，由它从原 JSONL 定点读取并截断。 -- 只有当 `distill` 的消息窗口和工具事件读取都无法提供压缩边界或特殊旧格式细节时，才定向读取原始 session JSONL；说明原因，结构化投影必要字段并设置硬输出上限。 -- 不要对 session JSONL 运行会回显整行的 `rg -n`，也不要用 `head` 或 `sed` 把“若干行”当作小窗口。JSONL 单行可能包含数万 token；`rg -l` 只能用于定位文件，不能作为证据读取器。 +- 工具索引只保存每条调用参数最多 2 KB 的有界搜索投影（为 structured anchors、head 和 tail 设置 fixed quotas）、调用坐标和匹配结果的 result locator；精确 call ID 由 dedicated `call_id` column 匹配。不保存工具结果正文，result body is never stored in the SQLite search index。需要结果时给 `read-tool-event` 加 `--include-result`，由它根据 result locator 从原 JSONL 定点读取并截断。 +- 当 `distill` 的有界消息或工具读取器无法提供所需细节时，report the bounded-reader limitation, improve the locator or anchor, or stop；never direct-read raw rollout or session JSONL，即使面对压缩边界、旧格式或缺失 locator 也没有 raw fallback。 +- 不要扫描、打开或直接读取 session/rollout JSONL，也不要对它运行 `rg -n`、`rg -l`、`head`、`sed`、`jq`、Python 等 shell/content reader。JSONL 单行可能包含数万 token；`filePath` 和 byte locator 只供 `distill` 的有界读取器内部使用。 - 若任务还匹配其他工作流 skill，先完成历史取证，再把核验后的上下文交给后续流程。 - `distill` 只负责取数、检索和暂存；结论由你基于证据生成。 -- `profile-digest` / `aggregates` / `stats` 是地图，不是结论。写入 Memory/Profile/Twin 前必须用 `read-window`、`session-brief` 或原 session 内容核验。 +- `profile-digest` / `aggregates` / `stats` 是地图，不是结论。写入 Memory/Profile/Twin 前必须用 `read-window`、`session-brief` 或其他有界 `distill` reader 核验；不能把“核验原 session”解释为直接读取 session JSONL。 - 不把 assistant echo、IDE/file context、task notification、agent/subagent prompt、工具输出噪声当作主证据。 - 蒸馏长期偏好、Memory、Rules 或 Patterns 时，检索命令加 `--evidence-only`；普通历史定位不要加，以免隐藏诊断线索。 - Memory/Profile 的标准流程止于 `distill evolve-write`。告诉用户结果已暂存，并由用户在 UI 中预览、确认和同步；不要代替用户运行 `distill evolve-sync --execute`。 @@ -64,16 +64,19 @@ For exact tool provenance, use a separate bounded ladder. If the user already gi distill refresh distill tool-search "" --format jsonl --max-chars 500 --limit 20 --page 1 --date 90d distill read-tool-event --event-idx --max-chars 2000 +distill read-tool-event --event-idx --around "" --max-chars 2000 distill read-tool-event --event-idx --include-result --max-chars 2000 ``` -`tool-search` searches only bounded call inputs, never tool results. Its `filePath` is provenance, not permission to print that JSONL. `read-tool-event` seeks to the indexed byte range, projects the selected call, and caps model-visible output. If the result is requested, it follows the indexed `callId` locally and applies the same cap without persisting the result. +`tool-search` searches only bounded call-input projections, never tool results. The 2 KB projection assigns fixed quotas to a prioritized subset of structured anchors (paths, basenames, commands, patch headers, and nested tool names), head, and tail, so one category cannot consume the others' entire budget. It does not promise to retain every arbitrarily numerous anchor; a filename actually within the fixed tail remains searchable without indexing or printing the full input. Exact call IDs are matched separately against the dedicated `call_id` column rather than being described as input-projection anchors. Its `filePath` is provenance, not permission to print that JSONL. -Whenever you recommend `--include-result`, say explicitly that the result is read on demand from the source JSONL and is not stored in the SQLite search index. This distinction explains why the command is safe and why `tool-search` cannot search result bodies. +`read-tool-event` seeks to the indexed byte range, returns a default bounded head projection, and caps model-visible output. If that head omits a known fact, use `--around ""` to center the same bounded budget on that anchor; do not increase the budget. If the result is requested, it follows the indexed result locator rather than scanning to EOF, applies the same cap, and does not persist the body. -`--max-chars 2000` is a hard ceiling for tool-event reads, not a first step in progressive expansion. If the needed fact is outside the returned projection, report the bounded-reader limitation or refine the anchor; never propose 4000, 10000, or an unbounded raw read. +Whenever you recommend `--include-result`, say explicitly that the result body is read on demand from the source JSONL through an indexed result locator and that the result body is never stored in the SQLite search index. This distinction explains why the command is safe and why `tool-search` cannot search result bodies. -If `tool-search` misses, refresh once, retry with a shorter stable basename or command fragment, then use message search to identify a better anchor. Do not compensate by increasing `--limit`, globally grepping transcripts, or reading the beginning of a rollout file. +`--max-chars 2000` is a hard ceiling for tool-event reads, not a first step in progressive expansion. If the needed fact is outside the returned projection, report the bounded-reader limitation, improve the locator or anchor, or stop; never propose 4000, 10000, a raw fallback, or any direct rollout/session JSONL read. + +If `tool-search` misses, refresh once, retry with a shorter stable basename or command fragment, then use message search to identify a better anchor. Because the index reserves fixed quotas for structured anchors and head + tail, do not compensate by increasing `--limit`, walking later pages indefinitely, globally grepping transcripts, or reading the beginning of a rollout file. Choose the scope before ORIENT, and use ORIENT only for broad profile, trend, correction-pattern, or cognitive-model analysis. If the user asks about a recent/project-specific topic, carry that `--date` / `--project` into the orienting commands; otherwise use `--date all --source all`. @@ -121,8 +124,9 @@ Important commands: | `distill read-window --batch '[...]'` | Verify several windows at once | | `distill evidence-audit --json` | Estimate contamination from prompts/tasks/context noise | | `distill tool-search "" --format jsonl --limit 20 --page 1` | Bounded search over tool-call inputs with JSONL locators | -| `distill read-tool-event --event-idx N --max-chars 2000` | Read exactly one indexed call from its source JSONL | -| `distill read-tool-event --event-idx N --include-result --max-chars 2000` | Also read the matching result locally without indexing it | +| `distill read-tool-event --event-idx N --max-chars 2000` | Read one indexed call as a bounded head projection | +| `distill read-tool-event --event-idx N --around "" --max-chars 2000` | Center the bounded call projection on an already known anchor | +| `distill read-tool-event --event-idx N --include-result --max-chars 2000` | Read the matching result through its indexed locator without indexing its body | `corrections` JSON exposes a stable `idx` and its nearest user/assistant pair. `search` JSON/JSONL exposes `sessionId`, `idx`, and `messageIndex`; use `sessionId + idx` as the read coordinate. `idx` is only ordered within one session, not a global message id. `read-window --radius N` reads the inclusive numeric range `idx-N ... idx+N`, so gaps may produce fewer than `2N+1` messages. @@ -130,9 +134,11 @@ Important commands: distill read-window --batch '[{"sessionId":"session_id","idx":123,"radius":2}]' ``` +Every `read-window` rendering uses the same bounded serializer: human, `--json`, and any other structured format are all output formats under one budget. Enforce `radius <= 5`, `batch size <= 5`, and `total serialized output <= 20 KB`; a smaller remaining budget truncates later messages/windows and reports truncation metadata. These are workflow-wide safety limits, not hints: do not reconstruct oversized context with multiple batches, later pages, or raw JSONL, and do not retry another format to obtain omitted text. Narrow the selected locators or anchor instead. + For manual triage, prefer `--format lines`; for scripts or incremental aggregation, prefer `--format jsonl`. `--limit N` is the page size and `--page 2`, `--page 3`, ... retrieves later pages. Keep the standard page size at 20 and page instead of requesting one oversized result. Mirrored copies of the same timestamped message are folded before paging, while `duplicateCount` / `duplicateSessionIds` preserve provenance in structured output. -`-C/-B/-A` are optional inline previews for a human terminal, not the Agent verification path. Search without inline context, select a few locators, then use `read-window`. Human `read-window` output is bounded and marks truncation; `--json` returns the complete text stored in the local index, which may already be shorter than the original transcript because ingestion has its own limits. +`-C/-B/-A` are optional inline previews for a human terminal, not the Agent verification path. Search without inline context, select a few locators, then use `read-window`. Every `read-window` format is bounded and marks per-message and total-output truncation; structured output is not an escape hatch for retrieving complete indexed text. Stop retrieval when you have enough direct user evidence. More searching after 2-4 strong, verified quotes usually adds noise. diff --git a/skills/distill-yourself/evals/evals.json b/skills/distill-yourself/evals/evals.json index 67f995b..b1221b9 100644 --- a/skills/distill-yourself/evals/evals.json +++ b/skills/distill-yourself/evals/evals.json @@ -27,25 +27,49 @@ { "id": 3, "prompt": "我看到仓库根目录有一个 .verify-search-fix-result-v2.json。请只写出你会如何从历史对话追查是哪个 Agent、通过什么工具调用创建它的命令序列，不要真正执行。", - "expected_output": "直接用 tool-search 搜索精确文件名，选择少量 sessionId+eventIdx 后用 read-tool-event 定点核验；不使用 rg -n、head、sed 或整份 session read。", + "expected_output": "直接用 tool-search 搜索精确文件名，选择少量 sessionId+eventIdx 后用 read-tool-event 定点核验；有界读取不足时改进 locator/anchor 或停止，不使用 rg、head、sed、jq、Python 或任何 raw session/rollout JSONL fallback。", "files": [], "expectations": [ "Starts with tool-search on the exact filename using JSONL, limit 20, and page 1", "Uses sessionId plus eventIdx with read-tool-event and max-chars 2000", - "Does not propose rg -n, head, sed, or printing raw rollout JSONL" + "Reports a bounded-reader limitation, refines the locator or anchor, or stops instead of proposing any direct raw session or rollout JSONL read" ] }, { "id": 4, "prompt": "我已经通过工具调用定位到一次失败的测试，现在需要核对这个调用对应的工具结果。只给安全的取证命令和停止条件，不要执行。", - "expected_output": "调用已经定位，因此直接用 read-tool-event --include-result --max-chars 2000 读取对应结果；说明结果不入索引，2000 是硬上限，并在一个精确事件足够时停止。", + "expected_output": "调用已经定位，因此直接用 read-tool-event --include-result --max-chars 2000 读取对应结果；说明 result locator 会入库，但 result body 只从源 JSONL 按需读取且不进入 SQLite/FTS，2000 是硬上限，并在一个精确事件足够时停止。", "files": [], "expectations": [ "Uses read-tool-event with --include-result and max-chars 2000", - "States that tool results are read from source JSONL but are not stored in the index", + "States that the result locator is indexed while the result body is read from source JSONL and is not stored in SQLite or FTS", "Never raises max-chars above 2000 and never proposes raw JSONL shell reads", "Stops after the exact event establishes the needed fact" ] + }, + { + "id": 5, + "prompt": "一个很长的工具调用把目标文件名 only-in-tail-result.json 放在参数末尾，开头 2KB 没有它。我需要追查创建者。只写安全检索与核验 SOP，不要执行。", + "expected_output": "直接 tool-search 精确文件名，因为 2KB 工具搜索投影为 structured anchors、head、tail 保留固定配额，目标位于参数末尾时可由 tail 命中；选 1-3 个 sessionId+eventIdx 后 read-tool-event，并用 --around 精确文件名读取对应区域。若仍不足则改进 locator/anchor、报告限制或停止；不扩大 2000 字符预算，也不存在 raw session/rollout JSONL fallback。", + "files": [], + "expectations": [ + "Uses tool-search on the exact tail-only filename and explains that fixed projection quotas reserve a tail region despite the prefix being absent", + "Selects at most 1-3 sessionId plus eventIdx locators and uses read-tool-event with max-chars 2000", + "Uses --around with the known filename if the bounded call projection needs refinement", + "If bounded retrieval is insufficient, refines the locator or anchor, reports the limitation, or stops; never recommends any raw session or rollout JSONL fallback" + ] + }, + { + "id": 6, + "prompt": "search 找到了 14 个候选窗口，每条正文都可能非常大。我想用 read-window --json 一次拿全，再分几批补齐。请只给安全的核验方案和停止条件，不要执行。", + "expected_output": "拒绝一次拿全或分批重建；先筛到最多 5 个窗口，radius 不超过 5，单次 read-window batch，并说明 human/json 等所有格式共享 20KB 总序列化预算，截断后应缩小 locator/anchor 而非多批、翻页或读 raw JSONL。", + "files": [], + "expectations": [ + "Limits read-window radius to at most 5, batch size to at most 5, and total serialized output to at most 20KB", + "States that JSON, human, and all other read-window formats share the same output budget", + "Refuses to reconstruct omitted context through multiple batches, later pages, another format, or raw JSONL", + "Narrows candidate locators or anchors and stops once the selected evidence is sufficient" + ] } ] } diff --git a/tests/test_db.py b/tests/test_db.py index 79e96ec..51554d1 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -256,6 +256,405 @@ def test_init_db_adds_project_identity_columns_to_legacy_sessions(self): self.assertEqual(row["project_key"], "finance-QuantaAlpha-claw") self.assertEqual(row["project_display"], "finance-QuantaAlpha-claw") + def test_init_db_repairs_private_cache_and_sqlite_permissions(self): + from pathlib import Path + from chatview.db import core as _dbcore + + cache_dir = Path(self._tmpdir) + db_path = cache_dir / "sessions.db" + db_path.touch() + os.chmod(cache_dir, 0o777) + os.chmod(db_path, 0o666) + + db.init_db() + + self.assertEqual(cache_dir.stat().st_mode & 0o777, 0o700) + self.assertEqual(db_path.stat().st_mode & 0o777, 0o600) + for suffix in ("-wal", "-shm"): + sidecar = Path(f"{db_path}{suffix}") + if sidecar.exists(): + self.assertEqual(sidecar.stat().st_mode & 0o777, 0o600) + + def test_init_db_stays_private_under_umask_zero(self): + from pathlib import Path + + cache_dir = Path(self._tmpdir) + db_path = cache_dir / "sessions.db" + old_umask = os.umask(0) + try: + db.init_db() + finally: + os.umask(old_umask) + + self.assertEqual(cache_dir.stat().st_mode & 0o777, 0o700) + self.assertEqual(db_path.stat().st_mode & 0o777, 0o600) + for suffix in ("-wal", "-shm"): + sidecar = Path(f"{db_path}{suffix}") + if sidecar.exists(): + self.assertEqual(sidecar.stat().st_mode & 0o777, 0o600) + + def test_readonly_open_does_not_attempt_permission_writes(self): + from unittest.mock import patch + from chatview import db as _db + from chatview.db import core as _dbcore + + _db.init_db() + _dbcore._close_thread_connection() + with patch.object( + _dbcore.os, + "chmod", + side_effect=PermissionError("read-only filesystem"), + ): + self.assertTrue(_db.prepare_search_db()) + + def test_session_bundle_rolls_back_all_tables_on_derived_write_failure(self): + from chatview import db as _db + from chatview.index import _write_session_bundle + + old_meta = dict(_META) + old_meta["id"] = "atomic-session" + old_meta["title"] = "Before" + _db.init_db() + _db.upsert_session(old_meta, _USER_TEXTS[:1], []) + + new_meta = dict(old_meta) + new_meta["title"] = "After" + new_meta["userTexts"] = [ + {"idx": 0, "text": "replacement", "ts": "2026-06-02"} + ] + new_meta["assistantSnippets"] = [] + new_meta["_tool_calls"] = [ + { + "event_idx": 0, + "line_number": 1, + "byte_offset": 0, + "byte_length": 10, + "block_index": 0, + "tool_name": "Read", + "raw_name": "Read", + "call_id": "atomic-call", + "input_text": "replacement", + "original_bytes": 11, + "input_truncated": False, + } + ] + + _db.begin_bulk() + try: + def fail_after_partial_insight(meta): + _db.get_conn().execute( + """INSERT INTO insight_tool_usage + (session_id, day, tool_name, count) VALUES (?,?,?,?)""", + (meta["id"], "2026-06-02", "Read", 1), + ) + raise RuntimeError("derived failure") + + with self.assertRaisesRegex(RuntimeError, "derived failure"): + _write_session_bundle( + _db, + new_meta, + 2.0, + fail_after_partial_insight, + ) + finally: + _db.end_bulk() + + row = _db.get_session_meta("atomic-session") + self.assertEqual(row["title"], "Before") + messages = _db.get_session_messages("atomic-session") + self.assertEqual([message["text"] for message in messages], [_USER_TEXTS[0]["text"]]) + self.assertEqual( + _db.get_conn().execute( + "SELECT count(*) FROM tool_calls WHERE session_id='atomic-session'" + ).fetchone()[0], + 0, + ) + self.assertEqual( + _db.get_conn().execute( + "SELECT count(*) FROM insight_tool_usage WHERE session_id='atomic-session'" + ).fetchone()[0], + 0, + ) + + def test_skinny_cache_reparses_source_when_sqlite_coverage_is_missing(self): + from contextlib import redirect_stdout + from io import StringIO + from pathlib import Path + from chatview import db as _db + from chatview import index as index_module + + root = Path(self._tmpdir) + codex_dir = root / "codex-sessions" + codex_dir.mkdir() + empty_dir = root / "empty" + empty_dir.mkdir() + cache_dir = root / "index-cache" + cache_dir.mkdir() + source = codex_dir / "rollout.jsonl" + records = [ + { + "timestamp": "2026-07-17T20:00:00Z", + "type": "session_meta", + "payload": {"id": "cache-loss", "cwd": "/workspace"}, + }, + { + "timestamp": "2026-07-17T20:00:01Z", + "type": "event_msg", + "payload": {"type": "user_message", "message": "restore me"}, + }, + { + "timestamp": "2026-07-17T20:00:02Z", + "type": "response_item", + "payload": { + "type": "function_call", + "name": "exec", + "call_id": "cache-call", + "arguments": '{"cmd":"echo restored"}', + }, + }, + { + "timestamp": "2026-07-17T20:00:03Z", + "type": "response_item", + "payload": { + "type": "function_call_output", + "call_id": "cache-call", + "output": "restored", + }, + }, + ] + source.write_text( + "".join(json.dumps(record) + "\n" for record in records), + encoding="utf-8", + ) + mtime = source.stat().st_mtime + cache_path = cache_dir / "index.json" + skinny = { + "_schema_version": index_module.INDEX_SCHEMA_VERSION, + "projects": {}, + "sessions": { + "codex-cache-loss": { + "id": "codex-cache-loss", + "title": "restore me", + "filePath": str(source), + "source": "codex", + "project": "codex", + "projectName": "workspace", + "userMessageCount": 1, + "messageCount": 1, + "toolCallCount": 1, + "_mtime": mtime, + } + }, + "_file_mtimes": {str(source): mtime}, + } + cache_path.write_text(json.dumps(skinny), encoding="utf-8") + + original = { + name: getattr(index_module, name) + for name in ( + "PROJECTS_DIR", + "CODEX_SESSIONS_DIR", + "CODEX_ARCHIVED_DIR", + "CACHE_DIR", + "INDEX_CACHE", + "_index", + ) + } + try: + index_module.PROJECTS_DIR = empty_dir + index_module.CODEX_SESSIONS_DIR = codex_dir + index_module.CODEX_ARCHIVED_DIR = empty_dir + index_module.CACHE_DIR = cache_dir + index_module.INDEX_CACHE = cache_path + index_module._index = {"projects": {}, "sessions": {}, "_file_mtimes": {}} + with redirect_stdout(StringIO()): + index_module.build_index( + force=False, known_files={str(source): mtime} + ) + finally: + for name, value in original.items(): + setattr(index_module, name, value) + + self.assertEqual( + _db.get_conn().execute( + "SELECT count(*) FROM messages WHERE session_id='codex-cache-loss'" + ).fetchone()[0], + 1, + ) + tool = _db.get_conn().execute( + "SELECT * FROM tool_calls WHERE session_id='codex-cache-loss'" + ).fetchone() + self.assertIsNotNone(tool) + self.assertGreater(tool["result_byte_offset"], tool["byte_offset"]) + self.assertEqual(len(tool["call_record_digest"]), 16) + self.assertIsNotNone( + _db.get_conn().execute( + "SELECT 1 FROM insight_state WHERE session_id='codex-cache-loss'" + ).fetchone() + ) + + def test_refresh_cleans_nullable_legacy_errors_before_malformed_sources_parse(self): + from contextlib import redirect_stdout + from io import StringIO + from pathlib import Path + from chatview import db as _db + from chatview import index as index_module + from chatview.db import core as _dbcore + + db_marker = "LEGACY_CLAUDE_RESULT_SECRET_DB_4C91F" + wal_marker = "LEGACY_CODEX_RESULT_SECRET_WAL_7A20D" + root = Path(self._tmpdir) + projects_dir = root / "claude-projects" + claude_dir = projects_dir / "legacy-project" + claude_dir.mkdir(parents=True) + codex_dir = root / "codex-sessions" + codex_dir.mkdir() + empty_dir = root / "empty" + empty_dir.mkdir() + claude_source = claude_dir / "broken-claude.jsonl" + codex_source = codex_dir / "broken-codex.jsonl" + claude_source.write_text("{not-valid-claude-jsonl\n", encoding="utf-8") + codex_source.write_text("{not-valid-codex-jsonl\n", encoding="utf-8") + + _db.init_db() + for session_id, source, source_name in ( + ("legacy-broken-claude", claude_source, "claude"), + ("legacy-broken-codex", codex_source, "codex"), + ): + old_meta = dict(_META) + old_meta.update( + { + "id": session_id, + "filePath": str(source), + "fileSize": source.stat().st_size, + "_mtime": source.stat().st_mtime, + "source": source_name, + } + ) + _db.upsert_session(old_meta, [], []) + conn = _db.get_conn() + self.assertEqual(conn.execute("PRAGMA secure_delete").fetchone()[0], 1) + conn.executescript(""" + DROP TABLE insight_errors; + CREATE TABLE insight_errors ( + session_id TEXT, + error_key TEXT, + day TEXT, + project TEXT, + count INTEGER DEFAULT 1, + PRIMARY KEY(session_id, error_key) + ); + """) + conn.executemany( + """INSERT INTO insight_errors + (session_id, error_key, day, project, count) VALUES (?,?,?,?,?)""", + ( + ("legacy-null", None, "2026-07-18", "legacy", 1), + ( + "legacy-broken-claude", + f"TypeError: {db_marker}", + "2026-07-18", + "legacy", + 1, + ), + ("legacy-broken-claude", "TypeError", "2026-07-18", "legacy", 1), + ("legacy-broken-codex", "Error", "2026-07-18", "legacy", 1), + ), + ) + conn.commit() + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + self.assertIn(db_marker.encode(), _dbcore.DB_PATH.read_bytes()) + conn.execute( + """INSERT INTO insight_errors + (session_id, error_key, day, project, count) VALUES (?,?,?,?,?)""", + ( + "legacy-broken-codex", + f"RuntimeError: {wal_marker}", + "2026-07-18", + "legacy", + 1, + ), + ) + conn.commit() + wal_path = Path(f"{_dbcore.DB_PATH}-wal") + self.assertIn(wal_marker.encode(), wal_path.read_bytes()) + + cache_dir = root / "index-cache" + original = { + name: getattr(index_module, name) + for name in ( + "PROJECTS_DIR", + "CODEX_SESSIONS_DIR", + "CODEX_ARCHIVED_DIR", + "CACHE_DIR", + "INDEX_CACHE", + "_index", + ) + } + try: + index_module.PROJECTS_DIR = projects_dir + index_module.CODEX_SESSIONS_DIR = codex_dir + index_module.CODEX_ARCHIVED_DIR = empty_dir + index_module.CACHE_DIR = cache_dir + index_module.INDEX_CACHE = cache_dir / "index.json" + index_module._index = {"projects": {}, "sessions": {}, "_file_mtimes": {}} + with redirect_stdout(StringIO()): + index_module.build_index( + force=True, + known_files={ + str(claude_source): claude_source.stat().st_mtime, + str(codex_source): codex_source.stat().st_mtime, + }, + ) + finally: + for name, value in original.items(): + setattr(index_module, name, value) + + self.assertEqual( + { + row[0] + for row in conn.execute("SELECT error_key FROM insight_errors") + }, + {"Error", "TypeError"}, + ) + for candidate in ( + _dbcore.DB_PATH, + Path(f"{_dbcore.DB_PATH}-wal"), + Path(f"{_dbcore.DB_PATH}-shm"), + ): + if candidate.exists(): + raw = candidate.read_bytes() + self.assertNotIn(db_marker.encode(), raw, candidate.name) + self.assertNotIn(wal_marker.encode(), raw, candidate.name) + + conn.executemany( + """INSERT INTO insight_errors + (session_id, error_key, day, project, count) VALUES (?,?,?,?,?)""", + ( + ("repeat-null", None, "2026-07-18", "legacy", 1), + ("repeat-noise", "ValueError: repeat", "2026-07-18", "legacy", 1), + ), + ) + conn.commit() + _db.init_db() + self.assertEqual( + sorted(row[0] for row in conn.execute("SELECT error_key FROM insight_errors")), + ["Error", "TypeError"], + ) + + _dbcore._close_thread_connection() + self.assertTrue(_db.prepare_search_db()) + self.assertEqual( + sorted( + row[0] + for row in _db.get_conn().execute( + "SELECT error_key FROM insight_errors" + ) + ), + ["Error", "TypeError"], + ) + _dbcore._close_thread_connection() + # --------------------------------------------------------------------------- # Session lifecycle tests diff --git a/tests/test_distill_skill_static.py b/tests/test_distill_skill_static.py index ca210cf..f04e0e5 100644 --- a/tests/test_distill_skill_static.py +++ b/tests/test_distill_skill_static.py @@ -43,12 +43,19 @@ def test_skill_description_routes_conversation_history_without_run_history_false self.assertNotIn("## Trigger Gate", text) self.assertIn("先用 `distill` 定位并核验证据", text) - self.assertIn("不先扫描原始 transcript", text) - self.assertIn("才定向读取原始 session JSONL", text) + self.assertIn("不扫描原始 transcript", text) + self.assertIn("never direct-read raw rollout or session JSONL", text) + self.assertIn( + "report the bounded-reader limitation, improve the locator or anchor, or stop", + text, + ) + self.assertNotIn("才定向读取原始 session JSONL", text) + self.assertNotIn("或原 session 内容核验", text) self.assertIn("不要从消息搜索直接跳到原始 JSONL", text) self.assertIn("不保存工具结果", text) - self.assertIn("不要对 session JSONL 运行会回显整行的 `rg -n`", text) - self.assertIn("不要用 `head` 或 `sed`", text) + self.assertIn("不要扫描、打开或直接读取 session/rollout JSONL", text) + for raw_reader in ("`rg -n`", "`rg -l`", "`head`", "`sed`", "`jq`", "Python"): + self.assertIn(raw_reader, text) self.assertIn("先完成历史取证", text) for coupled_name in ( "read_session.py", @@ -90,10 +97,28 @@ def test_skill_md_is_router_and_declares_required_references(self): self.assertIn("distill read-tool-event --event-idx ", text) self.assertIn("--include-result --max-chars 2000", text) self.assertIn("`--max-chars 2000` is a hard ceiling", text) - self.assertIn("never propose 4000, 10000, or an unbounded raw read", text) + self.assertIn( + "never propose 4000, 10000, a raw fallback, or any direct rollout/session JSONL read", + text, + ) self.assertIn("filePath` is provenance, not permission", text) - self.assertIn("result is read on demand from the source JSONL", text) - self.assertIn("is not stored in the SQLite search index", text) + self.assertIn("result locator", text) + self.assertIn("result body is read on demand from the source JSONL", text) + self.assertIn("result body is never stored in the SQLite search index", text) + self.assertIn("structured anchors", text) + self.assertIn("head + tail", text) + self.assertIn("fixed quotas", text) + self.assertIn("dedicated `call_id` column", text) + self.assertIn("does not promise to retain every arbitrarily numerous anchor", text) + self.assertIn("default bounded head projection", text) + self.assertNotIn("default head/tail projection", text) + self.assertIn("--around \"\"", text) + self.assertIn("`radius <= 5`", text) + self.assertIn("`batch size <= 5`", text) + self.assertIn("`total serialized output <= 20 KB`", text) + self.assertIn("all output formats", text) + self.assertIn("multiple batches, later pages, or raw JSONL", text) + self.assertNotIn("--json` returns the complete text stored in the local index", text) self.assertIn("--format lines", text) self.assertIn("--format jsonl", text) self.assertIn("--evidence-only", text) diff --git a/tests/test_retrieval_tools.py b/tests/test_retrieval_tools.py index c81388d..c4fee8f 100644 --- a/tests/test_retrieval_tools.py +++ b/tests/test_retrieval_tools.py @@ -987,7 +987,7 @@ def test_radius_uses_idx_range_and_does_not_fill_gaps(self): self.assertEqual([msg["idx"] for msg in data["messages"]], [2]) - def test_human_output_marks_truncation_but_json_keeps_indexed_text(self): + def test_human_and_json_outputs_share_the_same_message_limit(self): import contextlib import io import json @@ -1009,7 +1009,161 @@ def test_human_output_marks_truncation_but_json_keeps_indexed_text(self): structured = io.StringIO() with contextlib.redirect_stdout(structured): cmd_read_window(self._args(session="long-window", idx=0, radius=0, batch="", json=True)) - self.assertEqual(json.loads(structured.getvalue())["messages"][0]["text"], long_text) + raw_json = structured.getvalue() + payload = json.loads(raw_json) + message = payload["messages"][0] + self.assertNotEqual(message["text"], long_text) + self.assertLessEqual(len(message["text"]), 1200) + self.assertEqual(message["originalChars"], 1500) + self.assertTrue(message["outputTruncated"]) + self.assertTrue(payload["outputTruncated"]) + self.assertEqual(payload["omittedMessages"], 0) + self.assertEqual(payload["outputBytes"], len(raw_json.encode("utf-8"))) + self.assertLessEqual(payload["outputBytes"], 20000) + + def test_worst_legal_batch_preserves_targets_and_never_exceeds_output_budget(self): + import contextlib + import io + import json + from chatview.commands.retrieval import cmd_read_window + + user_messages = [] + assistant_messages = [] + for idx in range(11): + message = { + "idx": idx, + "text": "😀" * 4000, + "ts": f"2026-07-01T10:{idx:02d}:00Z", + } + (user_messages if idx % 2 == 0 else assistant_messages).append(message) + self._insert_session( + "budget-window", + "Budget", + "distill-yourself", + user_messages, + assistant_messages, + ) + batch = json.dumps([ + {"session": "budget-window", "idx": idx, "radius": 5} + for idx in range(3, 8) + ]) + + structured = io.StringIO() + with contextlib.redirect_stdout(structured): + cmd_read_window(self._args(session=None, idx=None, radius=2, batch=batch, json=True)) + raw_json = structured.getvalue() + payload = json.loads(raw_json) + self.assertEqual(payload["outputBytes"], len(raw_json.encode("utf-8"))) + self.assertLessEqual(payload["outputBytes"], 20000) + self.assertTrue(payload["outputTruncated"]) + self.assertGreater(payload["omittedMessages"], 0) + target_lengths = [] + for window in payload["windows"]: + self.assertIn(window["targetIndex"], [msg["idx"] for msg in window["messages"]]) + target = next( + msg for msg in window["messages"] + if msg["idx"] == window["targetIndex"] + ) + target_lengths.append(len(target["text"])) + self.assertEqual(len(set(target_lengths)), 1) + + human = io.StringIO() + with contextlib.redirect_stdout(human): + cmd_read_window(self._args(session=None, idx=None, radius=2, batch=batch, json=False)) + self.assertLessEqual(len(human.getvalue().encode("utf-8")), 20000) + self.assertIn("omittedMessages=", human.getvalue()) + + def test_rejects_radius_outside_zero_to_five(self): + with self.assertRaisesRegex(ValueError, "radius must be between 0 and 5"): + read_window_data("missing", idx=0, radius=-1) + with self.assertRaisesRegex(ValueError, "radius must be between 0 and 5"): + read_window_data("missing", idx=0, radius=6) + with self.assertRaisesRegex(ValueError, "radius must be between 0 and 5"): + read_window_data("missing", idx=0, radius=True) + with self.assertRaisesRegex(ValueError, "radius must be between 0 and 5"): + read_window_data("missing", idx=0, radius=1.9) + + def test_accepts_integer_radius_string(self): + self._insert_session( + "string-radius", + "String radius", + "distill-yourself", + [{"idx": 0, "text": "target", "ts": "2026-07-01T10:00:00Z"}], + ) + + data = read_window_data("string-radius", idx=0, radius="5") + + self.assertEqual(data["radius"], 5) + + def test_skips_oversized_context_and_keeps_later_short_context(self): + requests = [] + for window_index in range(5): + messages = [ + {"idx": 0, "text": "界" * 1000, "ts": "2026-07-01T10:00:00Z"} + ] + if window_index == 0: + messages.append({ + "idx": 1, + "text": "😀" * 1200, + "ts": "2026-07-01T10:01:00Z", + }) + elif window_index == 1: + messages.append({ + "idx": 1, + "text": "short context", + "ts": "2026-07-01T10:01:00Z", + }) + session = f"context-fit-{window_index}" + self._insert_session( + session, + "Context fit", + "distill-yourself", + messages, + ) + requests.append({"session": session, "idx": 0, "radius": 1}) + + data = read_windows_data(requests) + + self.assertEqual( + [msg["idx"] for msg in data["windows"][0]["messages"]], + [0], + ) + self.assertEqual( + [msg["idx"] for msg in data["windows"][1]["messages"]], + [0, 1], + ) + + def test_rejects_more_than_five_batch_items(self): + import json + from chatview.commands.retrieval import cmd_read_window + + requests = [ + {"session": "any", "idx": idx, "radius": 0} + for idx in range(6) + ] + + with self.assertRaisesRegex(ValueError, "at most 5"): + read_windows_data(requests) + with self.assertRaisesRegex(ValueError, "at most 5"): + cmd_read_window(self._args( + session=None, + idx=None, + radius=2, + batch=json.dumps(requests), + json=True, + )) + + def test_command_rejects_radius_six_instead_of_printing_success(self): + from chatview.commands.retrieval import cmd_read_window + + with self.assertRaisesRegex(ValueError, "radius must be between 0 and 5"): + cmd_read_window(self._args( + session="missing", + idx=0, + radius=6, + batch="", + json=True, + )) class TestSessionBriefData(RetrievalToolTestCase): diff --git a/tests/test_tool_call_retrieval.py b/tests/test_tool_call_retrieval.py index d1db39e..f554763 100644 --- a/tests/test_tool_call_retrieval.py +++ b/tests/test_tool_call_retrieval.py @@ -10,7 +10,8 @@ from chatview.commands.tool_calls import read_tool_event_data, tool_search_data from chatview.db import core as dbcore from chatview.parsers.claude import extract_metadata -from chatview.parsers.codex import extract_codex_metadata +from chatview.parsers.codex import _store_session_insights, extract_codex_metadata +from chatview.tool_events import make_bounded_index_text class ToolCallRetrievalTestCase(unittest.TestCase): @@ -99,6 +100,10 @@ def test_codex_indexes_calls_only_and_reads_one_raw_event(self): self.assertGreater(call["original_bytes"], 2000) self.assertGreater(call["byte_offset"], 0) self.assertGreater(call["byte_length"], 0) + self.assertGreater(call["result_byte_offset"], call["byte_offset"]) + self.assertGreater(call["result_byte_length"], 0) + self.assertEqual(len(call["call_record_digest"]), 16) + self.assertEqual(len(call["result_record_digest"]), 16) self._store_meta(meta) hits = tool_search_data(target, self._args()) @@ -106,6 +111,9 @@ def test_codex_indexes_calls_only_and_reads_one_raw_event(self): self.assertEqual(hits[0]["sessionId"], "codex-codex-tool-fixture") self.assertEqual(hits[0]["eventIdx"], 0) self.assertEqual(hits[0]["rawName"], "apply_patch") + call_id_hits = tool_search_data("call-create-result", self._args()) + self.assertEqual(len(call_id_hits), 1) + self.assertEqual(call_id_hits[0]["callId"], "call-create-result") self.assertEqual( tool_search_data("result_only_secret", self._args()), [] ) @@ -128,6 +136,135 @@ def test_codex_indexes_calls_only_and_reads_one_raw_event(self): self.assertEqual(len(hard_capped["input"]), 2000) self.assertTrue(hard_capped["outputTruncated"]) + def test_tail_anchors_are_searchable_and_around_reads_exact_region(self): + target = "middle-only-target-9f3f.json" + command = ( + "node -e \"" + "汉" * 1000 + + f"; tools.apply_patch({{path:'/workspace/deep/{target}'}})\"" + ) + path = self._tmpdir / "codex-tail.jsonl" + records = [ + { + "timestamp": "2026-07-17T20:00:00Z", + "type": "session_meta", + "payload": {"id": "codex-tail-fixture", "cwd": "/workspace"}, + }, + { + "timestamp": "2026-07-17T20:01:00Z", + "type": "response_item", + "payload": { + "type": "function_call", + "name": "exec", + "call_id": "call-tail", + "arguments": json.dumps({"cmd": command}), + }, + }, + ] + path.write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in records), + encoding="utf-8", + ) + + meta = extract_codex_metadata(str(path)) + call = meta["_tool_calls"][0] + self.assertLessEqual(len(call["input_text"].encode("utf-8")), 2000) + self.assertIn(target, call["input_text"]) + self.assertIn("tools.apply_patch", call["input_text"]) + self._store_meta(meta) + + hits = tool_search_data(target, self._args(limit=200, max_chars=5000)) + self.assertEqual(len(hits), 1) + self.assertLessEqual(len(hits[0]["snippet"]), 500) + event = read_tool_event_data( + hits[0]["sessionId"], + hits[0]["eventIdx"], + max_chars=160, + around=target, + ) + self.assertIn(target, event["input"]) + self.assertLessEqual(len(event["input"]), 160) + self.assertGreater(event["inputStartChar"], 0) + self.assertEqual(len(event["input"]), 160) + self.assertEqual(event["inputEndChar"], event["originalChars"]) + + def test_bounded_projection_reserves_anchors_head_and_tail(self): + head_token = "unique-head-token-41" + tail_target = "tail-only-target-77.json" + value = { + "head": head_token + "-" + ("h" * 900), + "items": [ + {"path": f"/workspace/generated/path-{index:03d}.json"} + for index in range(120) + ] + [{"path": f"/workspace/final/{tail_target}"}], + "after": "z" * 2000, + } + + projected, original_bytes, truncated = make_bounded_index_text(value) + + self.assertTrue(truncated) + self.assertGreater(original_bytes, 2000) + self.assertLessEqual(len(projected.encode("utf-8")), 2000) + self.assertIn(head_token, projected) + self.assertIn(tail_target, projected) + + def test_utf8_input_budget_boundaries(self): + for ascii_count, expected_bytes, expected_truncated in ( + (1996, 1999, False), + (1997, 2000, False), + (1998, 2001, True), + ): + projected, original_bytes, truncated = make_bounded_index_text( + "a" * ascii_count + "界" + ) + self.assertEqual(original_bytes, expected_bytes) + self.assertEqual(truncated, expected_truncated) + self.assertLessEqual(len(projected.encode("utf-8")), 2000) + hard_capped, _, hard_truncated = make_bounded_index_text( + "x" * 3000, max_bytes=10_000 + ) + self.assertTrue(hard_truncated) + self.assertLessEqual(len(hard_capped.encode("utf-8")), 2000) + + def test_tool_search_hard_caps_results_at_twenty(self): + source_path = self._tmpdir / "cap.jsonl" + source_path.write_text("", encoding="utf-8") + calls = [] + for event_idx in range(25): + calls.append( + { + "event_idx": event_idx, + "line_number": event_idx + 1, + "byte_offset": event_idx * 10, + "byte_length": 10, + "block_index": 0, + "ts": "2026-07-17T20:00:00Z", + "tool_name": "Bash", + "raw_name": "exec", + "call_id": f"call-{event_idx}", + "input_text": f"shared-hard-cap {event_idx}", + "original_bytes": 20, + "input_truncated": False, + } + ) + meta = { + "id": "codex-cap-fixture", + "title": "cap", + "date": "2026-07-17T20:00:00Z", + "lastDate": "2026-07-17T20:00:00Z", + "filePath": str(source_path), + "fileSize": 0, + "userMessageCount": 0, + "userTexts": [], + "assistantSnippets": [], + "preview": "", + "source": "codex", + "_tool_calls": calls, + } + self._store_meta(meta) + self.assertEqual( + len(tool_search_data("shared-hard-cap", self._args(limit=999))), 20 + ) + def test_claude_locator_selects_the_exact_tool_block(self): path = self._tmpdir / "claude.jsonl" records = [ @@ -195,10 +332,201 @@ def test_claude_locator_selects_the_exact_tool_block(self): self.assertIn("second.json", event["input"]) self.assertNotIn("first.json", event["input"]) self.assertEqual(event["result"], "claude_result_only_secret") + self.assertEqual(event["resultStatus"], "found") self.assertEqual( tool_search_data("claude_result_only_secret", self._args()), [] ) + def test_missing_and_stale_result_locators_are_explicit(self): + path = self._tmpdir / "missing-result.jsonl" + records = [ + { + "timestamp": "2026-07-17T20:00:00Z", + "type": "session_meta", + "payload": {"id": "missing-result", "cwd": "/workspace"}, + }, + { + "timestamp": "2026-07-17T20:01:00Z", + "type": "response_item", + "payload": { + "type": "function_call", + "name": "exec", + "call_id": "call-missing", + "arguments": "{}", + }, + }, + ] + path.write_text( + "".join(json.dumps(row) + "\n" for row in records), encoding="utf-8" + ) + meta = extract_codex_metadata(str(path)) + self._store_meta(meta) + missing = read_tool_event_data(meta["id"], 0, include_result=True) + self.assertIsNone(missing["result"]) + self.assertEqual(missing["resultStatus"], "not_indexed") + self.assertIn("refresh", missing["resultMessage"]) + + call = meta["_tool_calls"][0] + call.update( + { + "result_line_number": 1, + "result_byte_offset": 0, + "result_byte_length": len((json.dumps(records[0]) + "\n").encode()), + "result_block_index": 0, + } + ) + db.replace_tool_calls(meta["id"], meta["_tool_calls"]) + with self.assertRaisesRegex(RuntimeError, "result.*changed|no longer"): + read_tool_event_data(meta["id"], 0, include_result=True) + + def test_same_length_call_and_result_rewrites_fail_digest_check(self): + path = self._tmpdir / "digest-stale.jsonl" + records = [ + { + "timestamp": "2026-07-17T20:00:00Z", + "type": "session_meta", + "payload": {"id": "digest-stale", "cwd": "/workspace"}, + }, + { + "timestamp": "2026-07-17T20:01:00Z", + "type": "response_item", + "payload": { + "type": "custom_tool_call", + "name": "apply_patch", + "call_id": "digest-call", + "input": "same length input", + }, + }, + { + "timestamp": "2026-07-17T20:01:01Z", + "type": "response_item", + "payload": { + "type": "custom_tool_call_output", + "call_id": "digest-call", + "output": "result-original", + }, + }, + ] + original = "".join(json.dumps(row) + "\n" for row in records) + path.write_text(original, encoding="utf-8") + meta = extract_codex_metadata(str(path)) + self._store_meta(meta) + + path.write_text(original.replace("apply_patch", "other_patch"), encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "digest is stale"): + read_tool_event_data(meta["id"], 0) + + path.write_text( + original.replace("result-original", "result-modified"), encoding="utf-8" + ) + with self.assertRaisesRegex(RuntimeError, "digest is stale"): + read_tool_event_data(meta["id"], 0, include_result=True) + + def test_result_body_never_persists_in_sqlite_or_sidecars(self): + codex_marker = "CODEX_RESULT_BODY_SECRET_83F09" + claude_marker = "CLAUDE_RESULT_BODY_SECRET_72A18" + codex_path = self._tmpdir / "result-secret-codex.jsonl" + codex_records = [ + { + "timestamp": "2026-07-18T05:00:00Z", + "type": "session_meta", + "payload": {"id": "result-secret-codex", "cwd": "/workspace"}, + }, + { + "timestamp": "2026-07-18T05:00:01Z", + "type": "response_item", + "payload": { + "type": "function_call", + "name": "exec", + "call_id": "codex-secret-call", + "arguments": '{"cmd":"false"}', + }, + }, + { + "timestamp": "2026-07-18T05:00:02Z", + "type": "response_item", + "payload": { + "type": "function_call_output", + "call_id": "codex-secret-call", + "output": f"Error: {codex_marker}", + }, + }, + ] + codex_path.write_text( + "".join(json.dumps(record) + "\n" for record in codex_records), + encoding="utf-8", + ) + + claude_path = self._tmpdir / "result-secret-claude.jsonl" + claude_records = [ + { + "type": "user", + "sessionId": "result-secret-claude", + "timestamp": "2026-07-18T06:00:00Z", + "message": {"content": [{"type": "text", "text": "run it"}]}, + }, + { + "type": "assistant", + "timestamp": "2026-07-18T06:00:01Z", + "message": { + "content": [ + { + "type": "tool_use", + "id": "claude-secret-call", + "name": "Bash", + "input": {"command": "false"}, + } + ] + }, + }, + { + "type": "user", + "toolUseResult": {"status": "failed"}, + "timestamp": "2026-07-18T06:00:02Z", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "claude-secret-call", + "content": f"TypeError: {claude_marker}", + } + ] + }, + }, + ] + claude_path.write_text( + "".join(json.dumps(record) + "\n" for record in claude_records), + encoding="utf-8", + ) + + metas = [ + extract_codex_metadata(str(codex_path)), + extract_metadata(str(claude_path)), + ] + self.assertEqual(metas[0]["_insight_errors"], [("Error", "2026-07-18")]) + self.assertEqual( + metas[1]["_insight_errors"], [("TypeError", "2026-07-18")] + ) + for meta in metas: + self._store_meta(meta) + _store_session_insights(meta) + + error_keys = { + row[0] + for row in db.get_conn().execute("SELECT error_key FROM insight_errors") + } + self.assertEqual(error_keys, {"Error", "TypeError"}) + for marker in (codex_marker, claude_marker): + self.assertEqual(tool_search_data(marker, self._args()), []) + marker_bytes = marker.encode("utf-8") + for candidate in ( + dbcore.DB_PATH, + Path(f"{dbcore.DB_PATH}-wal"), + Path(f"{dbcore.DB_PATH}-shm"), + ): + if candidate.exists(): + self.assertNotIn(marker_bytes, candidate.read_bytes(), candidate.name) + if __name__ == "__main__": unittest.main() From abff85f366f9e400ca9e6ede6892ed113f846927 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sat, 18 Jul 2026 02:38:54 -0400 Subject: [PATCH 05/15] docs: standardize distill skill sync --- skills/distill-yourself/SKILL.md | 1 + tests/test_distill_skill_static.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/skills/distill-yourself/SKILL.md b/skills/distill-yourself/SKILL.md index e5a3f7c..2482ee0 100644 --- a/skills/distill-yourself/SKILL.md +++ b/skills/distill-yourself/SKILL.md @@ -22,6 +22,7 @@ description: "本地 Claude Code/Codex/agent 对话历史的唯一检索入口 - Memory/Profile 的标准流程止于 `distill evolve-write`。告诉用户结果已暂存，并由用户在 UI 中预览、确认和同步；不要代替用户运行 `distill evolve-sync --execute`。 - 只有用户明确要求使用 CLI 同步时，才把 `distill evolve-sync` 作为后备入口，并仍须先展示完整预览/diff、再次取得明确确认。不要手搓 Claude/Codex 配置格式。 - Twin 写回仍走 `distill twin-sync`；任何对 `~/.claude/`、`~/.codex/AGENTS.md` 或 `~/.agents/skills/` 的写入都必须先展示完整预览/diff，并得到用户明确确认。 +- 经用户确认更新本 Skill 后，调用项目内置的 `distill install-skill --force`，不要手工复制；该命令将仓库内 `skills/distill-yourself` 同步到 `~/.claude/skills/distill-yourself` 和 `~/.agents/skills/distill-yourself`。 - Codex/Claude 子进程可能隔离用户配置、rules、plugins；面向子进程的 prompt 必须自包含，不依赖当前 session 的隐式记忆。 ## CLI Entry diff --git a/tests/test_distill_skill_static.py b/tests/test_distill_skill_static.py index f04e0e5..142c0d4 100644 --- a/tests/test_distill_skill_static.py +++ b/tests/test_distill_skill_static.py @@ -127,6 +127,8 @@ def test_skill_md_is_router_and_declares_required_references(self): self.assertIn('"sessionId":"session_id","idx":123,"radius":2', text) self.assertIn("~/.agents/skills/distill-yourself", text) self.assertNotIn("~/.codex/skills/distill-yourself", text) + self.assertIn("distill install-skill --force", text) + self.assertIn("不要手工复制", text) self.assertIn("写回 ~/.claude", text) self.assertNotIn("search-plus", text) self.assertNotIn("-C 1", text) From 2a9667ad9b00b7e054503e273ad74be90a4f2f53 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sat, 18 Jul 2026 03:05:11 -0400 Subject: [PATCH 06/15] feat: add exact history date ranges --- chatview/cli.py | 26 ++++++++++++++ chatview/commands/analysis.py | 45 ++++++------------------ chatview/commands/retrieval.py | 30 +++++++++------- chatview/commands/tool_calls.py | 2 ++ chatview/db/sessions.py | 35 ++++++++++++------ chatview/db/tool_calls.py | 13 +++++-- chatview/time_scope.py | 34 ++++++++++++++++++ skills/distill-yourself/SKILL.md | 5 +++ skills/distill-yourself/evals/evals.json | 11 ++++++ tests/test_commands_analysis.py | 44 +++++++++++++++++++++++ tests/test_db.py | 30 ++++++++++++++++ tests/test_distill_skill_static.py | 5 +++ tests/test_tool_call_retrieval.py | 8 +++++ 13 files changed, 229 insertions(+), 59 deletions(-) create mode 100644 chatview/time_scope.py diff --git a/chatview/cli.py b/chatview/cli.py index a3e310b..e575370 100644 --- a/chatview/cli.py +++ b/chatview/cli.py @@ -8,6 +8,7 @@ import argparse import io import sys +from datetime import datetime from chatview.commands.analysis import ( cmd_sessions, @@ -66,6 +67,13 @@ def _max_chars(value): return parsed +def _calendar_date(value): + try: + return datetime.strptime(value, "%Y-%m-%d").strftime("%Y-%m-%d") + except ValueError as exc: + raise argparse.ArgumentTypeError("expected YYYY-MM-DD") from exc + + def main(): parser = argparse.ArgumentParser( description="CLI tools for analyzing Claude Code / Codex conversation history.", @@ -75,6 +83,7 @@ def main(): python3 analyze.py sessions --date 7d --source claude python3 analyze.py read python3 analyze.py search "redis cache" --date 30d + python3 analyze.py search "redis cache" --start 2026-07-01 --end 2026-07-07 python3 analyze.py corrections --date 7d --project myproject python3 analyze.py errors --date 30d python3 analyze.py decisions --date 7d @@ -88,6 +97,14 @@ def main(): shared.add_argument( "--date", default="7d", help="Time filter: 1d, 7d, 30d, 90d, all (default: 7d)" ) + shared.add_argument( + "--start", type=_calendar_date, default=None, + help="Inclusive start date (YYYY-MM-DD); use with --end for an exact range", + ) + shared.add_argument( + "--end", type=_calendar_date, default=None, + help="Inclusive end date (YYYY-MM-DD); use with --start for an exact range", + ) shared.add_argument( "--source", default="all", @@ -477,6 +494,15 @@ def main(): if not args.command: parser.print_help() sys.exit(1) + if getattr(args, "start", "") or getattr(args, "end", ""): + explicit_date = any( + token == "--date" or token.startswith("--date=") + for token in sys.argv[1:] + ) + if explicit_date: + parser.error("--date cannot be combined with --start or --end") + if args.start and args.end and args.start > args.end: + parser.error("--start must be on or before --end") cmds = { "sessions": cmd_sessions, diff --git a/chatview/commands/analysis.py b/chatview/commands/analysis.py index 5905b5b..55a1a75 100644 --- a/chatview/commands/analysis.py +++ b/chatview/commands/analysis.py @@ -4,7 +4,6 @@ import json import os import sys -from datetime import datetime, timedelta from pathlib import Path from chatview import index as _idx @@ -20,6 +19,7 @@ from chatview.commands.evidence import artifact_reason, dedupe_message_results, page_results from chatview.project_identity import project_matches from chatview.snippets import make_query_snippet +from chatview.time_scope import date_bounds, date_in_bounds # --------------------------------------------------------------------------- @@ -103,8 +103,7 @@ def cmd_install_skill(args): def _apply_filters(sessions: dict, args) -> dict: """Filter sessions by date/source/project from CLI args.""" filtered = {} - now = datetime.now() - days_map = {"1d": 1, "7d": 7, "30d": 30, "90d": 90} + start_date, end_date = date_bounds(args) for sid, m in sessions.items(): # Source @@ -115,18 +114,8 @@ def _apply_filters(sessions: dict, args) -> dict: if args.project and not project_matches(m, args.project, substring=True): continue # Date - if args.date and args.date != "all": - date_str = m.get("date", "") - if date_str: - try: - d = datetime.fromisoformat(date_str.replace("Z", "+00:00")).replace( - tzinfo=None - ) - max_days = days_map.get(args.date, 9999) - if (now - d).total_seconds() > max_days * 86400: - continue - except Exception: - pass + if not date_in_bounds(m.get("date", ""), start_date, end_date): + continue filtered[sid] = m return filtered @@ -162,12 +151,12 @@ def _get_filtered_db(args) -> list: from chatview import db as _db _ensure_project_identity_backfill(args) - days_map = {"1d": 1, "7d": 7, "30d": 30, "90d": 90} - max_days = days_map.get(getattr(args, "date", ""), 99999) + start_date, end_date = date_bounds(args) return _db.get_filtered_sessions( source=getattr(args, "source", "all"), project=getattr(args, "project", ""), - max_days=max_days, + start_date=start_date, + end_date=end_date, ) @@ -405,11 +394,7 @@ def cmd_search(args): _prepare_search_db(args) - now = datetime.now() - days_map = {"1d": 1, "7d": 7, "30d": 30, "90d": 90} - min_date = "" - if args.date and args.date != "all": - min_date = (now - timedelta(days=days_map.get(args.date, 9999))).strftime("%Y-%m-%d") + min_date, max_date = date_bounds(args) page = max(getattr(args, "page", 1), 1) page_size = max(getattr(args, "limit", 20), 1) @@ -428,6 +413,7 @@ def cmd_search(args): source=getattr(args, "source", "all"), project=getattr(args, "project", ""), min_date=min_date, + max_date=max_date, ) # Rebuild from the current prefix so duplicate metadata stays complete. @@ -447,17 +433,8 @@ def cmd_search(args): substring=True, ): continue - if args.date and args.date != "all": - date_str = r.get("ts") or "" - if date_str: - try: - d = datetime.fromisoformat(date_str.replace("Z", "+00:00")).replace( - tzinfo=None - ) - if (now - d).days > days_map.get(args.date, 9999): - continue - except Exception: - pass + if not date_in_bounds(r.get("ts") or "", min_date, max_date): + continue text = r.get("text", "") reason = artifact_reason(text, r.get("title", "")) diff --git a/chatview/commands/retrieval.py b/chatview/commands/retrieval.py index bd21308..5a33ec4 100644 --- a/chatview/commands/retrieval.py +++ b/chatview/commands/retrieval.py @@ -5,7 +5,7 @@ import json import re import sqlite3 -from datetime import datetime, timedelta +from datetime import datetime from math import log2 from chatview.commands.evidence import ( @@ -15,6 +15,7 @@ page_results, ) from chatview.snippets import make_query_snippet +from chatview.time_scope import date_bounds _SEARCH_HIGH_RETRIEVAL_WINDOW = 20 @@ -142,23 +143,22 @@ def _eligible_session_ids(args, prepare: bool = True) -> set: if prepare: _ensure_project_identity_backfill(args) - days_map = {"1d": 1, "7d": 7, "30d": 30, "90d": 90} - max_days = days_map.get(getattr(args, "date", ""), 99999) + start_date, end_date = date_bounds(args) sessions = _db.get_filtered_sessions( source=getattr(args, "source", "all"), project=getattr(args, "project", ""), - max_days=max_days, + start_date=start_date, + end_date=end_date, ) return {s["id"] for s in sessions} def _min_date_for_args(args) -> str: - days = {"1d": 1, "7d": 7, "30d": 30, "90d": 90}.get( - getattr(args, "date", "") - ) - if days is None: - return "" - return (datetime.utcnow() - timedelta(days=days)).strftime("%Y-%m-%d") + return date_bounds(args)[0] + + +def _max_date_for_args(args) -> str: + return date_bounds(args)[1] def _sql_scope_ids(args, eligible: set) -> set | None: @@ -168,6 +168,8 @@ def _sql_scope_ids(args, eligible: set) -> set | None: return eligible if getattr(args, "date", "all") not in ("", "all"): return eligible + if getattr(args, "start", "") or getattr(args, "end", ""): + return eligible return None @@ -331,6 +333,7 @@ def search_high_data(query: str, args) -> list: source=getattr(args, "source", "all"), project=getattr(args, "project", ""), min_date=_min_date_for_args(args), + max_date=_max_date_for_args(args), role=requested_role, ): if row["session_id"] in eligible: @@ -354,6 +357,7 @@ def search_high_data(query: str, args) -> list: source=getattr(args, "source", "all"), project=getattr(args, "project", ""), min_date=_min_date_for_args(args), + max_date=_max_date_for_args(args), )): if row["session_id"] not in eligible: continue @@ -856,12 +860,12 @@ def _metadata_anchor_session_ids(args, anchors: list) -> set: """Return sessions whose title/project carries a topic anchor.""" from chatview import db as _db - days_map = {"1d": 1, "7d": 7, "30d": 30, "90d": 90} - max_days = days_map.get(getattr(args, "date", ""), 99999) + start_date, end_date = date_bounds(args) sessions = _db.get_filtered_sessions( source=getattr(args, "source", "all"), project=getattr(args, "project", ""), - max_days=max_days, + start_date=start_date, + end_date=end_date, ) ids = set() for session in sessions: diff --git a/chatview/commands/tool_calls.py b/chatview/commands/tool_calls.py index a612845..116b52e 100644 --- a/chatview/commands/tool_calls.py +++ b/chatview/commands/tool_calls.py @@ -41,6 +41,8 @@ def tool_search_data(query: str, args) -> list: source=getattr(args, "source", "all"), project=getattr(args, "project", ""), date_range=getattr(args, "date", "all"), + start_date=getattr(args, "start", ""), + end_date=getattr(args, "end", ""), limit=min(max(getattr(args, "limit", 20), 1), 20), page=max(getattr(args, "page", 1), 1), ) diff --git a/chatview/db/sessions.py b/chatview/db/sessions.py index ec8e608..e651221 100644 --- a/chatview/db/sessions.py +++ b/chatview/db/sessions.py @@ -229,7 +229,10 @@ def prune_stale_sessions(valid_file_paths) -> int: # --------------------------------------------------------------------------- # Queries # --------------------------------------------------------------------------- -def get_filtered_sessions(source="all", project="", date="", max_days=99999) -> list: +def get_filtered_sessions( + source="all", project="", date="", max_days=99999, + start_date="", end_date="", +) -> list: """Return list of session dicts filtered by source/project/date range.""" conn = get_conn() clauses = [] @@ -256,14 +259,18 @@ def get_filtered_sessions(source="all", project="", date="", max_days=99999) -> needle = f"%{escaped}%" params.extend([needle, needle, needle]) - if date: - clauses.append("date >= ?") - params.append(date) + if date and not start_date: + start_date = date + + if max_days < 99999 and not start_date: + start_date = (datetime.utcnow() - timedelta(days=max_days)).strftime("%Y-%m-%d") - if max_days < 99999: - cutoff = (datetime.utcnow() - timedelta(days=max_days)).strftime("%Y-%m-%d") - clauses.append("date >= ?") - params.append(cutoff) + if start_date: + clauses.append("SUBSTR(COALESCE(date, ''), 1, 10) >= ?") + params.append(start_date) + if end_date: + clauses.append("SUBSTR(COALESCE(date, ''), 1, 10) <= ?") + params.append(end_date) where = ("WHERE " + " AND ".join(clauses)) if clauses else "" sql = f"SELECT * FROM sessions {where} ORDER BY date DESC" @@ -363,6 +370,7 @@ def search_fts( source="all", project="", min_date="", + max_date="", ) -> list: """Full-text search on messages. Returns dicts with message + session info.""" conn = get_conn() @@ -384,8 +392,11 @@ def search_fts( )""") filter_params.extend([needle, needle, needle]) if min_date: - filters.append("COALESCE(NULLIF(m.ts, ''), s.date, '') >= ?") + filters.append("SUBSTR(COALESCE(NULLIF(m.ts, ''), s.date, ''), 1, 10) >= ?") filter_params.append(min_date) + if max_date: + filters.append("SUBSTR(COALESCE(NULLIF(m.ts, ''), s.date, ''), 1, 10) <= ?") + filter_params.append(max_date) filter_sql = (" AND " + " AND ".join(filters)) if filters else "" fts_sql = f""" @@ -434,6 +445,7 @@ def search_title_fts( source="all", project="", min_date="", + max_date="", ) -> list: """Full-text search on session titles and project names.""" conn = get_conn() @@ -452,8 +464,11 @@ def search_title_fts( )""") filter_params.extend([needle, needle, needle]) if min_date: - filters.append("COALESCE(s.date, '') >= ?") + filters.append("SUBSTR(COALESCE(s.date, ''), 1, 10) >= ?") filter_params.append(min_date) + if max_date: + filters.append("SUBSTR(COALESCE(s.date, ''), 1, 10) <= ?") + filter_params.append(max_date) filter_sql = (" AND " + " AND ".join(filters)) if filters else "" fts_sql = f""" SELECT s.id AS session_id, s.title, s.project_name, s.project_key, diff --git a/chatview/db/tool_calls.py b/chatview/db/tool_calls.py index 38f491c..78ea1c2 100644 --- a/chatview/db/tool_calls.py +++ b/chatview/db/tool_calls.py @@ -93,6 +93,8 @@ def search_tool_calls( source: str = "all", project: str = "", date_range: str = "all", + start_date: str = "", + end_date: str = "", limit: int = 20, page: int = 1, ) -> list: @@ -118,10 +120,17 @@ def search_tool_calls( )""" ) params.extend([needle, needle, needle]) - min_date = _min_date(date_range) + min_date = start_date or ("" if end_date else _min_date(date_range)) if min_date: - filters.append("COALESCE(NULLIF(tc.ts, ''), s.date, '') >= ?") + filters.append( + "SUBSTR(COALESCE(NULLIF(tc.ts, ''), s.date, ''), 1, 10) >= ?" + ) params.append(min_date) + if end_date: + filters.append( + "SUBSTR(COALESCE(NULLIF(tc.ts, ''), s.date, ''), 1, 10) <= ?" + ) + params.append(end_date) filter_sql = (" AND " + " AND ".join(filters)) if filters else "" page_size = min(max(int(limit), 1), 20) offset = (max(int(page), 1) - 1) * page_size diff --git a/chatview/time_scope.py b/chatview/time_scope.py new file mode 100644 index 0000000..50bfb97 --- /dev/null +++ b/chatview/time_scope.py @@ -0,0 +1,34 @@ +"""Shared calendar-date scope helpers for history retrieval commands.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + + +_SHORTCUT_DAYS = {"1d": 1, "7d": 7, "30d": 30, "90d": 90} + + +def date_bounds(args, *, now: datetime | None = None) -> tuple[str, str]: + """Return inclusive YYYY-MM-DD bounds; exact bounds override --date.""" + start = getattr(args, "start", "") or "" + end = getattr(args, "end", "") or "" + if start or end: + return start, end + + days = _SHORTCUT_DAYS.get(getattr(args, "date", "")) + if days is None: + return "", "" + current = now or datetime.now(timezone.utc) + return (current - timedelta(days=days)).strftime("%Y-%m-%d"), "" + + +def date_in_bounds(value: str, start: str = "", end: str = "") -> bool: + """Match an ISO-like timestamp by its encoded calendar date.""" + day = (value or "")[:10] + if not day: + return True + if start and day < start: + return False + if end and day > end: + return False + return True diff --git a/skills/distill-yourself/SKILL.md b/skills/distill-yourself/SKILL.md index 2482ee0..8d11761 100644 --- a/skills/distill-yourself/SKILL.md +++ b/skills/distill-yourself/SKILL.md @@ -44,9 +44,12 @@ description: "本地 Claude Code/Codex/agent 对话历史的唯一检索入口 Choose the scope before retrieval. For an exact fact, phrase, session, or project episode, use the fast path: refresh once, retrieve one page of at most 20 candidates, then verify only the best 2-4 windows. Do not load digest/aggregates/stats for a simple lookup. +Resolve the user's time wording before retrieval. For a reproducible calendar range, use `--start YYYY-MM-DD --end YYYY-MM-DD`; both boundaries are inclusive, either boundary may be omitted, and these options must not be combined with `--date`. Treat `--date 1d/7d/30d/90d/all` as a quick relative shortcut only. When the user says “过去一周” or another relative period, resolve it against the current date, state the interpreted dates, and carry the same exact range through every locating command. If the user requests commands only, state the interpretation in a short shell comment before the commands. By default, “过去 N 天” includes today and the preceding `N-1` calendar dates; surface a materially different interpretation instead of silently changing it. + ```bash distill refresh distill search "" --recall normal --role user --evidence-only --format jsonl --max-chars 500 --limit 20 --page 1 --date 90d +distill search "" --recall normal --role user --format jsonl --max-chars 500 --limit 20 --page 1 --start 2026-07-01 --end 2026-07-07 distill read-window --batch '[{"sessionId":"session_id","idx":123,"radius":2}]' ``` @@ -81,6 +84,8 @@ If `tool-search` misses, refresh once, retry with a shorter stable basename or c Choose the scope before ORIENT, and use ORIENT only for broad profile, trend, correction-pattern, or cognitive-model analysis. If the user asks about a recent/project-specific topic, carry that `--date` / `--project` into the orienting commands; otherwise use `--date all --source all`. +For exact calendar requests, carry `--start` / `--end` instead of `--date` into `search`, `tool-search`, `sessions`, `queries`, `corrections`, `find-repeats`, and other history-locating commands. Once a precise `sessionId + idx` or `sessionId + eventIdx` is selected, the bounded reader uses that locator directly and does not need the time filter repeated. + For broad global analysis, start with: ```bash diff --git a/skills/distill-yourself/evals/evals.json b/skills/distill-yourself/evals/evals.json index b1221b9..99cad8b 100644 --- a/skills/distill-yourself/evals/evals.json +++ b/skills/distill-yourself/evals/evals.json @@ -70,6 +70,17 @@ "Refuses to reconstruct omitted context through multiple batches, later pages, another format, or raw JSONL", "Narrows candidate locators or anchors and stops once the selected evidence is sufficient" ] + }, + { + "id": 7, + "prompt": "假设今天是 2026-07-18。帮我从过去一周的历史对话中定位我关于上下文爆掉的原话。现在只写检索和核验命令，不要执行。", + "expected_output": "先把过去一周解释为包含今天在内的 2026-07-12 到 2026-07-18，并明确复述该范围；使用 search --start 2026-07-12 --end 2026-07-18 定位，再选择少量 sessionId+idx 用 read-window 核验；不把 --start/--end 与 --date 混用。", + "files": [], + "expectations": [ + "Resolves the relative week to the inclusive dates 2026-07-12 through 2026-07-18 and states that interpretation", + "Uses search with --start 2026-07-12 and --end 2026-07-18 without also using --date", + "Selects a small number of sessionId plus idx locators and verifies them with read-window" + ] } ] } diff --git a/tests/test_commands_analysis.py b/tests/test_commands_analysis.py index 36a07cd..9c44222 100644 --- a/tests/test_commands_analysis.py +++ b/tests/test_commands_analysis.py @@ -25,6 +25,8 @@ def _default_args(**kwargs): source="all", project="", date="", + start="", + end="", limit=20, json=False, keyword="", @@ -65,6 +67,21 @@ def test_legacy_index_filter_matches_canonical_alias(self): self.assertEqual(set(filtered), {"canonical"}) + def test_exact_date_range_is_inclusive_and_overrides_shortcut(self): + from chatview.commands.analysis import _apply_filters + + sessions = { + day: {"date": f"{day}T12:00:00Z", "source": "codex"} + for day in ("2026-07-01", "2026-07-02", "2026-07-03") + } + + filtered = _apply_filters( + sessions, + _default_args(date="1d", start="2026-07-01", end="2026-07-02"), + ) + + self.assertEqual(set(filtered), {"2026-07-01", "2026-07-02"}) + class TestCmdStats(unittest.TestCase): """cmd_stats reads the DB and prints a statistics summary.""" @@ -713,6 +730,33 @@ def test_search_defaults_to_normal_recall_and_twenty_results(self): self.assertEqual(args.limit, 20) self.assertEqual(args.max_chars, 500) + def test_search_accepts_inclusive_exact_date_range(self): + import chatview.cli as cli + + argv = [ + "distill", "search", "needle", + "--start", "2026-07-01", "--end", "2026-07-07", + ] + with patch.object(sys, "argv", argv), patch.object(cli, "cmd_search") as command: + cli.main() + + args = command.call_args.args[0] + self.assertEqual(args.start, "2026-07-01") + self.assertEqual(args.end, "2026-07-07") + + def test_search_rejects_reversed_or_ambiguous_date_ranges(self): + import chatview.cli as cli + + cases = ( + ["distill", "search", "needle", "--start", "2026-07-08", "--end", "2026-07-07"], + ["distill", "search", "needle", "--date", "7d", "--start", "2026-07-01"], + ) + for argv in cases: + with self.subTest(argv=argv), patch.object(sys, "argv", argv): + with self.assertRaises(SystemExit) as raised: + cli.main() + self.assertEqual(raised.exception.code, 2) + def test_search_plus_is_an_illegal_command(self): import chatview.cli as cli diff --git a/tests/test_db.py b/tests/test_db.py index 51554d1..cf4a962 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -675,6 +675,36 @@ def test_search_fts_finds_user_text(self): texts = [r["text"] for r in results] self.assertTrue(any("unique search term" in t for t in texts)) + def test_exact_date_bounds_filter_sessions_and_messages_inclusively(self): + for day in ("2026-07-01", "2026-07-02", "2026-07-03"): + meta = { + **_META, + "id": f"range-{day}", + "filePath": f"/tmp/range-{day}.jsonl", + "date": f"{day}T10:00:00Z", + } + db.upsert_session( + meta, + [{"idx": 0, "text": "exactrangefilterneedle", "ts": f"{day}T10:00:00Z"}], + [], + ) + + sessions = db.get_filtered_sessions( + start_date="2026-07-01", end_date="2026-07-02" + ) + messages = db.search_fts( + "exactrangefilterneedle", min_date="2026-07-01", max_date="2026-07-02" + ) + + self.assertEqual( + {row["id"] for row in sessions}, + {"range-2026-07-01", "range-2026-07-02"}, + ) + self.assertEqual( + {row["session_id"] for row in messages}, + {"range-2026-07-01", "range-2026-07-02"}, + ) + def test_reupsert_replaces_fts(self): db.upsert_session(_META, _USER_TEXTS, _ASST_SNIPPETS) diff --git a/tests/test_distill_skill_static.py b/tests/test_distill_skill_static.py index 142c0d4..41bbcb6 100644 --- a/tests/test_distill_skill_static.py +++ b/tests/test_distill_skill_static.py @@ -87,6 +87,11 @@ def test_skill_md_is_router_and_declares_required_references(self): self.assertNotIn("§认知模型", text) self.assertIn("Choose the scope before ORIENT", text) self.assertIn("Do not load digest/aggregates/stats for a simple lookup", text) + self.assertIn("--start YYYY-MM-DD --end YYYY-MM-DD", text) + self.assertIn("both boundaries are inclusive", text) + self.assertIn("must not be combined with `--date`", text) + self.assertIn("short shell comment before the commands", text) + self.assertIn("includes today and the preceding `N-1` calendar dates", text) self.assertIn('distill search "" --recall normal --role user --evidence-only --format jsonl', text) self.assertIn('distill find-repeats "" --limit 5 --json', text) self.assertIn("messageIndex", text) diff --git a/tests/test_tool_call_retrieval.py b/tests/test_tool_call_retrieval.py index f554763..07588ed 100644 --- a/tests/test_tool_call_retrieval.py +++ b/tests/test_tool_call_retrieval.py @@ -117,6 +117,14 @@ def test_codex_indexes_calls_only_and_reads_one_raw_event(self): self.assertEqual( tool_search_data("result_only_secret", self._args()), [] ) + self.assertEqual( + len(tool_search_data(target, self._args(start="2026-07-17", end="2026-07-17"))), + 1, + ) + self.assertEqual( + tool_search_data(target, self._args(end="2026-07-16")), + [], + ) event = read_tool_event_data( hits[0]["sessionId"], 0, max_chars=80, include_result=True From cf0756109e4c556fe345bbe0ff33ca472dc7a1e9 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sat, 18 Jul 2026 03:18:25 -0400 Subject: [PATCH 07/15] fix: serialize history index refreshes --- chatview/commands/analysis.py | 26 ++++++----- chatview/index.py | 82 +++++++++++++++++++++++++++++++-- tests/test_commands_analysis.py | 18 ++++++++ tests/test_index_fts.py | 77 +++++++++++++++++++++++++++++++ 4 files changed, 187 insertions(+), 16 deletions(-) diff --git a/chatview/commands/analysis.py b/chatview/commands/analysis.py index 55a1a75..af1837f 100644 --- a/chatview/commands/analysis.py +++ b/chatview/commands/analysis.py @@ -7,7 +7,7 @@ from pathlib import Path from chatview import index as _idx -from chatview.index import build_index +from chatview.index import build_index, IndexDatabaseLockedError from chatview.session_loader import load_session_from_file from chatview.parsers.codex import _CODEX_TOOL_NAMES from chatview.commands.search_context import ( @@ -46,17 +46,21 @@ def cmd_refresh(args): """ want_json = getattr(args, "json", False) force = getattr(args, "force", False) - if want_json: - # build_index() prints progress to stdout — suppress it so --json - # output stays pure JSON that json.loads() can consume. - old = sys.stdout - sys.stdout = io.StringIO() - try: + try: + if want_json: + # build_index() prints progress to stdout — suppress it so --json + # output stays pure JSON that json.loads() can consume. + old = sys.stdout + sys.stdout = io.StringIO() + try: + index = build_index(force=force) + finally: + sys.stdout = old + else: index = build_index(force=force) - finally: - sys.stdout = old - else: - index = build_index(force=force) + except IndexDatabaseLockedError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + raise SystemExit(2) from exc summary = { "sessions": len(index.get("sessions", {})), diff --git a/chatview/index.py b/chatview/index.py index fd53855..14176c6 100644 --- a/chatview/index.py +++ b/chatview/index.py @@ -8,7 +8,10 @@ import os import time import threading +import fcntl +import sqlite3 from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager from pathlib import Path from chatview.project_identity import PROJECT_IDENTITY_VERSION @@ -46,6 +49,19 @@ INDEX_STALE_CHECK_INTERVAL = float(os.environ.get("INDEX_STALE_CHECK_INTERVAL", "10")) +class IndexDatabaseLockedError(RuntimeError): + """A different process owns the SQLite writer lock during refresh.""" + + +def _raise_if_database_locked(exc: Exception) -> None: + if isinstance(exc, sqlite3.OperationalError) and "database is locked" in str(exc).lower(): + raise IndexDatabaseLockedError( + "Search index database is busy in another process. Wait for that " + "refresh to finish, or restart the long-running chat-view server so " + "it loads cross-process refresh locking, then retry." + ) from exc + + def _secure_index_cache(*, precreate_file: bool = False) -> None: CACHE_DIR.mkdir(parents=True, exist_ok=True, mode=0o700) os.chmod(CACHE_DIR, 0o700) @@ -60,6 +76,20 @@ def _secure_index_cache(*, precreate_file: bool = False) -> None: os.chmod(INDEX_CACHE, 0o600) +@contextmanager +def _process_build_lock(): + """Serialize index/database refreshes across app and CLI processes.""" + _secure_index_cache() + lock_path = CACHE_DIR / "index-build.lock" + with open(lock_path, "a+", encoding="utf-8") as lock_file: + os.chmod(lock_path, 0o600) + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + def _cached(key, compute_fn): """Return cached result if index hasn't changed, else compute and cache. @@ -239,6 +269,30 @@ def _db_session_coverage(_db) -> dict[str, dict]: return coverage +def _extract_live_metadata(extractor, file_path: str, expected_mtime: float): + """Retry once when a live session grows during extraction. + + If it keeps growing during the retry, retain the retry's starting mtime so + the next stale check schedules another refresh instead of treating a + possibly incomplete tail as current. + """ + meta = extractor(file_path) + try: + latest_mtime = os.path.getmtime(file_path) + except OSError: + return meta, expected_mtime + if latest_mtime == expected_mtime: + return meta, latest_mtime + + retry_mtime = latest_mtime + meta = extractor(file_path) + try: + final_mtime = os.path.getmtime(file_path) + except OSError: + return meta, retry_mtime + return meta, final_mtime if final_mtime == retry_mtime else retry_mtime + + def _has_db_session_coverage(meta: dict, coverage: dict | None, mtime: float) -> bool: if not coverage or coverage.get("insight_mtime") is None: return False @@ -269,7 +323,8 @@ def build_index(force: bool = False, known_files: dict = None) -> dict: _secure_index_cache() with _build_lock: - return _build_index_locked(force=force, known_files=known_files) + with _process_build_lock(): + return _build_index_locked(force=force, known_files=known_files) def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: @@ -352,12 +407,19 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: try: with ThreadPoolExecutor(max_workers=MAX_SEARCH_WORKERS) as pool: futures = { - pool.submit(extract_metadata, fp): (fp, pn) for fp, pn in to_parse + pool.submit( + _extract_live_metadata, + extract_metadata, + fp, + current_files.get(fp, 0), + ): (fp, pn) + for fp, pn in to_parse } for future in as_completed(futures): fp, pn = futures[future] try: - meta = future.result() + meta, indexed_mtime = future.result() + current_files[fp] = indexed_mtime if meta: meta["project"] = pn meta["projectName"] = pretty_project_name(pn) @@ -379,6 +441,7 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: if bulk_n % 50 == 0: _db.bulk_commit() except Exception as e: + _raise_if_database_locked(e) print(f"Error parsing {fp}: {e}") finally: _db.end_bulk() @@ -444,12 +507,19 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: try: with ThreadPoolExecutor(max_workers=MAX_SEARCH_WORKERS) as pool: futures = { - pool.submit(extract_codex_metadata, fp): fp for fp in codex_to_parse + pool.submit( + _extract_live_metadata, + extract_codex_metadata, + fp, + current_files.get(fp, 0), + ): fp + for fp in codex_to_parse } for future in as_completed(futures): fp = futures[future] try: - meta = future.result() + meta, indexed_mtime = future.result() + current_files[fp] = indexed_mtime if meta: meta["projectName"] = _codex_project_name(meta.get("cwd", "")) meta["project"] = "codex" @@ -470,6 +540,7 @@ def _build_index_locked(force: bool = False, known_files: dict = None) -> dict: if bulk_n % 50 == 0: _db.bulk_commit() except Exception as e: + _raise_if_database_locked(e) print(f"Error parsing Codex {fp}: {e}") finally: _db.end_bulk() @@ -639,6 +710,7 @@ def _parse_for_backfill(args): changed=bool(new_sessions or codex_new or pruned_count), ) except Exception as e: + _raise_if_database_locked(e) print(f"DB post-process error: {e}") db_count = _db.get_conn().execute("SELECT count(*) FROM sessions").fetchone()[0] diff --git a/tests/test_commands_analysis.py b/tests/test_commands_analysis.py index 9c44222..778ceb4 100644 --- a/tests/test_commands_analysis.py +++ b/tests/test_commands_analysis.py @@ -83,6 +83,24 @@ def test_exact_date_range_is_inclusive_and_overrides_shortcut(self): self.assertEqual(set(filtered), {"2026-07-01", "2026-07-02"}) +class TestCmdRefresh(unittest.TestCase): + def test_database_lock_is_reported_once_and_exits_nonzero(self): + from chatview.commands.analysis import cmd_refresh + from chatview.index import IndexDatabaseLockedError + + stderr = io.StringIO() + with patch( + "chatview.commands.analysis.build_index", + side_effect=IndexDatabaseLockedError("busy; retry"), + ), contextlib.redirect_stderr(stderr): + with self.assertRaises(SystemExit) as raised: + cmd_refresh(_default_args(force=False)) + + self.assertEqual(raised.exception.code, 2) + self.assertEqual(stderr.getvalue().count("ERROR:"), 1) + self.assertIn("busy; retry", stderr.getvalue()) + + class TestCmdStats(unittest.TestCase): """cmd_stats reads the DB and prints a statistics summary.""" diff --git a/tests/test_index_fts.py b/tests/test_index_fts.py index 1f0edfc..49f49c1 100644 --- a/tests/test_index_fts.py +++ b/tests/test_index_fts.py @@ -1,5 +1,6 @@ import json import os +import sqlite3 import shutil import tempfile import threading @@ -59,6 +60,34 @@ def test_unchanged_refresh_does_nothing(self): self.assertEqual(db.aggregate_count, 0) +class TestLiveSessionExtraction(unittest.TestCase): + def test_retries_once_when_file_grows_during_parse(self): + from chatview.index import _extract_live_metadata + + with tempfile.TemporaryDirectory() as tmpdir: + source = Path(tmpdir) / "live.jsonl" + source.write_text("first\n", encoding="utf-8") + initial_mtime = source.stat().st_mtime + calls = [] + + def extractor(file_path): + text = Path(file_path).read_text(encoding="utf-8") + calls.append(text) + if len(calls) == 1: + Path(file_path).write_text(text + "tail\n", encoding="utf-8") + bumped_ns = Path(file_path).stat().st_mtime_ns + 1_000_000 + os.utime(file_path, ns=(bumped_ns, bumped_ns)) + return {"text": text} + + meta, indexed_mtime = _extract_live_metadata( + extractor, str(source), initial_mtime + ) + + self.assertEqual(len(calls), 2) + self.assertEqual(meta["text"], "first\ntail\n") + self.assertEqual(indexed_mtime, source.stat().st_mtime) + + class TestBuildIndexMutex(unittest.TestCase): """Concurrent build_index calls should not double-parse JSONL files.""" @@ -173,6 +202,38 @@ def run(): self.assertLessEqual(count, 2, f"File parsed {count} times; Thread 2 should not add extra parse calls") + def test_process_build_lock_serializes_independent_callers(self): + """The file lock also serializes callers that do not share _build_lock.""" + from chatview import index as _idx + + first_entered = threading.Event() + release_first = threading.Event() + second_entered = threading.Event() + + def first(): + with _idx._process_build_lock(): + first_entered.set() + release_first.wait(timeout=5) + + def second(): + first_entered.wait(timeout=5) + with _idx._process_build_lock(): + second_entered.set() + + t1 = threading.Thread(target=first) + t2 = threading.Thread(target=second) + t1.start() + t2.start() + self.assertTrue(first_entered.wait(timeout=5)) + self.assertFalse(second_entered.wait(timeout=0.1)) + release_first.set() + t1.join(timeout=5) + t2.join(timeout=5) + + self.assertFalse(t1.is_alive()) + self.assertFalse(t2.is_alive()) + self.assertTrue(second_entered.is_set()) + def test_concurrent_force_build_actually_rebuilds(self): """A force=True call should rebuild even when another build just completed.""" from chatview import index as _idx @@ -197,6 +258,22 @@ def counting_extract(fp): self.assertGreaterEqual(parse_count["n"], 1, "force=True must trigger re-parse even if cache is fresh") + def test_database_lock_aborts_refresh_before_cache_write(self): + from chatview import index as _idx + + proj_dir = _idx.PROJECTS_DIR / "locked-proj" + self._write_minimal_jsonl(proj_dir) + + with unittest.mock.patch.object( + _idx, + "_write_session_bundle", + side_effect=sqlite3.OperationalError("database is locked"), + ): + with self.assertRaises(_idx.IndexDatabaseLockedError): + _idx.build_index() + + self.assertFalse(_idx.INDEX_CACHE.exists()) + class TestIndexCacheStripsMessageBodies(unittest.TestCase): """After build_index(), the on-disk index.json must not contain userTexts or From 7652ef63018314d0d028ef2801b151ddb4d09cdc Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sun, 19 Jul 2026 08:25:10 -0400 Subject: [PATCH 08/15] fix read-window access to read-only indexes --- chatview/commands/retrieval.py | 2 +- tests/test_retrieval_tools.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/chatview/commands/retrieval.py b/chatview/commands/retrieval.py index 5a33ec4..bc7892b 100644 --- a/chatview/commands/retrieval.py +++ b/chatview/commands/retrieval.py @@ -1030,7 +1030,7 @@ def _read_window_unbudgeted( from chatview import db as _db radius = _validated_window_radius(radius) - _db.init_db() + _db.prepare_search_db() meta = _db.get_session_meta(session_id) if not meta: raise KeyError(f"Session not found: {session_id}") diff --git a/tests/test_retrieval_tools.py b/tests/test_retrieval_tools.py index c4fee8f..f7ec8e9 100644 --- a/tests/test_retrieval_tools.py +++ b/tests/test_retrieval_tools.py @@ -906,6 +906,28 @@ def test_cjk_strong_evidence_requires_multiple_independent_topic_segments(self): class TestReadWindowData(RetrievalToolTestCase): + def test_existing_index_is_readable_without_cache_write_permission(self): + self._insert_session( + "readonly-window", + "Read-only window", + "distill-yourself", + [{"idx": 0, "text": "read-only evidence", "ts": "2026-07-01T10:00:00Z"}], + ) + dbcore._close_thread_connection() + self.addCleanup(dbcore._close_thread_connection) + + with patch.object( + dbcore, + "_secure_cache_paths", + side_effect=PermissionError("cache is not writable"), + ): + data = read_window_data("readonly-window", idx=0, radius=0) + + self.assertEqual( + [message["text"] for message in data["messages"]], + ["read-only evidence"], + ) + def test_returns_messages_around_requested_index(self): self._insert_session( "window-session", From 69c19b876f488fa773e49b7a1f67095afb0f0bc1 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sun, 19 Jul 2026 08:27:27 -0400 Subject: [PATCH 09/15] align Python support and close retrieval test DBs --- .github/workflows/test.yml | 4 +++- README.md | 2 +- chatview/index.py | 2 ++ docs/USER_GUIDE.md | 2 +- pyproject.toml | 2 +- tests/test_retrieval_tools.py | 1 + 6 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60e649e..edb6eb3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Syntax / import smoke check - run: python -m py_compile server.py db.py analyze.py + run: | + python -m py_compile server.py db.py analyze.py + python -m chatview.cli --help - name: Run test suite run: python -m unittest discover -s tests -v # 零依赖,无需 pip install diff --git a/README.md b/README.md index af475f6..2608df3 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ [News](#news) · [快速开始](#快速开始) · [功能特性](#功能特性) · [使用指南](docs/USER_GUIDE.md) · [API 参考](#rest-api) [![Homepage](https://img.shields.io/badge/首页-Distill_Yourself-0F766E?style=flat-square)](https://quantaalpha.com/Distill-Yourself/) -[![Python](https://img.shields.io/badge/Python-3.8+-3776AB?style=flat-square&logo=python&logoColor=white)](https://python.org) +[![Python](https://img.shields.io/badge/Python-3.9+-3776AB?style=flat-square&logo=python&logoColor=white)](https://python.org) [![Claude Code + Codex](https://img.shields.io/badge/数据源-Claude_Code_%2B_Codex-F97316?style=flat-square)](.) [![License](https://img.shields.io/badge/License-MIT-blue?style=flat-square)](LICENSE) diff --git a/chatview/index.py b/chatview/index.py index 14176c6..bb58171 100644 --- a/chatview/index.py +++ b/chatview/index.py @@ -4,6 +4,8 @@ build_index() / schedule_index_refresh_if_stale() for the rest of the app. """ +from __future__ import annotations + import json import os import time diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 7b8ee97..42e41e9 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -31,7 +31,7 @@ flowchart LR ## 一键安装 -Distill Yourself 零依赖：不需要 pip、npm 或 Docker，本机有 Python 3.8 及以上版本即可运行。 +Distill Yourself 零依赖：不需要 pip、npm 或 Docker，本机有 Python 3.9 及以上版本即可运行。 ```bash python3 server.py diff --git a/pyproject.toml b/pyproject.toml index 3535cbb..7a97811 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" description = "Search your Claude Code and Codex history. Distill durable lessons. Teach future sessions." readme = "README.md" license = "MIT" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [] [project.scripts] diff --git a/tests/test_retrieval_tools.py b/tests/test_retrieval_tools.py index f7ec8e9..285e218 100644 --- a/tests/test_retrieval_tools.py +++ b/tests/test_retrieval_tools.py @@ -37,6 +37,7 @@ def setUp(self): db.init_db() def tearDown(self): + dbcore._close_thread_connection() dbcore.DB_PATH = self._orig_db_path dbcore.CACHE_DIR = self._orig_cache_dir dbcore._local = threading.local() From 7513fb69f1a2d1ef32f7d3288847fa09cd9b7e5d Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sun, 19 Jul 2026 08:38:07 -0400 Subject: [PATCH 10/15] chore ignore verification artifacts --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7c127e7..12772bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .visual-ask/ .DS_Store .cache/ +.verify/ .claude/ .knowhow/ .superpowers/ From 9faff9a13b912f63f55e0abfe0fb1cea5aa93b65 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sun, 19 Jul 2026 09:40:15 -0400 Subject: [PATCH 11/15] fix weak assistant search matches --- chatview/search.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/chatview/search.py b/chatview/search.py index 0fc431a..60cdc00 100644 --- a/chatview/search.py +++ b/chatview/search.py @@ -243,6 +243,10 @@ def _append_anchor_content_matches( if key in seen: continue text = row["text"] or "" + if row["role"] == "assistant" and not _assistant_partial_match_allowed( + text, query_lower + ): + continue if min_coverage and _query_coverage(text, query_lower) < min_coverage: continue score = _content_score(text, query_lower, row["role"] or "") @@ -303,6 +307,8 @@ def _append_adjacent_assistant_matches( if key in seen: continue assistant_text = assistant["text"] or "" + if not _assistant_partial_match_allowed(assistant_text, query_lower): + continue if _query_coverage(assistant_text, query_lower) <= 0: continue combined = f"{user_row['text']}\n{assistant_text}" @@ -419,6 +425,14 @@ def _fuzzy_match_content(text_lower: str, query_lower: str, tokens: list): return False, 0 +def _assistant_partial_match_allowed(text: str, query_lower: str) -> bool: + tokens = _tokenize_query(query_lower) + if len(tokens) < 3: + return True + matched, _ = _fuzzy_match_content(text.casefold(), query_lower, tokens) + return matched + + def _is_identifier_anchor(token: str) -> bool: if re.search(r"[_0-9]", token): return len(token) >= 8 From 6c9841ce0c95ee484a93a964adbe583d3552d4de Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sun, 19 Jul 2026 11:45:35 -0400 Subject: [PATCH 12/15] feat: add composable bounded history windows --- chatview/cli.py | 37 ++- chatview/commands/retrieval.py | 421 ++++++++++++++++++++++------ chatview/commands/search_context.py | 39 ++- chatview/db/__init__.py | 4 + chatview/db/core.py | 9 + chatview/db/sessions.py | 61 +++- chatview/handlers/twin.py | 4 +- chatview/parsers/claude.py | 8 +- chatview/parsers/codex.py | 8 +- tests/test_assistant_full_index.py | 38 +++ tests/test_commands_analysis.py | 14 +- tests/test_retrieval_tools.py | 196 +++++++++---- 12 files changed, 679 insertions(+), 160 deletions(-) diff --git a/chatview/cli.py b/chatview/cli.py index e575370..b029696 100644 --- a/chatview/cli.py +++ b/chatview/cli.py @@ -165,7 +165,10 @@ def main(): "-A", "--after", type=int, help="Show N messages after each match" ) p_search.add_argument( - "-n", "--line-number", action="store_true", help="Prefix context lines with session:idx:role" + "-n", + "--line-number", + action="store_true", + help="Prefix context lines with explicit session, idx, and role labels", ) p_search.add_argument( "--format", choices=("standard", "lines", "jsonl"), default="standard", @@ -184,20 +187,44 @@ def main(): p_read_window = sub.add_parser( "read-window", parents=[shared], - help="Read a small message window around a match index", + help="Read a composable bounded message window around a match index", ) p_read_window.add_argument("session", nargs="?", help="Exact session ID") p_read_window.add_argument("--idx", type=int, help="Target message index") p_read_window.add_argument( - "--radius", + "-B", + "--before", type=int, default=2, - help="How many message indexes before/after to include", + help="Actual indexed messages before the target (default: 2, max: 20)", + ) + p_read_window.add_argument( + "-A", + "--after", + type=int, + default=2, + help="Actual indexed messages after the target (default: 2, max: 20)", + ) + p_read_window.add_argument( + "--include", + default="user:2000,assistant:600", + help=( + "Role projections as role:max-chars pairs " + "(default: user:2000,assistant:600)" + ), + ) + p_read_window.add_argument( + "--around", + default="", + help="Center the target projection on one unique exact string", ) p_read_window.add_argument( "--batch", default="", - help='JSON list of {"session": "...", "idx": N, "radius": N} items', + help=( + 'JSON list of {"sessionId":"...","idx":N,"before":N,' + '"after":N,"include":"user:2000,assistant:600"} items' + ), ) p_tool_search = sub.add_parser( diff --git a/chatview/commands/retrieval.py b/chatview/commands/retrieval.py index bc7892b..8bf0b03 100644 --- a/chatview/commands/retrieval.py +++ b/chatview/commands/retrieval.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os import re import sqlite3 from datetime import datetime @@ -969,34 +970,157 @@ def find_repeats_data(query: str, args) -> dict: } -_READ_WINDOW_MAX_RADIUS = 5 +_READ_WINDOW_MAX_CONTEXT = 20 _READ_WINDOW_MAX_BATCH = 5 -_READ_WINDOW_MESSAGE_CHARS = 1200 _READ_WINDOW_OUTPUT_BYTES = 20000 +_READ_WINDOW_DEFAULT_INCLUDE = {"user": 2000, "assistant": 600} +_READ_WINDOW_MAX_ROLE_CHARS = 2000 -def _validated_window_radius(radius: int) -> int: - if isinstance(radius, bool) or isinstance(radius, float): - raise ValueError("radius must be between 0 and 5") - if isinstance(radius, int): - value = radius - elif isinstance(radius, str) and re.fullmatch(r"[+-]?\d+", radius.strip()): - value = int(radius) +def _validated_context_count(value, name: str) -> int: + if isinstance(value, bool) or isinstance(value, float): + raise ValueError(f"{name} must be between 0 and 20") + if isinstance(value, int): + parsed = value + elif isinstance(value, str) and re.fullmatch(r"[+-]?\d+", value.strip()): + parsed = int(value) else: - raise ValueError("radius must be between 0 and 5") - if not 0 <= value <= _READ_WINDOW_MAX_RADIUS: - raise ValueError("radius must be between 0 and 5") - return value + raise ValueError(f"{name} must be between 0 and 20") + if not 0 <= parsed <= _READ_WINDOW_MAX_CONTEXT: + raise ValueError(f"{name} must be between 0 and 20") + return parsed + + +def _normalize_window_include(value=None) -> dict[str, int]: + if value is None or value == "": + return dict(_READ_WINDOW_DEFAULT_INCLUDE) + if isinstance(value, str): + pairs = [] + for part in value.split(","): + if ":" not in part: + raise ValueError("--include must use role:max-chars pairs") + role, chars = part.split(":", 1) + pairs.append((role.strip(), chars.strip())) + elif isinstance(value, dict): + pairs = list(value.items()) + else: + raise ValueError("include must be a role:max-chars string or object") + + include = {} + for role, raw_chars in pairs: + if role not in ("user", "assistant"): + raise ValueError(f"Unknown include role: {role}") + if role in include: + raise ValueError(f"Duplicate include role: {role}") + if isinstance(raw_chars, bool): + raise ValueError(f"Character budget for {role} must be between 1 and 2000") + try: + chars = int(raw_chars) + except (TypeError, ValueError) as exc: + raise ValueError( + f"Character budget for {role} must be between 1 and 2000" + ) from exc + if not 1 <= chars <= _READ_WINDOW_MAX_ROLE_CHARS: + raise ValueError(f"Character budget for {role} must be between 1 and 2000") + include[role] = chars + if not include: + raise ValueError("include must select at least one role") + return {role: include[role] for role in ("user", "assistant") if role in include} + + +def _exact_match_positions(text: str, anchor: str) -> list[int]: + positions = [] + start = 0 + while True: + found = text.find(anchor, start) + if found < 0: + return positions + positions.append(found) + start = found + 1 + + +def _project_window_text(text: str, max_chars: int, around: str = "") -> dict: + value = text or "" + match_start = None + match_count = 0 + if around: + positions = _exact_match_positions(value, around) + match_count = len(positions) + if match_count == 0: + raise ValueError("--around text was not found in the target message") + if match_count > 1: + raise ValueError( + f"--around text matched {match_count} times; use a longer unique anchor" + ) + if len(around) > max_chars: + raise ValueError("--around text is longer than the target character budget") + match_start = positions[0] + spare = max_chars - len(around) + start = max(0, match_start - spare // 2) + end = min(len(value), start + max_chars) + start = max(0, end - max_chars) + else: + start = 0 + end = min(len(value), max_chars) + return { + "text": value[start:end], + "sliceStart": start, + "sliceEnd": end, + "matchStart": match_start, + "matchCount": match_count, + "shownChars": end - start, + "truncated": start > 0 or end < len(value), + } -def _truncate_window_text( - text: str, max_chars: int = _READ_WINDOW_MESSAGE_CHARS -) -> tuple[str, bool]: - value = text or "" - if len(value) <= max_chars: - return value, False - marker = f"\n[… truncated from {len(value)} chars]" - return value[:max(max_chars - len(marker), 0)] + marker[:max_chars], True +def _public_window_message( + message: dict, + include: dict[str, int], + *, + around: str = "", + budget_cap: int | None = None, +) -> dict | None: + role = message.get("role", "") + if role not in include: + return None + requested_chars = include[role] + effective_chars = ( + min(requested_chars, budget_cap) + if budget_cap is not None + else requested_chars + ) + if around and effective_chars < len(around): + raise ValueError("20 KB output budget cannot preserve the complete --around anchor") + source_text = message.get("text", "") or "" + projection = _project_window_text(source_text, effective_chars, around) + source_complete_raw = message.get("source_complete") + source_complete = ( + bool(source_complete_raw) if source_complete_raw is not None else None + ) + truncation_reason = None + if projection["truncated"]: + truncation_reason = ( + "output_budget" + if budget_cap is not None and effective_chars < requested_chars + else "role_limit" + ) + return { + "idx": message.get("idx"), + "role": role, + "ts": message.get("ts", ""), + "text": projection["text"], + "requestedChars": requested_chars, + "indexedChars": len(source_text), + "sourceChars": message.get("source_chars"), + "sourceComplete": source_complete, + "shownChars": projection["shownChars"], + "truncated": projection["truncated"], + "truncationReason": truncation_reason, + "sliceStart": projection["sliceStart"], + "sliceEnd": projection["sliceEnd"], + "matchStart": projection["matchStart"], + "matchCount": projection["matchCount"], + } def _window_json(data: dict) -> str: @@ -1013,45 +1137,90 @@ def _window_json(data: dict) -> str: def _refresh_window_budget_metadata(data: dict, windows: list[dict]): for window in windows: window["outputTruncated"] = bool( - window["omittedMessages"] - or any(message["outputTruncated"] for message in window["messages"]) + window["budgetOmittedMessages"] + or any(message["truncated"] for message in window["messages"]) ) - data["omittedMessages"] = sum(window["omittedMessages"] for window in windows) + data["budgetOmittedMessages"] = sum( + window["budgetOmittedMessages"] for window in windows + ) data["outputTruncated"] = bool( - data["omittedMessages"] or any(window["outputTruncated"] for window in windows) + data["budgetOmittedMessages"] + or any(window["outputTruncated"] for window in windows) ) _window_json(data) +def _window_freshness(meta: dict) -> dict: + file_path = meta.get("file_path") or "" + indexed_mtime = meta.get("file_mtime") + if not file_path or indexed_mtime is None: + return { + "indexState": "unknown", + "indexedFileMtime": indexed_mtime, + "sourceFileMtime": None, + } + try: + source_mtime = os.path.getmtime(file_path) + except OSError: + return { + "indexState": "source_missing", + "indexedFileMtime": indexed_mtime, + "sourceFileMtime": None, + } + return { + "indexState": "fresh" if float(indexed_mtime) == float(source_mtime) else "stale", + "indexedFileMtime": indexed_mtime, + "sourceFileMtime": source_mtime, + } + + def _read_window_unbudgeted( - session_id: str, idx: int, radius: int -) -> tuple[dict, list[tuple[dict, str]]]: - """Load one DB window and return public message projections plus source text.""" + session_id: str, + idx: int, + before: int, + after: int, + include=None, + around: str = "", +) -> tuple[dict, list[tuple[dict, str, int]]]: + """Load one exact target and its actual neighboring indexed messages.""" from chatview import db as _db - radius = _validated_window_radius(radius) + before = _validated_context_count(before, "before") + after = _validated_context_count(after, "after") + effective_include = _normalize_window_include(include) _db.prepare_search_db() meta = _db.get_session_meta(session_id) if not meta: raise KeyError(f"Session not found: {session_id}") sid = meta["id"] - start = idx - radius - end = idx + radius - messages = [] + target = _db.get_message_at_index(sid, idx) + if not target: + raise KeyError(f"Message not found: {sid} idx {idx}") + if around and target.get("role", "") not in effective_include: + raise ValueError("--around requires the target role to be included") + selected = _db.get_message_neighbors(sid, idx, before, after) + [target] + selected.sort(key=lambda message: (message.get("idx", 0), message.get("id", 0))) candidates = [] - for message in _db.get_message_window(sid, start, end): - source_text = message.get("text", "") or "" - text, truncated = _truncate_window_text(source_text) - public = { - "idx": message.get("idx"), - "role": message.get("role", ""), - "ts": message.get("ts", ""), - "text": text, - "originalChars": len(source_text), - "outputTruncated": truncated, - } - messages.append(public) - candidates.append((public, source_text)) + filtered_messages = 0 + target_position = next( + position for position, message in enumerate(selected) if message.get("idx") == idx + ) + for position, message in enumerate(selected): + target_around = around if message.get("idx") == idx else "" + public = _public_window_message( + message, effective_include, around=target_around + ) + if public is None: + filtered_messages += 1 + continue + candidates.append( + ( + abs(position - target_position), + message, + target_around, + effective_include[message.get("role", "")], + ) + ) window = { "sessionId": sid, "title": meta.get("title") or "Untitled", @@ -1059,27 +1228,40 @@ def _read_window_unbudgeted( "source": meta.get("source") or "", "date": meta.get("date") or "", "targetIndex": idx, - "radius": radius, - "messages": messages, - "omittedMessages": 0, - "outputTruncated": any(message["outputTruncated"] for message in messages), + "coordinateScope": "current-session-index-snapshot", + "before": before, + "after": after, + "effectiveInclude": effective_include, + "around": around, + "anchor": { + "idx": idx, + "role": target.get("role", ""), + "ts": target.get("ts", ""), + "included": target.get("role", "") in effective_include, + "bodyStatus": ( + "included" if target.get("role", "") in effective_include else "filtered" + ), + }, + **_window_freshness(meta), + "selectedMessages": len(selected), + "filteredMessages": filtered_messages, + "messages": [], + "budgetOmittedMessages": 0, + "outputTruncated": False, } return window, candidates -def _budget_read_windows( - raw_windows: list[tuple[dict, list[tuple[dict, str]]]], batch: bool -) -> dict: +def _budget_read_windows(raw_windows: list[tuple[dict, list]], batch: bool) -> dict: """Apply one output budget, preserving every target before surrounding context.""" windows = [window for window, _ in raw_windows] targets = [] contexts = [] for window_index, (window, window_candidates) in enumerate(raw_windows): window["messages"] = [] - window["omittedMessages"] = len(window_candidates) - for message, source_text in window_candidates: - distance = abs(int(message["idx"]) - int(window["targetIndex"])) - candidate = (distance, window_index, message, source_text) + window["budgetOmittedMessages"] = len(window_candidates) + for distance, message, around, requested_cap in window_candidates: + candidate = (distance, window_index, message, around, requested_cap) (targets if distance == 0 else contexts).append(candidate) if batch: @@ -1087,7 +1269,7 @@ def _budget_read_windows( "windows": windows, "outputBytes": 0, "outputTruncated": False, - "omittedMessages": 0, + "budgetOmittedMessages": 0, } else: data = windows[0] @@ -1096,21 +1278,23 @@ def _budget_read_windows( def place_all_targets(max_chars: int) -> bool: for window_index, (_window, window_candidates) in enumerate(raw_windows): windows[window_index]["messages"] = [] - windows[window_index]["omittedMessages"] = len(window_candidates) - for _distance, window_index, message, source_text in targets: - candidate = dict(message) - candidate["text"], candidate["outputTruncated"] = _truncate_window_text( - source_text, max_chars + windows[window_index]["budgetOmittedMessages"] = len(window_candidates) + for _distance, window_index, message, around, _requested_cap in targets: + candidate = _public_window_message( + message, + windows[window_index]["effectiveInclude"], + around=around, + budget_cap=max_chars, ) windows[window_index]["messages"].append(candidate) - windows[window_index]["omittedMessages"] -= 1 + windows[window_index]["budgetOmittedMessages"] -= 1 _refresh_window_budget_metadata(data, windows) return data["outputBytes"] <= _READ_WINDOW_OUTPUT_BYTES # Find one common cap so no earlier window can consume another target's # share of the budget. Short targets remain complete; long targets are fair. - low = 0 - high = _READ_WINDOW_MESSAGE_CHARS + low = max((len(item[3]) for item in targets), default=0) + high = _READ_WINDOW_MAX_ROLE_CHARS target_cap = None while low <= high: middle = (low + high) // 2 @@ -1123,17 +1307,20 @@ def place_all_targets(max_chars: int) -> bool: raise ValueError("read-window metadata exceeds the 20000-byte output budget") place_all_targets(target_cap) - for _distance, window_index, message, _source_text in sorted( + for _distance, window_index, message, around, _requested_cap in sorted( contexts, key=lambda item: (item[0], item[1], item[2]["idx"]), ): window = windows[window_index] - window["messages"].append(message) - window["omittedMessages"] -= 1 + candidate = _public_window_message( + message, window["effectiveInclude"], around=around + ) + window["messages"].append(candidate) + window["budgetOmittedMessages"] -= 1 _refresh_window_budget_metadata(data, windows) if data["outputBytes"] > _READ_WINDOW_OUTPUT_BYTES: window["messages"].pop() - window["omittedMessages"] += 1 + window["budgetOmittedMessages"] += 1 _refresh_window_budget_metadata(data, windows) # A later context item may have a shorter projection and still fit. continue @@ -1144,9 +1331,18 @@ def place_all_targets(max_chars: int) -> bool: return data -def read_window_data(session_id: str, idx: int, radius: int = 2) -> dict: - """Return bounded DB-backed message context around a message index.""" - raw = _read_window_unbudgeted(session_id, idx, radius) +def read_window_data( + session_id: str, + idx: int, + before: int = 2, + after: int = 2, + include=None, + around: str = "", +) -> dict: + """Return bounded DB-backed context around one exact message coordinate.""" + raw = _read_window_unbudgeted( + session_id, idx, before, after, include=include, around=around + ) return _budget_read_windows([raw], batch=False) @@ -1157,13 +1353,34 @@ def read_windows_data(requests: list) -> dict: raise ValueError("--batch accepts at most 5 items") windows = [] for i, request in enumerate(requests): + unknown = set(request) - { + "session", + "sessionId", + "idx", + "before", + "after", + "include", + "around", + } + if unknown: + raise ValueError( + f"Batch item {i} has unknown fields: {', '.join(sorted(unknown))}" + ) session = request.get("session") or request.get("sessionId") if not session: raise ValueError(f"Batch item {i} missing session") if "idx" not in request: raise ValueError(f"Batch item {i} missing idx") - radius = request.get("radius", 2) - windows.append(_read_window_unbudgeted(session, int(request["idx"]), radius)) + windows.append( + _read_window_unbudgeted( + session, + int(request["idx"]), + request.get("before", 2), + request.get("after", 2), + include=request.get("include"), + around=request.get("around", ""), + ) + ) return _budget_read_windows(windows, batch=True) @@ -1272,24 +1489,61 @@ def _human_windows_output(data: dict) -> str: windows = data.get("windows") if "windows" in data else [data] parts = [] for window in windows: + include_text = ",".join( + f"{role}:{chars}" for role, chars in window["effectiveInclude"].items() + ) parts.append(f"# {window['title']}\n") parts.append( f"# {window['project']} | target idx:{window['targetIndex']} " - f"radius:{window['radius']}\n\n" + f"before:{window['before']} after:{window['after']} " + f"include:{include_text}\n" + ) + parts.append( + f"# indexState={window['indexState']} " + f"coordinateScope={window['coordinateScope']}\n\n" ) + if window["indexState"] == "stale": + parts.append( + "⟪STALE INDEX · run `distill refresh` before claiming recent " + "content is absent⟫\n\n" + ) + if not window["anchor"]["included"]: + parts.append( + f"[session={window['sessionId']} | idx={window['anchor']['idx']} | " + f"role={window['anchor']['role']}]\n" + "⟪FILTERED · target body excluded by --include⟫\n\n" + ) for message in window["messages"]: parts.append( - f"--- {message.get('role', '').upper()} idx:{message.get('idx')} " - f"{message.get('ts', '')[:16]} ---\n" + f"[session={window['sessionId']} | idx={message.get('idx')} | " + f"role={message.get('role', '')}] " + f"{message.get('ts', '')[:16]}\n" + ) + parts.append((message.get("text") or "") + "\n") + if message["truncated"]: + parts.append( + f"⟪TRUNCATED · {message['truncationReason']} · shown " + f"{message['shownChars']} of {message['indexedChars']} indexed chars⟫\n" + ) + if message["sourceComplete"] is None: + parts.append( + "⟪SOURCE COMPLETENESS UNKNOWN · legacy indexed message⟫\n" + ) + elif message["sourceComplete"] is False: + parts.append("⟪INDEX LIMIT · source message is not fully indexed⟫\n") + parts.append("\n") + if window["budgetOmittedMessages"]: + parts.append( + f"⟪OMITTED · {window['budgetOmittedMessages']} surrounding messages " + "excluded by the 20 KB output budget⟫\n\n" ) - parts.append((message.get("text") or "") + "\n\n") body = "".join(parts) output_bytes = 0 for _ in range(8): summary = ( f"# outputBytes={output_bytes} " f"outputTruncated={str(bool(data['outputTruncated'])).lower()} " - f"omittedMessages={data['omittedMessages']}\n" + f"budgetOmittedMessages={data['budgetOmittedMessages']}\n" ) rendered = body + summary size = len(rendered.encode("utf-8")) @@ -1316,7 +1570,14 @@ def cmd_read_window(args): if not args.session or args.idx is None: print("read-window requires SESSION and --idx, or --batch JSON") return - data = read_window_data(args.session, args.idx, args.radius) + data = read_window_data( + args.session, + args.idx, + getattr(args, "before", 2), + getattr(args, "after", 2), + include=getattr(args, "include", None), + around=getattr(args, "around", ""), + ) if args.json: print(_window_json(data), end="") return diff --git a/chatview/commands/search_context.py b/chatview/commands/search_context.py index 824ae78..b037046 100644 --- a/chatview/commands/search_context.py +++ b/chatview/commands/search_context.py @@ -24,7 +24,13 @@ def format_grep_window(session_id, idx, args): from chatview import db as _db before, after = grep_bounds(args) - messages = _db.get_message_window(session_id, idx - before, idx + after) + target = _db.get_message_at_index(session_id, idx) + if not target: + return [] + messages = ( + _db.get_message_neighbors(session_id, idx, before, after) + [target] + ) + messages.sort(key=lambda message: (message.get("idx", 0), message.get("id", 0))) lines = [] for msg in messages: role = msg.get("role", "") @@ -36,7 +42,9 @@ def format_grep_window(session_id, idx, args): ) text = " ".join(data["snippet"].strip().split()) if getattr(args, "line_number", False): - lines.append(f"{session_id}:{msg_idx}:{role}: {text}") + lines.append( + f"[session={session_id} | idx={msg_idx} | role={role}] {text}" + ) else: lines.append(f" {role} idx:{msg_idx}: {text}") return lines @@ -54,9 +62,14 @@ def attach_search_context(results: list, args) -> list: for row in results: if row.get("candidateType", "message") != "message" or row.get("idx") is None: continue - messages = _db.get_message_window( - row.get("sessionId", ""), row["idx"] - before, row["idx"] + after - ) + session_id = row.get("sessionId", "") + target = _db.get_message_at_index(session_id, row["idx"]) + if not target: + continue + messages = _db.get_message_neighbors( + session_id, row["idx"], before, after + ) + [target] + messages.sort(key=lambda message: (message.get("idx", 0), message.get("id", 0))) context = [] for msg in messages: snippet_data = make_query_snippet( @@ -95,7 +108,16 @@ def format_search_lines(results: list, args) -> list: }] else: messages = [] - for msg in _db.get_message_window(session_id, idx - before, idx + after): + target = _db.get_message_at_index(session_id, idx) + if not target: + continue + selected = _db.get_message_neighbors( + session_id, idx, before, after + ) + [target] + selected.sort( + key=lambda message: (message.get("idx", 0), message.get("id", 0)) + ) + for msg in selected: data = make_query_snippet( msg.get("text") or "", query, max_chars=max_chars ) @@ -110,5 +132,8 @@ def format_search_lines(results: list, args) -> list: continue seen.add(key) text = " ".join((msg.get("snippet") or "").strip().split()) - lines.append(f"{session_id}:{msg.get('idx')}:{msg.get('role', '')}: {text}") + lines.append( + f"[session={session_id} | idx={msg.get('idx')} | " + f"role={msg.get('role', '')}] {text}" + ) return lines diff --git a/chatview/db/__init__.py b/chatview/db/__init__.py index 25be581..38444e0 100644 --- a/chatview/db/__init__.py +++ b/chatview/db/__init__.py @@ -30,6 +30,8 @@ get_session_by_partial_id, get_session_messages, get_message_window, + get_message_at_index, + get_message_neighbors, verify_fts_integrity, ) from .tool_calls import ( @@ -142,6 +144,8 @@ "get_session_by_partial_id", "get_session_messages", "get_message_window", + "get_message_at_index", + "get_message_neighbors", "verify_fts_integrity", "replace_tool_calls", "search_tool_calls", diff --git a/chatview/db/core.py b/chatview/db/core.py index 7c09c02..d1c69ab 100644 --- a/chatview/db/core.py +++ b/chatview/db/core.py @@ -264,6 +264,9 @@ def query_in_chunks( "project_identity_version", "source", "starred", + # messages + "source_chars", + "source_complete", # tool_calls "event_idx", "line_number", @@ -430,6 +433,8 @@ def init_db(): role TEXT, text TEXT, ts TEXT, + source_chars INTEGER, + source_complete INTEGER, FOREIGN KEY (session_id) REFERENCES sessions(id) ); CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id); @@ -805,6 +810,10 @@ def init_db(): _ensure_column(conn, "sessions", "project_key", "TEXT") _ensure_column(conn, "sessions", "project_display", "TEXT") _ensure_column(conn, "sessions", "project_identity_version", "INTEGER") + # Existing message rows intentionally remain NULL/unknown. New or changed + # sessions populate these fields without forcing a full history re-index. + _ensure_column(conn, "messages", "source_chars", "INTEGER") + _ensure_column(conn, "messages", "source_complete", "INTEGER") conn.execute( "UPDATE sessions SET project_key=project_name WHERE COALESCE(project_key, '')=''" ) diff --git a/chatview/db/sessions.py b/chatview/db/sessions.py index e651221..e155418 100644 --- a/chatview/db/sessions.py +++ b/chatview/db/sessions.py @@ -18,7 +18,8 @@ def upsert_session(meta: dict, user_texts: list, assistant_snippets: list): meta keys: id, title, date, lastDate, filePath, fileSize, _mtime, userMessageCount, preview, project, projectName, projectKey, projectDisplay, source - user_texts / assistant_snippets: list of {idx, text, ts} + user_texts / assistant_snippets: list of + {idx, text, ts, sourceChars?, sourceComplete?} """ conn = get_conn() sid = meta["id"] @@ -94,7 +95,19 @@ def upsert_session(meta: dict, user_texts: list, assistant_snippets: list): rows = [] for item in user_texts: rows.append( - (sid, item.get("idx"), "user", item.get("text", ""), item.get("ts", "")) + ( + sid, + item.get("idx"), + "user", + item.get("text", ""), + item.get("ts", ""), + item.get("sourceChars"), + ( + 1 + if item.get("sourceComplete") is True + else 0 if item.get("sourceComplete") is False else None + ), + ) ) for item in assistant_snippets: rows.append( @@ -104,12 +117,20 @@ def upsert_session(meta: dict, user_texts: list, assistant_snippets: list): "assistant", item.get("text", ""), item.get("ts", ""), + item.get("sourceChars", len(item.get("text", "") or "")), + ( + 1 + if item.get("sourceComplete", True) is True + else 0 if item.get("sourceComplete") is False else None + ), ) ) if rows: conn.executemany( - "INSERT INTO messages (session_id, idx, role, text, ts) VALUES (?,?,?,?,?)", + """INSERT INTO messages + (session_id, idx, role, text, ts, source_chars, source_complete) + VALUES (?,?,?,?,?,?,?)""", rows, ) # Sync FTS @@ -566,3 +587,37 @@ def get_message_window(session_id: str, start_idx: int, end_idx: int) -> list: (session_id, start_idx, end_idx), ).fetchall() return [dict(r) for r in rows] + + +def get_message_at_index(session_id: str, idx: int) -> dict | None: + """Return one exact message coordinate, rejecting ambiguous old data.""" + conn = get_conn() + rows = conn.execute( + """SELECT * FROM messages + WHERE session_id=? AND idx=? + ORDER BY id LIMIT 2""", + (session_id, idx), + ).fetchall() + if len(rows) > 1: + raise ValueError(f"Ambiguous message coordinate: {session_id} idx {idx}") + return dict(rows[0]) if rows else None + + +def get_message_neighbors( + session_id: str, idx: int, before: int, after: int +) -> list: + """Return actual neighboring message rows, independent of sparse idx gaps.""" + conn = get_conn() + left = conn.execute( + """SELECT * FROM messages + WHERE session_id=? AND idx < ? + ORDER BY idx DESC, id DESC LIMIT ?""", + (session_id, idx, before), + ).fetchall() + right = conn.execute( + """SELECT * FROM messages + WHERE session_id=? AND idx > ? + ORDER BY idx ASC, id ASC LIMIT ?""", + (session_id, idx, after), + ).fetchall() + return [dict(row) for row in reversed(left)] + [dict(row) for row in right] diff --git a/chatview/handlers/twin.py b/chatview/handlers/twin.py index d787a8f..9027ce3 100644 --- a/chatview/handlers/twin.py +++ b/chatview/handlers/twin.py @@ -740,11 +740,11 @@ def _build_twin_stage1_prompt( # Task 1. First, run `python3 {cli_path} twin-events --limit 1000 --json` to see ALL existing events. The default limit=50 will miss records（默认 limit=50 会漏）. Check what's already been captured — avoid duplicates. -2. Run `python3 {cli_path} corrections --limit 100` to get correction events. Each result includes a stable `idx` and the nearest user/assistant pair in `conversation`. +2. Run `python3 {cli_path} corrections --limit 100` to get correction events. Each result includes a session-scoped `idx` and the nearest user/assistant pair in `conversation`. 3. Run `python3 {cli_path} highlights --limit 20` to collect high-signal candidate sessions and summaries. 4. Also run `python3 {cli_path} queries --limit 50` and look for acceptance patterns — cases where the user did NOT correct the AI (positive signals). 5. From corrections/highlights/queries summaries, collect candidate new events. For each candidate, first run `python3 {cli_path} twin-search events --q "keyword" --json` using the strongest keywords from the candidate; discard candidates already covered by existing events. -6. 只对查重后幸存、且仍需要补充上下文的候选做深读。For correction candidates, run `python3 {cli_path} read-window --idx --radius 2`; for candidates without `idx`, run `python3 {cli_path} read -s`. 查重前不深读任何 session; do not deep-read any session before keyword deduplication. +6. 只对查重后幸存、且仍需要补充上下文的候选做深读。For correction candidates, run `python3 {cli_path} read-window --idx `; for candidates without `idx`, run `python3 {cli_path} read -s`. 查重前不深读任何 session; do not deep-read any session before keyword deduplication. From these, extract new evidence events. Compare with existing events — if an event already exists for the same session and similar situation, use `twin-edit` to update/enrich it. Only `twin-add` genuinely new events. diff --git a/chatview/parsers/claude.py b/chatview/parsers/claude.py index 24a08d7..824a3d4 100644 --- a/chatview/parsers/claude.py +++ b/chatview/parsers/claude.py @@ -122,7 +122,13 @@ def extract_metadata(filepath: str): text = _extract_user_text(raw_content) if text.strip(): user_texts.append( - {"idx": msg_index, "text": text[:2000], "ts": ts} + { + "idx": msg_index, + "text": text, + "ts": ts, + "sourceChars": len(text), + "sourceComplete": True, + } ) _prev_user_msg = text[:200] if text.strip() else _prev_user_msg msg_index += 1 diff --git a/chatview/parsers/codex.py b/chatview/parsers/codex.py index ef3ce91..0012078 100644 --- a/chatview/parsers/codex.py +++ b/chatview/parsers/codex.py @@ -167,7 +167,13 @@ def extract_codex_metadata(filepath: str): text = payload.get("message", "") if text.strip(): user_texts.append( - {"idx": msg_index, "text": text[:2000], "ts": ts} + { + "idx": msg_index, + "text": text, + "ts": ts, + "sourceChars": len(text), + "sourceComplete": True, + } ) msg_index += 1 diff --git a/tests/test_assistant_full_index.py b/tests/test_assistant_full_index.py index 91a229b..e45962e 100644 --- a/tests/test_assistant_full_index.py +++ b/tests/test_assistant_full_index.py @@ -38,6 +38,22 @@ def test_claude_metadata_keeps_assistant_text_past_300_chars(self): self.assertIn(tail, meta["assistantSnippets"][0]["text"]) + def test_claude_metadata_keeps_complete_user_text_past_2000_chars(self): + tail = "user-tail-after-2000-claude" + long_message = "用" * 2100 + tail + path = self._write_jsonl([{ + "type": "user", + "sessionId": "claude-full-user", + "timestamp": "2026-07-08T00:00:00Z", + "message": {"content": [{"type": "text", "text": long_message}]}, + }]) + + meta = extract_metadata(path) + + self.assertIn(tail, meta["userTexts"][0]["text"]) + self.assertEqual(meta["userTexts"][0]["sourceChars"], len(long_message)) + self.assertTrue(meta["userTexts"][0]["sourceComplete"]) + def test_claude_metadata_keeps_all_assistant_text_blocks(self): second_block_token = "assistant-second-block-token-claude" path = self._write_jsonl([ @@ -93,6 +109,28 @@ def test_codex_metadata_keeps_assistant_text_past_300_chars(self): self.assertIn(tail, meta["assistantSnippets"][0]["text"]) + def test_codex_metadata_keeps_complete_user_text_past_2000_chars(self): + tail = "user-tail-after-2000-codex" + long_message = "用" * 2100 + tail + path = self._write_jsonl([ + { + "type": "session_meta", + "timestamp": "2026-07-08T00:00:00Z", + "payload": {"id": "codex-full-user", "cwd": "/tmp/project"}, + }, + { + "type": "event_msg", + "timestamp": "2026-07-08T00:00:01Z", + "payload": {"type": "user_message", "message": long_message}, + }, + ]) + + meta = extract_codex_metadata(path) + + self.assertIn(tail, meta["userTexts"][0]["text"]) + self.assertEqual(meta["userTexts"][0]["sourceChars"], len(long_message)) + self.assertTrue(meta["userTexts"][0]["sourceComplete"]) + def test_codex_metadata_keeps_all_assistant_text_blocks(self): second_block_token = "assistant-second-block-token-codex" path = self._write_jsonl([ diff --git a/tests/test_commands_analysis.py b/tests/test_commands_analysis.py index 778ceb4..39a7a1d 100644 --- a/tests/test_commands_analysis.py +++ b/tests/test_commands_analysis.py @@ -427,7 +427,7 @@ def test_query_centered_snippet_is_identical_in_lines_and_jsonl(self): lines_out = io.StringIO() with contextlib.redirect_stdout(lines_out): cmd_search(_default_args(query="中段关键词", format="lines")) - line_snippet = lines_out.getvalue().split(": ", 1)[1].strip() + line_snippet = lines_out.getvalue().split("] ", 1)[1].strip() self.assertEqual(line_snippet, record["snippet"]) self.assertIn("中段关键词", record["snippet"]) @@ -459,8 +459,10 @@ def test_cmd_search_context_prints_grep_style_window(self): ) text = out.getvalue() - self.assertIn("search-sess-001:1:assistant", text) - self.assertIn("search-sess-001:2:user", text) + self.assertIn( + "[session=search-sess-001 | idx=1 | role=assistant]", text + ) + self.assertIn("[session=search-sess-001 | idx=2 | role=user]", text) self.assertIn("Assistant context line", text) self.assertIn("needle-token", text) @@ -479,8 +481,10 @@ def test_cmd_search_lines_format_is_compact_grep_output(self): ) text = out.getvalue() - self.assertIn("search-sess-001:1:assistant", text) - self.assertIn("search-sess-001:2:user", text) + self.assertIn( + "[session=search-sess-001 | idx=1 | role=assistant]", text + ) + self.assertIn("[session=search-sess-001 | idx=2 | role=user]", text) self.assertNotIn("Found ", text) self.assertNotIn("read-window:", text) diff --git a/tests/test_retrieval_tools.py b/tests/test_retrieval_tools.py index 285e218..1878c35 100644 --- a/tests/test_retrieval_tools.py +++ b/tests/test_retrieval_tools.py @@ -396,8 +396,10 @@ def test_cmd_search_high_context_prints_grep_style_window(self): ) text = out.getvalue() - self.assertIn("high-recall-context:1:assistant", text) - self.assertIn("high-recall-context:2:user", text) + self.assertIn( + "[session=high-recall-context | idx=1 | role=assistant]", text + ) + self.assertIn("[session=high-recall-context | idx=2 | role=user]", text) self.assertIn("Assistant context for search plus.", text) self.assertIn("high-recall-needle", text) @@ -922,7 +924,7 @@ def test_existing_index_is_readable_without_cache_write_permission(self): "_secure_cache_paths", side_effect=PermissionError("cache is not writable"), ): - data = read_window_data("readonly-window", idx=0, radius=0) + data = read_window_data("readonly-window", idx=0, before=0, after=0) self.assertEqual( [message["text"] for message in data["messages"]], @@ -944,7 +946,7 @@ def test_returns_messages_around_requested_index(self): ], ) - data = read_window_data("window-session", idx=2, radius=1) + data = read_window_data("window-session", idx=2, before=1, after=1) self.assertEqual(data["sessionId"], "window-session") self.assertEqual([m["idx"] for m in data["messages"]], [1, 2, 3]) @@ -966,7 +968,7 @@ def test_reads_window_without_loading_full_session(self): ) with patch("chatview.db.get_session_messages", side_effect=AssertionError("full session load")): - data = read_window_data("window-session", idx=2, radius=1) + data = read_window_data("window-session", idx=2, before=1, after=1) self.assertEqual([m["idx"] for m in data["messages"]], [1, 2, 3]) @@ -986,15 +988,15 @@ def test_batch_returns_multiple_windows(self): ) data = read_windows_data([ - {"session": "window-session", "idx": 0, "radius": 0}, - {"session": "window-session", "idx": 2, "radius": 1}, + {"session": "window-session", "idx": 0, "before": 0, "after": 0}, + {"session": "window-session", "idx": 2, "before": 1, "after": 1}, ]) self.assertEqual(len(data["windows"]), 2) self.assertEqual([m["idx"] for m in data["windows"][0]["messages"]], [0]) self.assertEqual([m["idx"] for m in data["windows"][1]["messages"]], [1, 2, 3]) - def test_radius_uses_idx_range_and_does_not_fill_gaps(self): + def test_before_after_count_actual_messages_across_sparse_indexes(self): self._insert_session( "sparse-window", "Sparse", @@ -1006,9 +1008,9 @@ def test_radius_uses_idx_range_and_does_not_fill_gaps(self): ], ) - data = read_window_data("sparse-window", idx=2, radius=1) + data = read_window_data("sparse-window", idx=2, before=1, after=1) - self.assertEqual([msg["idx"] for msg in data["messages"]], [2]) + self.assertEqual([msg["idx"] for msg in data["messages"]], [0, 2, 5]) def test_human_and_json_outputs_share_the_same_message_limit(self): import contextlib @@ -1026,21 +1028,30 @@ def test_human_and_json_outputs_share_the_same_message_limit(self): human = io.StringIO() with contextlib.redirect_stdout(human): - cmd_read_window(self._args(session="long-window", idx=0, radius=0, batch="")) - self.assertIn("truncated from 1500 chars", human.getvalue()) + cmd_read_window(self._args( + session="long-window", idx=0, before=0, after=0, + include="user:1000", around="", batch="", + )) + self.assertIn("⟪TRUNCATED · role_limit", human.getvalue()) + self.assertIn("[session=long-window | idx=0 | role=user]", human.getvalue()) structured = io.StringIO() with contextlib.redirect_stdout(structured): - cmd_read_window(self._args(session="long-window", idx=0, radius=0, batch="", json=True)) + cmd_read_window(self._args( + session="long-window", idx=0, before=0, after=0, + include="user:1000", around="", batch="", json=True, + )) raw_json = structured.getvalue() payload = json.loads(raw_json) message = payload["messages"][0] self.assertNotEqual(message["text"], long_text) - self.assertLessEqual(len(message["text"]), 1200) - self.assertEqual(message["originalChars"], 1500) - self.assertTrue(message["outputTruncated"]) + self.assertEqual(len(message["text"]), 1000) + self.assertNotIn("TRUNCATED", message["text"]) + self.assertEqual(message["indexedChars"], 1500) + self.assertTrue(message["truncated"]) + self.assertEqual(message["truncationReason"], "role_limit") self.assertTrue(payload["outputTruncated"]) - self.assertEqual(payload["omittedMessages"], 0) + self.assertEqual(payload["budgetOmittedMessages"], 0) self.assertEqual(payload["outputBytes"], len(raw_json.encode("utf-8"))) self.assertLessEqual(payload["outputBytes"], 20000) @@ -1067,56 +1078,134 @@ def test_worst_legal_batch_preserves_targets_and_never_exceeds_output_budget(sel assistant_messages, ) batch = json.dumps([ - {"session": "budget-window", "idx": idx, "radius": 5} + { + "session": "budget-window", "idx": idx, + "before": 5, "after": 5, + "include": {"user": 2000, "assistant": 600}, + } for idx in range(3, 8) ]) structured = io.StringIO() with contextlib.redirect_stdout(structured): - cmd_read_window(self._args(session=None, idx=None, radius=2, batch=batch, json=True)) + cmd_read_window(self._args(session=None, idx=None, batch=batch, json=True)) raw_json = structured.getvalue() payload = json.loads(raw_json) self.assertEqual(payload["outputBytes"], len(raw_json.encode("utf-8"))) self.assertLessEqual(payload["outputBytes"], 20000) self.assertTrue(payload["outputTruncated"]) - self.assertGreater(payload["omittedMessages"], 0) - target_lengths = [] + self.assertGreater(payload["budgetOmittedMessages"], 0) + target_lengths = {"user": [], "assistant": []} for window in payload["windows"]: self.assertIn(window["targetIndex"], [msg["idx"] for msg in window["messages"]]) target = next( msg for msg in window["messages"] if msg["idx"] == window["targetIndex"] ) - target_lengths.append(len(target["text"])) - self.assertEqual(len(set(target_lengths)), 1) + target_lengths[target["role"]].append(len(target["text"])) + for lengths in target_lengths.values(): + self.assertEqual(len(set(lengths)), 1) human = io.StringIO() with contextlib.redirect_stdout(human): - cmd_read_window(self._args(session=None, idx=None, radius=2, batch=batch, json=False)) + cmd_read_window(self._args(session=None, idx=None, batch=batch, json=False)) self.assertLessEqual(len(human.getvalue().encode("utf-8")), 20000) - self.assertIn("omittedMessages=", human.getvalue()) - - def test_rejects_radius_outside_zero_to_five(self): - with self.assertRaisesRegex(ValueError, "radius must be between 0 and 5"): - read_window_data("missing", idx=0, radius=-1) - with self.assertRaisesRegex(ValueError, "radius must be between 0 and 5"): - read_window_data("missing", idx=0, radius=6) - with self.assertRaisesRegex(ValueError, "radius must be between 0 and 5"): - read_window_data("missing", idx=0, radius=True) - with self.assertRaisesRegex(ValueError, "radius must be between 0 and 5"): - read_window_data("missing", idx=0, radius=1.9) - - def test_accepts_integer_radius_string(self): + self.assertIn("budgetOmittedMessages=", human.getvalue()) + + def test_rejects_invalid_context_counts(self): + with self.assertRaisesRegex(ValueError, "before must be between 0 and 20"): + read_window_data("missing", idx=0, before=-1) + with self.assertRaisesRegex(ValueError, "after must be between 0 and 20"): + read_window_data("missing", idx=0, after=21) + + def test_default_include_and_filtered_anchor_are_explicit(self): + self._insert_session( + "include-window", + "Include", + "distill-yourself", + [{"idx": 0, "text": "user context", "ts": "2026-07-01T10:00:00Z"}], + [{"idx": 1, "text": "assistant target", "ts": "2026-07-01T10:01:00Z"}], + ) + + default = read_window_data("include-window", idx=0, before=0, after=0) + filtered = read_window_data( + "include-window", idx=1, before=1, after=0, include="user:2000" + ) + + self.assertEqual(default["effectiveInclude"], {"user": 2000, "assistant": 600}) + self.assertFalse(filtered["anchor"]["included"]) + self.assertEqual(filtered["anchor"]["bodyStatus"], "filtered") + self.assertEqual([message["idx"] for message in filtered["messages"]], [0]) + + def test_around_requires_one_unique_target_match(self): self._insert_session( - "string-radius", - "String radius", + "around-window", + "Around", "distill-yourself", - [{"idx": 0, "text": "target", "ts": "2026-07-01T10:00:00Z"}], + [{ + "idx": 0, + "text": "a" * 900 + "unique-center" + "z" * 900, + "ts": "2026-07-01T10:00:00Z", + }], ) - data = read_window_data("string-radius", idx=0, radius="5") + data = read_window_data( + "around-window", idx=0, before=0, after=0, + include="user:200", around="unique-center", + ) + self.assertIn("unique-center", data["messages"][0]["text"]) + self.assertGreater(data["messages"][0]["sliceStart"], 0) - self.assertEqual(data["radius"], 5) + with self.assertRaisesRegex(ValueError, "was not found"): + read_window_data( + "around-window", idx=0, before=0, after=0, + include="user:200", around="missing-center", + ) + + self._insert_session( + "repeated-around", "Repeated", "distill-yourself", + [{"idx": 0, "text": "same same", "ts": "2026-07-01T10:00:00Z"}], + ) + with self.assertRaisesRegex(ValueError, "matched 2 times"): + read_window_data( + "repeated-around", idx=0, before=0, after=0, + include="user:200", around="same", + ) + + def test_reports_source_completeness_and_index_freshness(self): + source = Path(self._tmpdir) / "freshness.jsonl" + source.write_text("{}\n", encoding="utf-8") + indexed_mtime = source.stat().st_mtime + meta = { + "id": "fresh-window", + "title": "Fresh", + "date": "2026-07-01", + "lastDate": "2026-07-01", + "filePath": str(source), + "fileSize": source.stat().st_size, + "_mtime": indexed_mtime, + "userMessageCount": 1, + "preview": "complete", + "project": "distill-yourself", + "projectName": "distill-yourself", + "source": "codex", + } + db.upsert_session(meta, [{ + "idx": 0, + "text": "complete", + "ts": "2026-07-01T10:00:00Z", + "sourceChars": 8, + "sourceComplete": True, + }], []) + + fresh = read_window_data("fresh-window", idx=0, before=0, after=0) + self.assertEqual(fresh["indexState"], "fresh") + self.assertTrue(fresh["messages"][0]["sourceComplete"]) + self.assertEqual(fresh["messages"][0]["sourceChars"], 8) + + os.utime(source, (indexed_mtime + 10, indexed_mtime + 10)) + stale = read_window_data("fresh-window", idx=0, before=0, after=0) + self.assertEqual(stale["indexState"], "stale") def test_skips_oversized_context_and_keeps_later_short_context(self): requests = [] @@ -1143,7 +1232,7 @@ def test_skips_oversized_context_and_keeps_later_short_context(self): "distill-yourself", messages, ) - requests.append({"session": session, "idx": 0, "radius": 1}) + requests.append({"session": session, "idx": 0, "before": 0, "after": 1}) data = read_windows_data(requests) @@ -1161,7 +1250,7 @@ def test_rejects_more_than_five_batch_items(self): from chatview.commands.retrieval import cmd_read_window requests = [ - {"session": "any", "idx": idx, "radius": 0} + {"session": "any", "idx": idx, "before": 0, "after": 0} for idx in range(6) ] @@ -1171,22 +1260,17 @@ def test_rejects_more_than_five_batch_items(self): cmd_read_window(self._args( session=None, idx=None, - radius=2, batch=json.dumps(requests), json=True, )) - def test_command_rejects_radius_six_instead_of_printing_success(self): - from chatview.commands.retrieval import cmd_read_window - - with self.assertRaisesRegex(ValueError, "radius must be between 0 and 5"): - cmd_read_window(self._args( - session="missing", - idx=0, - radius=6, - batch="", - json=True, - )) + def test_missing_target_is_an_error(self): + self._insert_session( + "missing-target", "Missing", "distill-yourself", + [{"idx": 0, "text": "nearby", "ts": "2026-07-01T10:00:00Z"}], + ) + with self.assertRaisesRegex(KeyError, "Message not found"): + read_window_data("missing-target", idx=1, before=1, after=1) class TestSessionBriefData(RetrievalToolTestCase): From d98540d38cf141b940e727cca962f031bfe22a76 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sun, 19 Jul 2026 11:45:49 -0400 Subject: [PATCH 13/15] docs: teach flexible history window retrieval --- skills/distill-yourself/SKILL.md | 27 +++++++++++-------- skills/distill-yourself/evals/evals.json | 26 ++++++++++++++++-- .../references/twin-cognitive-model.md | 6 ++--- tests/test_distill_skill_static.py | 22 +++++++++++---- 4 files changed, 60 insertions(+), 21 deletions(-) diff --git a/skills/distill-yourself/SKILL.md b/skills/distill-yourself/SKILL.md index 8d11761..1b5a078 100644 --- a/skills/distill-yourself/SKILL.md +++ b/skills/distill-yourself/SKILL.md @@ -16,6 +16,7 @@ description: "本地 Claude Code/Codex/agent 对话历史的唯一检索入口 - 不要扫描、打开或直接读取 session/rollout JSONL，也不要对它运行 `rg -n`、`rg -l`、`head`、`sed`、`jq`、Python 等 shell/content reader。JSONL 单行可能包含数万 token；`filePath` 和 byte locator 只供 `distill` 的有界读取器内部使用。 - 若任务还匹配其他工作流 skill，先完成历史取证，再把核验后的上下文交给后续流程。 - `distill` 只负责取数、检索和暂存；结论由你基于证据生成。 +- 不创建 Episode 或 Resume Context 这类持久模式。把 `search` 找到的 `sessionId + idx` 与 `read-window` 的范围、角色投影和正文预算自由组合，由当前 Agent 根据任务语义选择需要的多轮片段。 - `profile-digest` / `aggregates` / `stats` 是地图，不是结论。写入 Memory/Profile/Twin 前必须用 `read-window`、`session-brief` 或其他有界 `distill` reader 核验；不能把“核验原 session”解释为直接读取 session JSONL。 - 不把 assistant echo、IDE/file context、task notification、agent/subagent prompt、工具输出噪声当作主证据。 - 蒸馏长期偏好、Memory、Rules 或 Patterns 时，检索命令加 `--evidence-only`；普通历史定位不要加，以免隐藏诊断线索。 @@ -29,7 +30,7 @@ description: "本地 Claude Code/Codex/agent 对话历史的唯一检索入口 优先使用 PATH 上的 `distill`。如果没有，把命令替换成 `python3 /analyze.py`。 -探索命令读取本地 `~/.claude/` 和 `~/.codex/` 历史；`distill refresh` 会重建本地 SQLite/cache。`evolve-write` 只修改持久蒸馏数据，Memory/Profile 的配置同步默认由用户在 UI 中执行。`twin-batch` 和 `twin-sync --execute` 会修改持久认知模型数据或 Claude/Codex 配置。 +探索命令读取本地 `~/.claude/` 和 `~/.codex/` 历史；`distill refresh` 默认只增量更新新增或已变化的 session，`--force` 才全量重解析。`evolve-write` 只修改持久蒸馏数据，Memory/Profile 的配置同步默认由用户在 UI 中执行。`twin-batch` 和 `twin-sync --execute` 会修改持久认知模型数据或 Claude/Codex 配置。 ## Standard Workflow @@ -42,7 +43,7 @@ description: "本地 Claude Code/Codex/agent 对话历史的唯一检索入口 5. HANDOFF -> user previews and confirms configuration sync in the UI ``` -Choose the scope before retrieval. For an exact fact, phrase, session, or project episode, use the fast path: refresh once, retrieve one page of at most 20 candidates, then verify only the best 2-4 windows. Do not load digest/aggregates/stats for a simple lookup. +Choose the scope before retrieval. For an exact fact, phrase, session, or project discussion, use the fast path: refresh once, retrieve one page of at most 20 candidates, then verify only the best 2-4 windows. Do not load digest/aggregates/stats for a simple lookup. Resolve the user's time wording before retrieval. For a reproducible calendar range, use `--start YYYY-MM-DD --end YYYY-MM-DD`; both boundaries are inclusive, either boundary may be omitted, and these options must not be combined with `--date`. Treat `--date 1d/7d/30d/90d/all` as a quick relative shortcut only. When the user says “过去一周” or another relative period, resolve it against the current date, state the interpreted dates, and carry the same exact range through every locating command. If the user requests commands only, state the interpretation in a short shell comment before the commands. By default, “过去 N 天” includes today and the preceding `N-1` calendar dates; surface a materially different interpretation instead of silently changing it. @@ -50,13 +51,13 @@ Resolve the user's time wording before retrieval. For a reproducible calendar ra distill refresh distill search "" --recall normal --role user --evidence-only --format jsonl --max-chars 500 --limit 20 --page 1 --date 90d distill search "" --recall normal --role user --format jsonl --max-chars 500 --limit 20 --page 1 --start 2026-07-01 --end 2026-07-07 -distill read-window --batch '[{"sessionId":"session_id","idx":123,"radius":2}]' +distill read-window --batch '[{"sessionId":"session_id","idx":123,"before":8,"after":5,"include":"user:2000,assistant:600"}]' ``` For exact tool provenance, use a separate bounded ladder. If the user already gives a filename, path, command fragment, patch target, or call ID, start at step 2: ```text -1. EPISODE -> use message search only when you still need to identify the episode +1. CONTEXT -> use message search only when you still need to identify the discussion 2. LOCATE -> tool-search the narrowest stable anchor 3. SELECT -> keep the best 1-3 sessionId + eventIdx locators 4. VERIFY -> read-tool-event for the exact call input @@ -111,7 +112,8 @@ Expand to `all` only when evidence is thin. |---|---|---| | Exact preference phrase or correction | `distill search "" --recall normal --role user --evidence-only --format jsonl --limit 20 --page 1` | Select 2-4 direct user hits, then `read-window --batch` | | Conceptual preference | Split into 2-4 concrete anchors, then run `search --recall normal` for each | If direct user evidence is thin, repeat with `--recall high`, then page | -| Topic, project, or historical episode | `distill find-repeats "" --limit 5 --json` | `session-brief` / `read-window --batch` for top candidates | +| Topic, project, or historical discussion | `distill find-repeats "" --limit 5 --json` | `session-brief` / `read-window --batch` for top candidates | +| Complete multi-turn discussion or compaction recovery | Locate 1-3 direct user anchors with `search` | Read a wider message window; separately verify relevant tool events, files, Git, or tests when needed | | Possible orchestration noise | `distill evidence-audit --json` | Treat `artifactReason` rows as diagnostic only | | Session-level context | `distill session-brief ` | `distill read-window --idx N` | | Tool call, command, patch, or file provenance | `distill tool-search "" --format jsonl --limit 20 --page 1` | `distill read-tool-event --event-idx N`; add `--include-result` only if needed | @@ -120,13 +122,13 @@ Important commands: | Command | Purpose | |---|---| -| `distill corrections --limit 100` | Correction anchors with stable `idx` and the nearest user/assistant pair in `conversation` | +| `distill corrections --limit 100` | Correction anchors with a session-scoped `idx` and the nearest user/assistant pair in `conversation` | | `distill queries --limit 50` | User requests, including possible acceptance signals | | `distill highlights --limit 20` | High-signal sessions ranked by correction/decision density | -| `distill search "" --recall normal --format lines --limit 20 --page 1` | Compact grep-style exact results with `session:idx:role` locators | +| `distill search "" --recall normal --format lines --limit 20 --page 1` | Compact exact results with explicit `session`, `idx`, and `role` labels | | `distill search "" --recall high --format jsonl --limit 20 --page 1` | Hybrid high-recall results with match reasons, artifact downranking, and `idx` | | `distill find-repeats "" --limit 5 --json` | Evidence buckets: strong, related, weak, artifacts | -| `distill read-window --idx N --radius 2` | Small context window around a hit | +| `distill read-window --idx N --before 8 --after 5 --include user:2000,assistant:600` | Compose a bounded multi-turn window around a hit | | `distill read-window --batch '[...]'` | Verify several windows at once | | `distill evidence-audit --json` | Estimate contamination from prompts/tasks/context noise | | `distill tool-search "" --format jsonl --limit 20 --page 1` | Bounded search over tool-call inputs with JSONL locators | @@ -134,13 +136,16 @@ Important commands: | `distill read-tool-event --event-idx N --around "" --max-chars 2000` | Center the bounded call projection on an already known anchor | | `distill read-tool-event --event-idx N --include-result --max-chars 2000` | Read the matching result through its indexed locator without indexing its body | -`corrections` JSON exposes a stable `idx` and its nearest user/assistant pair. `search` JSON/JSONL exposes `sessionId`, `idx`, and `messageIndex`; use `sessionId + idx` as the read coordinate. `idx` is only ordered within one session, not a global message id. `read-window --radius N` reads the inclusive numeric range `idx-N ... idx+N`, so gaps may produce fewer than `2N+1` messages. +`corrections` and `search` JSON/JSONL expose `sessionId`, `idx`, and sometimes `messageIndex`; use `sessionId + idx` as the read coordinate. `idx` belongs to the current session index snapshot: it is neither a global ID nor a permanent cross-refresh identifier, and gaps are allowed. A missing or duplicate target is an error rather than a guessed match. ```bash -distill read-window --batch '[{"sessionId":"session_id","idx":123,"radius":2}]' +distill read-window --idx 123 +distill read-window --batch '[{"sessionId":"session_id","idx":123,"before":8,"after":5,"include":"user:2000,assistant:600"}]' ``` -Every `read-window` rendering uses the same bounded serializer: human, `--json`, and any other structured format are all output formats under one budget. Enforce `radius <= 5`, `batch size <= 5`, and `total serialized output <= 20 KB`; a smaller remaining budget truncates later messages/windows and reports truncation metadata. These are workflow-wide safety limits, not hints: do not reconstruct oversized context with multiple batches, later pages, or raw JSONL, and do not retry another format to obtain omitted text. Narrow the selected locators or anchor instead. +`read-window` defaults to two actual indexed messages before and after the target and `--include user:2000,assistant:600`. Set `--before` / `--after` independently up to 20; it selects actual neighboring messages first, then applies the role projection. `--include` accepts role budgets such as `user:2000,assistant:600`; omit a role to filter it out. Use `--around ""` only when that literal occurs exactly once in the target role's full indexed source; zero or multiple matches are errors, so lengthen the anchor instead of guessing. + +Every `read-window` rendering uses the same bounded serializer: human, `--json`, and all output formats share `batch size <= 5` and `total serialized output <= 20 KB`. Structured `text` remains pure source projection; human output adds visible truncation markers, while JSON reports truncation, source completeness, coordinates, and freshness as metadata. Future parsed messages retain full user source before projection; legacy rows may report source completeness unknown and are not silently treated as complete. If freshness is stale or the source is missing, refresh before drawing a negative conclusion. Do not reconstruct one truncated transcript with multiple batches, later pages, another format, or raw JSONL; narrow the selected locators or exact anchor instead. For manual triage, prefer `--format lines`; for scripts or incremental aggregation, prefer `--format jsonl`. `--limit N` is the page size and `--page 2`, `--page 3`, ... retrieves later pages. Keep the standard page size at 20 and page instead of requesting one oversized result. Mirrored copies of the same timestamped message are folded before paging, while `duplicateCount` / `duplicateSessionIds` preserve provenance in structured output. diff --git a/skills/distill-yourself/evals/evals.json b/skills/distill-yourself/evals/evals.json index 99cad8b..70a47d9 100644 --- a/skills/distill-yourself/evals/evals.json +++ b/skills/distill-yourself/evals/evals.json @@ -62,10 +62,10 @@ { "id": 6, "prompt": "search 找到了 14 个候选窗口，每条正文都可能非常大。我想用 read-window --json 一次拿全，再分几批补齐。请只给安全的核验方案和停止条件，不要执行。", - "expected_output": "拒绝一次拿全或分批重建；先筛到最多 5 个窗口，radius 不超过 5，单次 read-window batch，并说明 human/json 等所有格式共享 20KB 总序列化预算，截断后应缩小 locator/anchor 而非多批、翻页或读 raw JSONL。", + "expected_output": "拒绝一次拿全或分批重建；先筛到最多 5 个窗口，按任务选择 before/after（各不超过 20）与 include 投影，单次 read-window batch，并说明 human/json 等所有格式共享 20KB 总序列化预算，截断后应缩小 locator 或使用唯一精确 around anchor，而非多批、翻页或读 raw JSONL。", "files": [], "expectations": [ - "Limits read-window radius to at most 5, batch size to at most 5, and total serialized output to at most 20KB", + "Limits read-window before and after to at most 20 each, batch size to at most 5, and total serialized output to at most 20KB", "States that JSON, human, and all other read-window formats share the same output budget", "Refuses to reconstruct omitted context through multiple batches, later pages, another format, or raw JSONL", "Narrows candidate locators or anchors and stops once the selected evidence is sufficient" @@ -81,6 +81,28 @@ "Uses search with --start 2026-07-12 and --end 2026-07-18 without also using --date", "Selects a small number of sessionId plus idx locators and verifies them with read-window" ] + }, + { + "id": 8, + "prompt": "我想把今天讨论‘完整对话链路怎么保存’的来龙去脉恢复出来，不是只找一句话。现在只写检索与读取方案，不要执行。", + "expected_output": "先用多个直接 user anchor 定位同一讨论的 sessionId+idx，再由 Agent 选择足够的 before/after 与 user/assistant include 组合读取上下多轮；必要时 batch 少量独立窗口。不要创建 Episode 或 Resume Context 模式，也不要假定 idx 跨刷新永久稳定。", + "files": [], + "expectations": [ + "Uses search to locate direct user anchors and composes multi-turn read-window ranges and role budgets", + "Treats idx as scoped to the current session index snapshot rather than a permanent ID", + "Does not introduce a persistent Episode or Resume Context mode" + ] + }, + { + "id": 9, + "prompt": "自动压缩后我要恢复当前任务：用户要求、Agent 已做的修改和测试状态都要核对。现在只写方案，不要执行，也不要发明 Plan 专用查询参数。", + "expected_output": "先以用户原话为 anchor 用 search+read-window 恢复约束和决定；再按需要独立核验相关 tool event、工作区文件、Git 和测试状态。消息窗口可扩大 before/after 并调整 include，但不创建 Resume Context 模式，也不发明 tool-search 的 session/tool/Plan 过滤参数。", + "files": [], + "expectations": [ + "Recovers user instructions from message anchors and verifies execution state from tool events, files, Git, or tests as separate evidence", + "Uses composable read-window parameters rather than a dedicated Resume Context mode", + "Does not invent Plan-specific or session/tool filter flags for tool-search" + ] } ] } diff --git a/skills/distill-yourself/references/twin-cognitive-model.md b/skills/distill-yourself/references/twin-cognitive-model.md index 34f19d6..bbbe47b 100644 --- a/skills/distill-yourself/references/twin-cognitive-model.md +++ b/skills/distill-yourself/references/twin-cognitive-model.md @@ -81,10 +81,10 @@ Required order: distill twin-search events --q "keyword" --json ``` -4. Only deep-read candidates that survive dedupe. `corrections` exposes a stable `idx` plus the nearest user/assistant pair in `conversation`, so correction candidates can be verified precisely: +4. Only deep-read candidates that survive dedupe. `corrections` exposes a session-scoped `idx` plus the nearest user/assistant pair in `conversation`, so correction candidates can be verified precisely: ```bash - distill read-window --idx --radius 2 + distill read-window --idx ``` `highlights` and `queries` may still require session-level reading. Search JSONL exposes `sessionId`, `idx`, and `messageIndex`; use `sessionId + idx` as the read coordinate. For quick manual triage, locate candidates first, then deep-read only survivors: @@ -98,7 +98,7 @@ Required order: Do not invent an idx. For older results that expose only `messageIndex`, map it to `idx`: ```bash - distill read-window --batch '[{"sessionId":"session_id","idx":123,"radius":2}]' + distill read-window --batch '[{"sessionId":"session_id","idx":123,"before":2,"after":2,"include":"user:2000,assistant:600"}]' ``` 5. After user confirmation, write new or enriched events with `twin-batch`: diff --git a/tests/test_distill_skill_static.py b/tests/test_distill_skill_static.py index 41bbcb6..49c75da 100644 --- a/tests/test_distill_skill_static.py +++ b/tests/test_distill_skill_static.py @@ -118,18 +118,28 @@ def test_skill_md_is_router_and_declares_required_references(self): self.assertIn("default bounded head projection", text) self.assertNotIn("default head/tail projection", text) self.assertIn("--around \"\"", text) - self.assertIn("`radius <= 5`", text) + self.assertIn("`--before` / `--after` independently up to 20", text) + self.assertIn("`--include user:2000,assistant:600`", text) + self.assertIn('`--around ""`', text) + self.assertIn("current session index snapshot", text) + self.assertIn("source completeness unknown", text) + self.assertIn("freshness", text) self.assertIn("`batch size <= 5`", text) self.assertIn("`total serialized output <= 20 KB`", text) - self.assertIn("all output formats", text) - self.assertIn("multiple batches, later pages, or raw JSONL", text) + self.assertIn("all output formats share", text) + self.assertIn( + "multiple batches, later pages, another format, or raw JSONL", text + ) self.assertNotIn("--json` returns the complete text stored in the local index", text) self.assertIn("--format lines", text) self.assertIn("--format jsonl", text) self.assertIn("--evidence-only", text) self.assertIn("--page", text) self.assertIn("--max-chars 500", text) - self.assertIn('"sessionId":"session_id","idx":123,"radius":2', text) + self.assertIn( + '"sessionId":"session_id","idx":123,"before":8,"after":5', text + ) + self.assertNotIn("--radius", text) self.assertIn("~/.agents/skills/distill-yourself", text) self.assertNotIn("~/.codex/skills/distill-yourself", text) self.assertIn("distill install-skill --force", text) @@ -219,7 +229,9 @@ def test_twin_reference_documents_current_six_stage_batch_workflow(self): self.assertIn("Do not invent an idx", text) self.assertIn('distill search "" --recall high', text) self.assertIn("use `sessionId + idx` as the read coordinate", text) - self.assertIn('"sessionId":"session_id","idx":123,"radius":2', text) + self.assertIn( + '"sessionId":"session_id","idx":123,"before":2,"after":2', text + ) if __name__ == "__main__": From 6dbdc487fb968007a804c9c3c256cff685ae24f5 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sun, 19 Jul 2026 12:18:49 -0400 Subject: [PATCH 14/15] fix: restore Python 3.9 session imports --- chatview/db/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatview/db/sessions.py b/chatview/db/sessions.py index e155418..bf957c7 100644 --- a/chatview/db/sessions.py +++ b/chatview/db/sessions.py @@ -589,7 +589,7 @@ def get_message_window(session_id: str, start_idx: int, end_idx: int) -> list: return [dict(r) for r in rows] -def get_message_at_index(session_id: str, idx: int) -> dict | None: +def get_message_at_index(session_id: str, idx: int) -> Optional[dict]: """Return one exact message coordinate, rejecting ambiguous old data.""" conn = get_conn() rows = conn.execute( From f1f76136a02d0f2bfdac11c28c508f941e7b5944 Mon Sep 17 00:00:00 2001 From: wanghuacan Date: Sun, 19 Jul 2026 13:01:00 -0400 Subject: [PATCH 15/15] ci: restore green pull request checks --- .github/workflows/test.yml | 2 ++ chatview/commands/retrieval.py | 2 +- chatview/handlers/twin.py | 4 ++-- tests/test_commands_twin.py | 4 ---- tests/test_db.py | 1 - tests/test_frontend_behavior.py | 1 - tests/test_handlers_data.py | 1 - tests/test_search_robustness.py | 5 +++-- tests/test_twin_run_lifecycle.py | 7 +++---- tests/test_twin_runs_api.py | 5 +---- 10 files changed, 12 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index edb6eb3..d2c8e51 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,6 +26,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install test dependencies + run: python -m pip install pytest - name: Syntax / import smoke check run: | python -m py_compile server.py db.py analyze.py diff --git a/chatview/commands/retrieval.py b/chatview/commands/retrieval.py index 8bf0b03..3cdb8c4 100644 --- a/chatview/commands/retrieval.py +++ b/chatview/commands/retrieval.py @@ -12,7 +12,7 @@ from chatview.commands.evidence import ( artifact_reason, dedupe_message_results, - is_noise_text, + is_noise_text, # noqa: F401 - compatibility re-export page_results, ) from chatview.snippets import make_query_snippet diff --git a/chatview/handlers/twin.py b/chatview/handlers/twin.py index 9027ce3..76cada7 100644 --- a/chatview/handlers/twin.py +++ b/chatview/handlers/twin.py @@ -850,7 +850,7 @@ def _build_twin_stage2_prompt( ) events_note = ( "\n\n> Note: events above are SUMMARIZED (id/lesson/signal_intensity) to " - f"stay within the prompt budget. Use `twin-get events ` via the CLI " + "stay within the prompt budget. Use `twin-get events ` via the CLI " "to fetch the full detail of any event you need." if events_slimmed else "" @@ -1043,7 +1043,7 @@ def _build_twin_stage3_prompt( ) cards_note = ( "\n\n> Note: cards above are SUMMARIZED (id/applies_when/judgment/agent_action/confidence/status) to " - f"stay within the prompt budget. Use `twin-get cards ` via the CLI " + "stay within the prompt budget. Use `twin-get cards ` via the CLI " "to fetch the full detail of any card you need." if cards_slimmed else "" diff --git a/tests/test_commands_twin.py b/tests/test_commands_twin.py index 61cb2a9..67cbba2 100644 --- a/tests/test_commands_twin.py +++ b/tests/test_commands_twin.py @@ -380,7 +380,6 @@ def _run_batch(self, payload): def test_batch_add_records_change_with_run_id(self): """batch add → twin_changes records action=add attributed to the batch run_id.""" - import chatview.db as _db from chatview.db.twin import changes_for_run run_id = "run-add-test-001" @@ -471,8 +470,6 @@ def test_batch_edit_cross_run_allowed_and_logs_change(self): def test_single_edit_without_run_id_no_change_logged(self): """cmd_twin_edit without run_id writes no twin_changes entry.""" - import chatview.db as _db - from chatview.db.twin import changes_for_run from chatview.commands.twin import cmd_twin_add, cmd_twin_edit # Add a card (no run_id) @@ -679,7 +676,6 @@ def test_cross_run_event_card_link_succeeds(self): evidence_count = full-corpus count, not run-scoped.""" import chatview.db as _db from chatview.commands.twin import _twin_link - from chatview.db.twin import changes_for_run # event from run-A, card from run-B self._seed_event("ev_xrun1", run_id="run-A") diff --git a/tests/test_db.py b/tests/test_db.py index cf4a962..894541a 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -258,7 +258,6 @@ def test_init_db_adds_project_identity_columns_to_legacy_sessions(self): def test_init_db_repairs_private_cache_and_sqlite_permissions(self): from pathlib import Path - from chatview.db import core as _dbcore cache_dir = Path(self._tmpdir) db_path = cache_dir / "sessions.db" diff --git a/tests/test_frontend_behavior.py b/tests/test_frontend_behavior.py index 77f496f..298fdcf 100644 --- a/tests/test_frontend_behavior.py +++ b/tests/test_frontend_behavior.py @@ -17,7 +17,6 @@ import sys import tempfile import threading -import time from collections import Counter from http.server import ThreadingHTTPServer from pathlib import Path diff --git a/tests/test_handlers_data.py b/tests/test_handlers_data.py index 0138045..1b0c2a8 100644 --- a/tests/test_handlers_data.py +++ b/tests/test_handlers_data.py @@ -8,7 +8,6 @@ import shutil import sys import tempfile -import threading import unittest from pathlib import Path diff --git a/tests/test_search_robustness.py b/tests/test_search_robustness.py index 0a899bb..f76043d 100644 --- a/tests/test_search_robustness.py +++ b/tests/test_search_robustness.py @@ -4,7 +4,6 @@ import hashlib import io import json -import os import shutil import sqlite3 import sys @@ -119,7 +118,9 @@ def test_page_two_matches_the_second_half_of_a_larger_first_page(self): page_two, _ = self._search(rows, page=2, limit=20) combined, _ = self._search(rows, page=1, limit=40) - keys = lambda data: [(row["sessionId"], row["idx"]) for row in data] + def keys(data): + return [(row["sessionId"], row["idx"]) for row in data] + self.assertEqual(keys(page_one) + keys(page_two), keys(combined)) diff --git a/tests/test_twin_run_lifecycle.py b/tests/test_twin_run_lifecycle.py index a248675..54410db 100644 --- a/tests/test_twin_run_lifecycle.py +++ b/tests/test_twin_run_lifecycle.py @@ -10,7 +10,6 @@ import json import os import shutil -import subprocess import sys import tempfile import threading @@ -21,9 +20,9 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT) -import chatview.db as _db -from chatview.db import core as _dbcore -from chatview.handlers import twin as _twin +import chatview.db as _db # noqa: E402 +from chatview.db import core as _dbcore # noqa: E402 +from chatview.handlers import twin as _twin # noqa: E402 # --------------------------------------------------------------------------- diff --git a/tests/test_twin_runs_api.py b/tests/test_twin_runs_api.py index 057e820..56e4e86 100644 --- a/tests/test_twin_runs_api.py +++ b/tests/test_twin_runs_api.py @@ -26,7 +26,7 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT) -import server +import server # noqa: E402 def _make_session_jsonl(session_id="twin-runs-api-sess-001"): @@ -255,7 +255,6 @@ def test_failed_run_with_no_completed_stage_and_no_changes_excluded(self): def test_only_latest_resumable_run_is_flagged(self): """When multiple resumable runs exist, only the newest gets resumable:true.""" import chatview.db as _db - import time # older resumable run run_id_old = "run_resumable_older" @@ -702,7 +701,6 @@ def test_analyze_explicit_run_id_resumes_that_run(self): _db.save_checkpoint(run_id, 2, "failed") _db.run_finish(run_id, "failed") - seen_run_ids = [] seen_skip_stages = [] class _FakeHandlerBody: @@ -753,7 +751,6 @@ def capture_stage(stage_num, *a, **kw): row = _db.run_get(run_id) # The run should have been re-used (run_id matches) # Verify by checking that the checkpoint for stage 1 was already completed before analyze - cp = _db.get_checkpoint(run_id) # Stage 1 was marked completed before — so it should be in skip_stages # We verify the twin_runs row was continued (not a new run_id) self.assertIsNotNone(row, "twin_runs row for explicit run_id should exist")