From b09ae837e2edf8361432edf3cd0ed0360668388a Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Mon, 30 Mar 2026 01:50:19 -0700 Subject: [PATCH 1/2] Move semantic hints from system prompt to user prompt System prompt is now static per schema version, enabling prompt caching across requests. Semantic hints (which change per image/description) are placed in the user prompt instead. The system prompt includes a pointer instructing the LLM to check the user message for hints. Fixes #129 --- src/agents/annotation_agent.py | 43 ++++++++++++++++++++++------ src/utils/hed_comprehensive_guide.py | 17 +++++++---- tests/test_comprehensive_guide.py | 40 +++++++++----------------- 3 files changed, 59 insertions(+), 41 deletions(-) diff --git a/src/agents/annotation_agent.py b/src/agents/annotation_agent.py index 82b3397..24dcaf2 100644 --- a/src/agents/annotation_agent.py +++ b/src/agents/annotation_agent.py @@ -60,21 +60,23 @@ def _build_system_prompt( self, vocabulary: list[str], extendable_tags: list[str], - semantic_hints: list[dict] | None = None, no_extend: bool = False, ) -> str: """Build the system prompt for the annotation agent. + The system prompt is static per schema version (vocabulary + rules) + to enable prompt caching across requests. Semantic hints are placed + in the user prompt instead. + Args: vocabulary: List of valid short-form HED tags extendable_tags: Tags that allow extension - semantic_hints: Optional semantic search results with relevant tags no_extend: If True, prohibit tag extensions Returns: Complete system prompt with all HED rules """ - return get_comprehensive_hed_guide(vocabulary, extendable_tags, semantic_hints, no_extend) + return get_comprehensive_hed_guide(vocabulary, extendable_tags, no_extend) def _format_tag_suggestions(self, tag_suggestions: dict[str, list[str]]) -> str: """Format tag suggestions into a clear instruction block. @@ -98,24 +100,47 @@ def _format_tag_suggestions(self, tag_suggestions: dict[str, list[str]]) -> str: ) return "\n".join(lines) + def _format_semantic_hints(self, semantic_hints: list[dict] | None) -> str: + """Format semantic hints for inclusion in the user prompt. + + Args: + semantic_hints: List of hint dicts with tag, score, source keys + + Returns: + Formatted hints section, or empty string if no hints + """ + if not semantic_hints: + return "" + + from src.utils.hed_comprehensive_guide import _format_semantic_hints + + return "\n" + _format_semantic_hints(semantic_hints) + def _build_user_prompt( self, description: str, validation_errors: list[str] | None = None, tag_suggestions: dict[str, list[str]] | None = None, previous_annotation: str | None = None, + semantic_hints: list[dict] | None = None, ) -> str: """Build the user prompt for annotation. + Semantic hints are included here (not in system prompt) so the + system prompt stays static and cacheable across requests. + Args: description: Natural language event description validation_errors: Previous validation errors (if retrying) tag_suggestions: LSP-suggested valid tags for invalid tags previous_annotation: The previous annotation attempt (for targeted correction) + semantic_hints: Optional semantic search hints for relevant tags Returns: User prompt string """ + hints_str = self._format_semantic_hints(semantic_hints) + if validation_errors: errors_str = "\n".join(f"- {error}" for error in validation_errors) suggestions_str = self._format_tag_suggestions(tag_suggestions or {}) @@ -133,12 +158,12 @@ def _build_user_prompt( Fix these errors and generate a corrected HED annotation for: {description} - +{hints_str} {replacement_note}CRITICAL: Output ONLY the raw HED annotation string.""" return f"""Generate a HED annotation for this event description: {description} - +{hints_str} CRITICAL: Output ONLY the raw HED annotation string.""" async def annotate(self, state: HedAnnotationState) -> dict: @@ -170,17 +195,16 @@ async def annotate(self, state: HedAnnotationState) -> dict: # Use empty list - LLM will still generate valid annotations extendable_tags = [] - # Build prompts with complete HED rules (including semantic hints if available) - semantic_hints = state.get("semantic_hints", []) + # Build system prompt with HED rules (static per schema version for caching) no_extend = state.get("no_extend", False) system_prompt = self._build_system_prompt( vocabulary, extendable_tags, - semantic_hints if semantic_hints else None, no_extend, ) - # Build user prompt with any feedback (use augmented errors with remediation for LLM) + # Build user prompt with feedback and semantic hints + semantic_hints = state.get("semantic_hints", []) feedbacks = [] if state.get("validation_errors_augmented"): feedbacks.extend(state["validation_errors_augmented"]) @@ -200,6 +224,7 @@ async def annotate(self, state: HedAnnotationState) -> dict: feedbacks or None, tag_suggestions or None, previous_annotation, + semantic_hints if semantic_hints else None, ) # Generate annotation diff --git a/src/utils/hed_comprehensive_guide.py b/src/utils/hed_comprehensive_guide.py index 1bf701d..b5d2274 100644 --- a/src/utils/hed_comprehensive_guide.py +++ b/src/utils/hed_comprehensive_guide.py @@ -974,7 +974,6 @@ def _build_output_format_section() -> str: def get_comprehensive_hed_guide( vocabulary_sample: list[str], extendable_tags: list[str], - semantic_hints: list[dict] | None = None, no_extend: bool = False, ) -> str: """Generate comprehensive HED annotation guide. @@ -983,11 +982,13 @@ def get_comprehensive_hed_guide( annotation agent. The guide includes vocabulary constraints, semantic rules, correction workflows, and output format instructions. + Note: Semantic hints are NOT included here to keep the system prompt + static across requests, enabling prompt caching. Hints are passed + in the user prompt instead. + Args: vocabulary_sample: Full list of valid HED tags (complete vocabulary) extendable_tags: Tags that allow extension - semantic_hints: Optional list of semantically relevant tags from search - Each dict has: tag, score, source, prefix (optional) no_extend: If True, add strict instructions to prohibit tag extensions Returns: @@ -997,16 +998,22 @@ def get_comprehensive_hed_guide( extend_str = ", ".join(extendable_tags) if not no_extend else "(Extensions disabled)" # Format optional sections - hints_section = _format_semantic_hints(semantic_hints) if semantic_hints else "" no_extend_warning = _build_no_extend_warning() if no_extend else "" # Assemble guide from modular sections + # Note: semantic hints are placed in the user prompt for cache efficiency sections = [ "# HED ANNOTATION GUIDE\n", no_extend_warning, _build_vocabulary_check_section(), _build_correction_workflow_section(), - hints_section, + ( + "## SEMANTIC HINTS\n\n" + "Check the user message for a SEMANTIC HINTS section with " + "potentially relevant tags from schema search. Use these as " + "guidance for tag selection, but verify each against the vocabulary.\n\n" + "---\n\n" + ), _build_semantic_rules_section(), _build_relation_tags_section(), _build_event_agent_section(), diff --git a/tests/test_comprehensive_guide.py b/tests/test_comprehensive_guide.py index 0609bd4..22ab6cb 100644 --- a/tests/test_comprehensive_guide.py +++ b/tests/test_comprehensive_guide.py @@ -45,38 +45,24 @@ def test_guide_with_no_extend_true(self): # Should show extensions as disabled assert "(Extensions disabled)" in guide - def test_guide_with_semantic_hints(self): - """Test guide generation with semantic hints.""" + def test_guide_has_semantic_hints_pointer(self): + """Test guide includes a pointer to check user message for semantic hints.""" vocabulary = ["Event", "Reward", "Animal-agent"] extendable_tags = ["Label"] - semantic_hints = [ - {"tag": "Reward", "prefix": "", "score": 0.95, "source": "keyword"}, - {"tag": "Animal-agent", "prefix": "", "score": 0.85, "source": "embedding"}, - ] - - guide = get_comprehensive_hed_guide( - vocabulary, extendable_tags, semantic_hints=semantic_hints - ) - - assert "POTENTIALLY RELEVANT TAGS" in guide - assert "Reward" in guide - assert "Animal-agent" in guide - # Check confidence indicators - assert "high" in guide.lower() or "0.95" in guide - - def test_guide_with_semantic_hints_and_no_extend(self): - """Test guide with both semantic hints and no_extend.""" + + guide = get_comprehensive_hed_guide(vocabulary, extendable_tags) + + # System prompt should point to user message for hints (not contain them) + assert "SEMANTIC HINTS" in guide + assert "user message" in guide.lower() + + def test_guide_no_extend_with_hints_pointer(self): + """Test guide with no_extend has both hints pointer and extension warning.""" vocabulary = ["Event", "Visual-presentation"] extendable_tags = ["Label"] - semantic_hints = [ - {"tag": "Visual-presentation", "prefix": "", "score": 0.9, "source": "keyword"}, - ] - guide = get_comprehensive_hed_guide( - vocabulary, extendable_tags, semantic_hints=semantic_hints, no_extend=True - ) + guide = get_comprehensive_hed_guide(vocabulary, extendable_tags, no_extend=True) - # Should have both features - assert "POTENTIALLY RELEVANT TAGS" in guide + assert "SEMANTIC HINTS" in guide assert "EXTENSIONS STRICTLY PROHIBITED" in guide assert "(Extensions disabled)" in guide From 2a33a32823e051f76b743584a0314baaec3926ba Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Mon, 30 Mar 2026 02:01:22 -0700 Subject: [PATCH 2/2] Address review findings for cache-friendly prompts - Rename _format_semantic_hints to format_semantic_hints (public API, used cross-module) - Align header: system prompt pointer and actual section both say "SEMANTIC HINTS" - Soften system prompt wording to "may include" (hints are optional) - Skip hints with empty tag keys - Add debug logging when hints are included in user prompt - Add 10 tests: user prompt with/without hints, confidence bucketing, system prompt caching invariant --- src/agents/annotation_agent.py | 7 +- src/utils/hed_comprehensive_guide.py | 17 ++-- tests/test_annotation_agent.py | 126 +++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 11 deletions(-) diff --git a/src/agents/annotation_agent.py b/src/agents/annotation_agent.py index 24dcaf2..6bb65c7 100644 --- a/src/agents/annotation_agent.py +++ b/src/agents/annotation_agent.py @@ -12,7 +12,7 @@ from src.agents.state import HedAnnotationState from src.utils import extract_text_content -from src.utils.hed_comprehensive_guide import get_comprehensive_hed_guide +from src.utils.hed_comprehensive_guide import format_semantic_hints, get_comprehensive_hed_guide from src.utils.json_schema_loader import HedJsonSchemaLoader, load_latest_schema logger = logging.getLogger(__name__) @@ -112,9 +112,8 @@ def _format_semantic_hints(self, semantic_hints: list[dict] | None) -> str: if not semantic_hints: return "" - from src.utils.hed_comprehensive_guide import _format_semantic_hints - - return "\n" + _format_semantic_hints(semantic_hints) + logger.debug("Including %d semantic hints in user prompt", len(semantic_hints)) + return "\n" + format_semantic_hints(semantic_hints) def _build_user_prompt( self, diff --git a/src/utils/hed_comprehensive_guide.py b/src/utils/hed_comprehensive_guide.py index b5d2274..066e6d9 100644 --- a/src/utils/hed_comprehensive_guide.py +++ b/src/utils/hed_comprehensive_guide.py @@ -10,8 +10,8 @@ """ -def _format_semantic_hints(hints: list[dict]) -> str: - """Format semantic hints for inclusion in the guide. +def format_semantic_hints(hints: list[dict]) -> str: + """Format semantic hints for inclusion in the user prompt. Args: hints: List of semantic search results, each with: @@ -21,7 +21,7 @@ def _format_semantic_hints(hints: list[dict]) -> str: - prefix: Optional library prefix (e.g., "sc:") Returns: - Formatted hints section for the guide + Formatted hints section for the user prompt """ if not hints: return "" @@ -33,6 +33,8 @@ def _format_semantic_hints(hints: list[dict]) -> str: for hint in hints: tag = hint.get("tag", "") + if not tag: + continue prefix = hint.get("prefix", "") score = hint.get("score", 0) full_tag = f"{prefix}{tag}" if prefix else tag @@ -45,7 +47,7 @@ def _format_semantic_hints(hints: list[dict]) -> str: low_conf.append(full_tag) lines = [ - "## POTENTIALLY RELEVANT TAGS", + "## SEMANTIC HINTS", "", "Based on your description, these schema tags may be relevant.", "Note: this list may contain false positives - use your judgment.", @@ -1009,9 +1011,10 @@ def get_comprehensive_hed_guide( _build_correction_workflow_section(), ( "## SEMANTIC HINTS\n\n" - "Check the user message for a SEMANTIC HINTS section with " - "potentially relevant tags from schema search. Use these as " - "guidance for tag selection, but verify each against the vocabulary.\n\n" + "The user message may include a SEMANTIC HINTS section with " + "potentially relevant tags from schema search. If present, use " + "these as guidance for tag selection, but verify each against " + "the vocabulary. If no hints section is present, proceed without them.\n\n" "---\n\n" ), _build_semantic_rules_section(), diff --git a/tests/test_annotation_agent.py b/tests/test_annotation_agent.py index 3126770..39f9eef 100644 --- a/tests/test_annotation_agent.py +++ b/tests/test_annotation_agent.py @@ -193,6 +193,132 @@ def test_no_previous_annotation_when_none(self): assert "Previous annotation:" not in result + def test_first_pass_with_semantic_hints(self): + """Semantic hints should appear in the user prompt on first pass.""" + agent = self._make_agent() + result = agent._build_user_prompt( + "A dog chasing a cat", + semantic_hints=[ + {"tag": "Animal-agent", "score": 0.9, "source": "hed-lsp"}, + {"tag": "Chase", "score": 0.7, "source": "hed-lsp"}, + ], + ) + + assert "SEMANTIC HINTS" in result + assert "Animal-agent" in result + assert "Chase" in result + assert "A dog chasing a cat" in result + + def test_correction_pass_with_semantic_hints(self): + """Semantic hints should also appear in correction prompts.""" + agent = self._make_agent() + result = agent._build_user_prompt( + "A dog chasing a cat", + validation_errors=["[TAG_INVALID] 'Chase' is not valid"], + semantic_hints=[ + {"tag": "Animal-agent", "score": 0.9, "source": "hed-lsp"}, + ], + ) + + assert "SEMANTIC HINTS" in result + assert "Animal-agent" in result + assert "TAG_INVALID" in result + + def test_no_hints_no_hints_section(self): + """No hints should not add a hints section to the user prompt.""" + agent = self._make_agent() + result = agent._build_user_prompt("A red circle") + + assert "SEMANTIC HINTS" not in result + + def test_empty_hints_no_hints_section(self): + """Empty hints list should not add a hints section.""" + agent = self._make_agent() + result = agent._build_user_prompt("A red circle", semantic_hints=[]) + + assert "SEMANTIC HINTS" not in result + + +class TestFormatSemanticHints: + """Tests for _format_semantic_hints method on AnnotationAgent.""" + + def _make_agent(self): + agent = object.__new__(AnnotationAgent) + return agent + + def test_none_returns_empty(self): + """None input should return empty string.""" + agent = self._make_agent() + assert agent._format_semantic_hints(None) == "" + + def test_empty_list_returns_empty(self): + """Empty list should return empty string.""" + agent = self._make_agent() + assert agent._format_semantic_hints([]) == "" + + def test_valid_hints_returns_content(self): + """Valid hints should produce formatted output.""" + agent = self._make_agent() + result = agent._format_semantic_hints( + [ + {"tag": "Red", "score": 0.9, "source": "hed-lsp"}, + ] + ) + + assert "SEMANTIC HINTS" in result + assert "Red" in result + + def test_confidence_bucketing(self): + """Hints should be categorized by confidence level.""" + agent = self._make_agent() + result = agent._format_semantic_hints( + [ + {"tag": "HighTag", "score": 0.95, "source": "hed-lsp"}, + {"tag": "MedTag", "score": 0.6, "source": "hed-lsp"}, + {"tag": "LowTag", "score": 0.3, "source": "hed-lsp"}, + ] + ) + + assert "High confidence" in result + assert "HighTag" in result + assert "Medium confidence" in result + assert "MedTag" in result + assert "Lower confidence" in result + assert "LowTag" in result + + def test_skips_empty_tags(self): + """Hints with empty tag should be skipped.""" + agent = self._make_agent() + result = agent._format_semantic_hints( + [ + {"tag": "", "score": 0.9, "source": "hed-lsp"}, + {"tag": "ValidTag", "score": 0.8, "source": "hed-lsp"}, + ] + ) + + assert "ValidTag" in result + + +class TestSystemPromptCaching: + """Tests that system prompt is static (no dynamic content) for caching.""" + + def test_system_prompt_has_hints_pointer_not_content(self): + """System prompt should reference hints but not contain actual hint data.""" + from src.utils.hed_comprehensive_guide import get_comprehensive_hed_guide + + guide = get_comprehensive_hed_guide( + vocabulary_sample=["Red", "Circle"], + extendable_tags=["Animal"], + ) + + # Should have the pointer section + assert "## SEMANTIC HINTS" in guide + assert "may include" in guide + # Should NOT contain dynamic hint content + assert "High confidence" not in guide + assert "Medium confidence" not in guide + assert "Lower confidence" not in guide + class TestPromptSections: """Tests for prompt structure and sections."""