From 5b89c68747dd6f65863d6b02cfc3d885360dc635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 12 Aug 2026 08:31:57 +0200 Subject: [PATCH] fix(recall): fill source_facts budget in rank order and flag truncation (#3221) --- hindsight-api-slim/hindsight_api/api/http.py | 9 ++ .../hindsight_api/engine/memory_engine.py | 62 ++++---- .../hindsight_api/engine/response_models.py | 8 + .../hindsight_api/engine/source_facts.py | 94 ++++++++++++ .../tests/test_source_facts_selection.py | 143 ++++++++++++++++++ .../tests/test_source_facts_tokens.py | 51 ++++++- hindsight-clients/go/api/openapi.yaml | 3 + hindsight-clients/go/model_recall_response.go | 46 ++++++ .../models/recall_response.py | 13 +- .../typescript/generated/types.gen.ts | 6 + hindsight-docs/docs/developer/api/recall.mdx | 8 + hindsight-docs/static/openapi.json | 12 ++ .../references/developer/api/recall.md | 7 + skills/hindsight-docs/references/openapi.json | 12 ++ 14 files changed, 436 insertions(+), 38 deletions(-) create mode 100644 hindsight-api-slim/hindsight_api/engine/source_facts.py create mode 100644 hindsight-api-slim/tests/test_source_facts_selection.py diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index 83db56e1ef..6b81e4fbd8 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -598,6 +598,14 @@ class RecallResponse(BaseModel): source_facts: dict[str, RecallResult] | None = Field( default=None, description="Source facts for observation-type results, keyed by fact ID" ) + source_facts_truncated: bool | None = Field( + default=None, + description=( + "Whether the source_facts map was cut short by the token budget. When true, some IDs in " + "results[].source_fact_ids have no entry in source_facts — the budget ran out, the " + "references are not dangling. Only set when source facts were requested." + ), + ) class EntityInput(BaseModel): @@ -4651,6 +4659,7 @@ def _fact_to_result(fact: "MemoryFact") -> RecallResult: entities=entities_response, chunks=chunks_response, source_facts=source_facts_response, + source_facts_truncated=core_result.source_facts_truncated, ) handler_duration = time.time() - handler_start diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index b10cbbf6c6..cdf55d9943 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -456,6 +456,7 @@ def validate_sql_schema(sql: str) -> None: from .search.reranking import CrossEncoderReranker, apply_combined_scoring from .search.tags import TagGroup, TagsMatch, build_tag_groups_where_clause, build_tags_where_clause from .search.types import ScoredResult +from .source_facts import select_source_facts_within_budget from .task_backend import TaskBackend # Recall ranking strategy: how the per-arm (semantic/bm25/graph/temporal) results are @@ -6569,6 +6570,7 @@ def to_tuple_format(results): source_fact_start = time.time() source_fact_ids_by_obs: dict[str, list[str]] = {} # obs_id -> [source_id, ...] source_facts_dict: dict[str, MemoryFact] | None = None + source_facts_truncated = False if include_source_facts: observation_ids = [uuid.UUID(sr.id) for sr in top_scored if sr.retrieval.fact_type == "observation"] if observation_ids: @@ -6610,18 +6612,23 @@ def _source_fact_dict( # Resolve each observation's sources. This is a recall hot path, so the SQL # store reads only the two columns it needs rather than a full memory row; a # store that owns its rows answers from its own objects via one addressed read. + # + # Both branches keep observation-rank order: the token budget below is filled + # in this order, so an unordered read would let a low-ranked observation + # spend the budget the top-ranked one needs (issue #3221). if store.writes_memory_rows_in_sql_for(bank_id): obs_rows = [ {"id": str(r["id"]), "source_memory_ids": r["source_memory_ids"]} for r in await sf_conn.fetch( f"SELECT id, source_memory_ids FROM {fq_table('memory_units')} " - f"WHERE id = ANY($1::uuid[]) AND fact_type = 'observation'", + f"WHERE id = ANY($1::uuid[]) AND fact_type = 'observation' " + f"ORDER BY array_position($1::uuid[], id)", observation_ids, ) ] else: - obs_rows = [ - {"id": m.unit_id, "source_memory_ids": m.source_memory_ids} + obs_by_id = { + m.unit_id: m for m in await store.get_memories( conn=sf_conn, fq_table=fq_table, @@ -6629,6 +6636,11 @@ def _source_fact_dict( unit_ids=[str(o) for o in observation_ids], ) if m.fact_type == "observation" + } + obs_rows = [ + {"id": m.unit_id, "source_memory_ids": m.source_memory_ids} + for m in (obs_by_id.get(str(o)) for o in observation_ids) + if m is not None ] # Collect unique source IDs in order of first appearance @@ -6691,7 +6703,6 @@ def _source_fact_dict( } encoding = _get_tiktoken_encoding() - source_facts_dict = {} def _make_source_fact(sid: str, r: Any) -> MemoryFact: return MemoryFact( @@ -6708,36 +6719,18 @@ def _make_source_fact(sid: str, r: Any) -> MemoryFact: tags=r["tags"] or None, ) - if max_source_facts_tokens_per_observation >= 0: - # Per-observation capping: each observation independently selects - # source facts up to its token budget. - for obs_id, sids in source_fact_ids_by_obs.items(): - obs_tokens = 0 - for sid in sids: - if sid not in source_row_by_id: - continue - r = source_row_by_id[sid] - fact_tokens = len(encoding.encode(r["text"])) - if obs_tokens + fact_tokens > max_source_facts_tokens_per_observation: - break - obs_tokens += fact_tokens - if sid not in source_facts_dict: - source_facts_dict[sid] = _make_source_fact(sid, r) - else: - # Global budget: fill in order of first appearance until exhausted. - total_source_tokens = 0 - for sid in source_ids_ordered: - if sid not in source_row_by_id: - continue - r = source_row_by_id[sid] - fact_tokens = len(encoding.encode(r["text"])) - if ( - max_source_facts_tokens >= 0 - and total_source_tokens + fact_tokens > max_source_facts_tokens - ): - break - source_facts_dict[sid] = _make_source_fact(sid, r) - total_source_tokens += fact_tokens + selection = select_source_facts_within_budget( + source_ids_ordered=source_ids_ordered, + source_fact_ids_by_obs=source_fact_ids_by_obs, + text_by_id={sid: r["text"] for sid, r in source_row_by_id.items()}, + max_total_tokens=max_source_facts_tokens, + max_tokens_per_observation=max_source_facts_tokens_per_observation, + count_tokens=lambda text: len(encoding.encode(text)), + ) + source_facts_truncated = selection.truncated + source_facts_dict = { + sid: _make_source_fact(sid, source_row_by_id[sid]) for sid in selection.ids + } # Source-fact enrichment is two SQL passes + tiktoken encoding; record it # only when requested (issue #2361). @@ -6901,6 +6894,7 @@ def _make_source_fact(sid: str, r: Any) -> MemoryFact: entities=entities_dict, chunks=chunks_dict, source_facts=source_facts_dict, + source_facts_truncated=source_facts_truncated if include_source_facts else None, ) except OperationCancelledError: diff --git a/hindsight-api-slim/hindsight_api/engine/response_models.py b/hindsight-api-slim/hindsight_api/engine/response_models.py index 1610c3dc31..bf31b28592 100644 --- a/hindsight-api-slim/hindsight_api/engine/response_models.py +++ b/hindsight-api-slim/hindsight_api/engine/response_models.py @@ -353,6 +353,14 @@ class RecallResult(BaseModel): source_facts: dict[str, MemoryFact] | None = Field( None, description="Source facts for observation-type results, keyed by fact ID" ) + source_facts_truncated: bool | None = Field( + None, + description=( + "Whether the source_facts map was cut short by the token budget. When true, some IDs in " + "results[].source_fact_ids have no entry in source_facts — the budget ran out, the " + "references are not dangling. Only set when source facts were requested." + ), + ) class ReflectResult(BaseModel): diff --git a/hindsight-api-slim/hindsight_api/engine/source_facts.py b/hindsight-api-slim/hindsight_api/engine/source_facts.py new file mode 100644 index 0000000000..6f1e14cca0 --- /dev/null +++ b/hindsight-api-slim/hindsight_api/engine/source_facts.py @@ -0,0 +1,94 @@ +"""Token-budget selection for the recall ``source_facts`` map. + +Recall returns each observation's ``source_fact_ids`` in full and resolves them +through a ``source_facts`` map that is filled up to a token budget. Which facts +make it into that map is what this module decides. + +Two properties matter to callers resolving provenance (issue #3221): + +* the budget is spent in **observation-rank order**, so truncation hits the tail + of the result list and the top-ranked results keep their provenance; +* one oversized fact skips itself only — it does not evict every shorter fact + behind it — and any skip is reported so the caller can tell a budget-truncated + map from a dangling reference. +""" + +from collections.abc import Callable +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SourceFactSelection: + """The source facts that fit the budget, plus whether anything was dropped.""" + + ids: list[str] + """Selected source fact IDs, in the order they were considered.""" + + truncated: bool + """True when at least one resolvable source fact was skipped for budget.""" + + +def select_source_facts_within_budget( + *, + source_ids_ordered: list[str], + source_fact_ids_by_obs: dict[str, list[str]], + text_by_id: dict[str, str], + max_total_tokens: int, + max_tokens_per_observation: int, + count_tokens: Callable[[str], int], +) -> SourceFactSelection: + """Pick the source facts that fit the requested token budget. + + ``source_ids_ordered`` and ``source_fact_ids_by_obs`` must both be in + observation-rank order — the budget is spent front to back, so their order + decides which results keep their provenance when the budget runs out. + + A non-negative ``max_tokens_per_observation`` gives every observation its own + budget (the global one is then unused); otherwise a single ``max_total_tokens`` + budget is shared, with a negative value meaning unlimited. IDs absent from + ``text_by_id`` are skipped without counting as truncation: those rows did not + resolve at all, which is a different condition from running out of budget. + """ + selected: list[str] = [] + seen: set[str] = set() + # Skipped for budget, not necessarily lost: under a per-observation cap the same + # source can be dropped by one observation's budget and still fit another's. Only + # a skip that leaves the fact out of the final map counts as truncation, so the + # flag means exactly what it says — some advertised ID has no entry. + skipped: set[str] = set() + + def _take(sid: str) -> None: + if sid not in seen: + seen.add(sid) + selected.append(sid) + + if max_tokens_per_observation >= 0: + # Per-observation capping: each observation independently selects source + # facts up to its own budget. + for sids in source_fact_ids_by_obs.values(): + obs_tokens = 0 + for sid in sids: + text = text_by_id.get(sid) + if text is None: + continue + fact_tokens = count_tokens(text) + if obs_tokens + fact_tokens > max_tokens_per_observation: + skipped.add(sid) + continue + obs_tokens += fact_tokens + _take(sid) + return SourceFactSelection(ids=selected, truncated=bool(skipped - seen)) + + total_tokens = 0 + for sid in source_ids_ordered: + text = text_by_id.get(sid) + if text is None: + continue + fact_tokens = count_tokens(text) + if max_total_tokens >= 0 and total_tokens + fact_tokens > max_total_tokens: + skipped.add(sid) + continue + total_tokens += fact_tokens + _take(sid) + + return SourceFactSelection(ids=selected, truncated=bool(skipped - seen)) diff --git a/hindsight-api-slim/tests/test_source_facts_selection.py b/hindsight-api-slim/tests/test_source_facts_selection.py new file mode 100644 index 0000000000..d2ad5d61cb --- /dev/null +++ b/hindsight-api-slim/tests/test_source_facts_selection.py @@ -0,0 +1,143 @@ +"""Tests for the recall source_facts token-budget selection (issue #3221). + +The budget must be spent in observation-rank order, an oversized fact must skip +only itself, and any budget skip must be reported so a caller can tell a +truncated map from a dangling reference. +""" + +from hindsight_api.engine.source_facts import select_source_facts_within_budget + + +def _count_words(text: str) -> int: + """Stand-in tokenizer: one token per word, so budgets read literally.""" + return len(text.split()) + + +def _select( + *, + by_obs: dict[str, list[str]], + texts: dict[str, str], + max_total_tokens: int = -1, + max_tokens_per_observation: int = -1, +): + ordered: list[str] = [] + seen: set[str] = set() + for sids in by_obs.values(): + for sid in sids: + if sid not in seen: + seen.add(sid) + ordered.append(sid) + return select_source_facts_within_budget( + source_ids_ordered=ordered, + source_fact_ids_by_obs=by_obs, + text_by_id=texts, + max_total_tokens=max_total_tokens, + max_tokens_per_observation=max_tokens_per_observation, + count_tokens=_count_words, + ) + + +class TestGlobalBudget: + def test_budget_is_spent_in_rank_order(self): + """The top-ranked observation keeps its provenance; the tail loses it.""" + by_obs = {"obs-rank-1": ["s1"], "obs-rank-2": ["s2"], "obs-rank-3": ["s3"]} + texts = {"s1": "one two", "s2": "three four", "s3": "five six"} + + selection = _select(by_obs=by_obs, texts=texts, max_total_tokens=2) + + assert selection.ids == ["s1"] + assert selection.truncated is True + + def test_unlimited_budget_keeps_everything(self): + by_obs = {"obs-1": ["s1", "s2"], "obs-2": ["s3"]} + texts = {"s1": "a b c", "s2": "d e f", "s3": "g h i"} + + selection = _select(by_obs=by_obs, texts=texts, max_total_tokens=-1) + + assert selection.ids == ["s1", "s2", "s3"] + assert selection.truncated is False + + def test_oversized_fact_does_not_evict_the_facts_behind_it(self): + """One long fact skips itself only — shorter facts behind it still fit.""" + by_obs = {"obs-1": ["long"], "obs-2": ["short-a"], "obs-3": ["short-b"]} + texts = {"long": "w " * 50, "short-a": "a", "short-b": "b"} + + selection = _select(by_obs=by_obs, texts=texts, max_total_tokens=2) + + assert selection.ids == ["short-a", "short-b"] + assert selection.truncated is True + + def test_shared_source_counted_once(self): + """A source cited by two observations is selected (and charged) once.""" + by_obs = {"obs-1": ["shared"], "obs-2": ["shared", "s2"]} + texts = {"shared": "a b", "s2": "c d"} + + selection = _select(by_obs=by_obs, texts=texts, max_total_tokens=4) + + assert selection.ids == ["shared", "s2"] + assert selection.truncated is False + + def test_unresolvable_id_is_not_truncation(self): + """A source row that did not resolve is skipped without flagging truncation.""" + by_obs = {"obs-1": ["s1", "missing"]} + texts = {"s1": "a b"} + + selection = _select(by_obs=by_obs, texts=texts, max_total_tokens=-1) + + assert selection.ids == ["s1"] + assert selection.truncated is False + + +class TestPerObservationBudget: + def test_each_observation_gets_its_own_budget(self): + by_obs = {"obs-1": ["s1"], "obs-2": ["s2"]} + texts = {"s1": "a b", "s2": "c d"} + + selection = _select(by_obs=by_obs, texts=texts, max_tokens_per_observation=2) + + assert selection.ids == ["s1", "s2"] + assert selection.truncated is False + + def test_oversized_fact_does_not_evict_the_facts_behind_it(self): + by_obs = {"obs-1": ["long", "short"]} + texts = {"long": "w " * 50, "short": "a"} + + selection = _select(by_obs=by_obs, texts=texts, max_tokens_per_observation=1) + + assert selection.ids == ["short"] + assert selection.truncated is True + + def test_per_observation_cap_takes_precedence_over_global(self): + """With a per-observation cap set, the global budget does not apply.""" + by_obs = {"obs-1": ["s1"], "obs-2": ["s2"]} + texts = {"s1": "a b", "s2": "c d"} + + selection = _select( + by_obs=by_obs, + texts=texts, + max_total_tokens=2, + max_tokens_per_observation=2, + ) + + assert selection.ids == ["s1", "s2"] + assert selection.truncated is False + + def test_fact_dropped_by_one_observation_but_kept_by_another_is_not_truncation(self): + """A source that still lands in the map was not lost, so nothing is flagged.""" + by_obs = {"obs-1": ["filler", "shared"], "obs-2": ["shared"]} + texts = {"filler": "a", "shared": "b"} + + # obs-1 spends its whole budget on "filler" and skips "shared"; obs-2 has room. + selection = _select(by_obs=by_obs, texts=texts, max_tokens_per_observation=1) + + assert set(selection.ids) == {"filler", "shared"} + assert selection.truncated is False + + def test_zero_cap_drops_every_fact_and_reports_truncation(self): + by_obs = {"obs-1": ["s1"]} + texts = {"s1": "a"} + + selection = _select(by_obs=by_obs, texts=texts, max_tokens_per_observation=0) + + assert selection.ids == [] + assert selection.truncated is True diff --git a/hindsight-api-slim/tests/test_source_facts_tokens.py b/hindsight-api-slim/tests/test_source_facts_tokens.py index e5b5dcce18..50466b6a4e 100644 --- a/hindsight-api-slim/tests/test_source_facts_tokens.py +++ b/hindsight-api-slim/tests/test_source_facts_tokens.py @@ -11,7 +11,7 @@ import pytest from hindsight_api.config import _get_raw_config -from hindsight_api.engine.memory_engine import Budget +from hindsight_api.engine.memory_engine import Budget, _get_tiktoken_encoding @pytest.fixture(autouse=True) @@ -168,3 +168,52 @@ async def test_no_source_facts_without_flag(self, memory, request_context): assert result.source_facts is None or len(result.source_facts) == 0 finally: await memory.delete_bank(bank_id, request_context=request_context) + + +class TestRecallSourceFactsRankOrder: + """The token budget is spent in result-rank order, not DB row order (issue #3221).""" + + @pytest.mark.asyncio + async def test_top_ranked_result_keeps_its_provenance(self, memory, request_context): + """A budget too small for every source still resolves the top result's sources.""" + bank_id = "test-sf-rank-order" + try: + await _setup_bank_with_observations(memory, bank_id, request_context) + + recall_kwargs = dict( + bank_id=bank_id, + query="Alice engineer", + fact_type=["observation"], + max_tokens=4096, + include_source_facts=True, + budget=Budget.MID, + request_context=request_context, + ) + unlimited = await memory.recall_async(**recall_kwargs, max_source_facts_tokens=-1) + + with_sources = [r for r in unlimited.results if r.source_fact_ids] + assert len(with_sources) >= 2, "fixture must produce several observations with sources" + assert unlimited.source_facts, "unlimited recall must resolve source facts" + assert unlimited.source_facts_truncated is False + + # Budget the top result's sources exactly: everything behind it must be + # what gets dropped. + encoding = _get_tiktoken_encoding() + top_ids = with_sources[0].source_fact_ids + budget = sum(len(encoding.encode(unlimited.source_facts[sid].text)) for sid in top_ids) + + tight = await memory.recall_async(**recall_kwargs, max_source_facts_tokens=budget) + + tight_top = next(r for r in tight.results if r.id == with_sources[0].id) + assert tight.source_facts is not None + assert set(tight_top.source_fact_ids) <= set(tight.source_facts), ( + "the top-ranked result lost provenance to a lower-ranked one" + ) + + # Every remaining result still advertises all of its sources, so a partial + # map has to say it is partial. + advertised = {sid for r in tight.results for sid in (r.source_fact_ids or [])} + if advertised - set(tight.source_facts): + assert tight.source_facts_truncated is True + finally: + await memory.delete_bank(bank_id, request_context=request_context) diff --git a/hindsight-clients/go/api/openapi.yaml b/hindsight-clients/go/api/openapi.yaml index 37c5650fe5..7ddb58d5d4 100644 --- a/hindsight-clients/go/api/openapi.yaml +++ b/hindsight-clients/go/api/openapi.yaml @@ -9051,6 +9051,9 @@ components: additionalProperties: $ref: '#/components/schemas/RecallResult' nullable: true + source_facts_truncated: + nullable: true + type: boolean required: - results title: RecallResponse diff --git a/hindsight-clients/go/model_recall_response.go b/hindsight-clients/go/model_recall_response.go index f2385d4a7f..ea34f0b4a0 100644 --- a/hindsight-clients/go/model_recall_response.go +++ b/hindsight-clients/go/model_recall_response.go @@ -26,6 +26,7 @@ type RecallResponse struct { Entities map[string]EntityStateResponse `json:"entities,omitempty"` Chunks map[string]ChunkData `json:"chunks,omitempty"` SourceFacts map[string]RecallResult `json:"source_facts,omitempty"` + SourceFactsTruncated NullableBool `json:"source_facts_truncated,omitempty"` } type _RecallResponse RecallResponse @@ -204,6 +205,48 @@ func (o *RecallResponse) SetSourceFacts(v map[string]RecallResult) { o.SourceFacts = v } +// GetSourceFactsTruncated returns the SourceFactsTruncated field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *RecallResponse) GetSourceFactsTruncated() bool { + if o == nil || IsNil(o.SourceFactsTruncated.Get()) { + var ret bool + return ret + } + return *o.SourceFactsTruncated.Get() +} + +// GetSourceFactsTruncatedOk returns a tuple with the SourceFactsTruncated field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *RecallResponse) GetSourceFactsTruncatedOk() (*bool, bool) { + if o == nil { + return nil, false + } + return o.SourceFactsTruncated.Get(), o.SourceFactsTruncated.IsSet() +} + +// HasSourceFactsTruncated returns a boolean if a field has been set. +func (o *RecallResponse) HasSourceFactsTruncated() bool { + if o != nil && o.SourceFactsTruncated.IsSet() { + return true + } + + return false +} + +// SetSourceFactsTruncated gets a reference to the given NullableBool and assigns it to the SourceFactsTruncated field. +func (o *RecallResponse) SetSourceFactsTruncated(v bool) { + o.SourceFactsTruncated.Set(&v) +} +// SetSourceFactsTruncatedNil sets the value for SourceFactsTruncated to be an explicit nil +func (o *RecallResponse) SetSourceFactsTruncatedNil() { + o.SourceFactsTruncated.Set(nil) +} + +// UnsetSourceFactsTruncated ensures that no value is present for SourceFactsTruncated, not even an explicit nil +func (o *RecallResponse) UnsetSourceFactsTruncated() { + o.SourceFactsTruncated.Unset() +} + func (o RecallResponse) MarshalJSON() ([]byte, error) { toSerialize,err := o.ToMap() if err != nil { @@ -227,6 +270,9 @@ func (o RecallResponse) ToMap() (map[string]interface{}, error) { if o.SourceFacts != nil { toSerialize["source_facts"] = o.SourceFacts } + if o.SourceFactsTruncated.IsSet() { + toSerialize["source_facts_truncated"] = o.SourceFactsTruncated.Get() + } return toSerialize, nil } diff --git a/hindsight-clients/python/hindsight_client_api/models/recall_response.py b/hindsight-clients/python/hindsight_client_api/models/recall_response.py index e4e6a59375..e815f1a3ef 100644 --- a/hindsight-clients/python/hindsight_client_api/models/recall_response.py +++ b/hindsight-clients/python/hindsight_client_api/models/recall_response.py @@ -17,7 +17,7 @@ import re # noqa: F401 import json -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, StrictBool from typing import Any, ClassVar, Dict, List, Optional from hindsight_client_api.models.chunk_data import ChunkData from hindsight_client_api.models.entity_state_response import EntityStateResponse @@ -34,7 +34,8 @@ class RecallResponse(BaseModel): entities: Optional[Dict[str, EntityStateResponse]] = None chunks: Optional[Dict[str, ChunkData]] = None source_facts: Optional[Dict[str, RecallResult]] = None - __properties: ClassVar[List[str]] = ["results", "trace", "entities", "chunks", "source_facts"] + source_facts_truncated: Optional[StrictBool] = None + __properties: ClassVar[List[str]] = ["results", "trace", "entities", "chunks", "source_facts", "source_facts_truncated"] model_config = ConfigDict( populate_by_name=True, @@ -123,6 +124,11 @@ def to_dict(self) -> Dict[str, Any]: if self.source_facts is None and "source_facts" in self.model_fields_set: _dict['source_facts'] = None + # set to None if source_facts_truncated (nullable) is None + # and model_fields_set contains the field + if self.source_facts_truncated is None and "source_facts_truncated" in self.model_fields_set: + _dict['source_facts_truncated'] = None + return _dict @classmethod @@ -154,7 +160,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: for _k, _v in obj["source_facts"].items() ) if obj.get("source_facts") is not None - else None + else None, + "source_facts_truncated": obj.get("source_facts_truncated") }) return _obj diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index 8617d1f6b4..8edca5312c 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -4089,6 +4089,12 @@ export type RecallResponse = { source_facts?: { [key: string]: RecallResult; } | null; + /** + * Source Facts Truncated + * + * Whether the source_facts map was cut short by the token budget. When true, some IDs in results[].source_fact_ids have no entry in source_facts — the budget ran out, the references are not dangling. Only set when source facts were requested. + */ + source_facts_truncated?: boolean | null; }; /** diff --git a/hindsight-docs/docs/developer/api/recall.mdx b/hindsight-docs/docs/developer/api/recall.mdx index 425fd2693f..35926c00be 100644 --- a/hindsight-docs/docs/developer/api/recall.mdx +++ b/hindsight-docs/docs/developer/api/recall.mdx @@ -146,6 +146,10 @@ When `include_chunks` is enabled, chunks are fetched based on the top-scored rer When enabled and `types` includes `observation`, each observation result is accompanied by the original contributing facts it was synthesized from. Source facts are returned in a top-level `source_facts` dict keyed by fact ID, and each observation result carries a `source_fact_ids` list for cross-referencing. Facts are deduplicated across observations. The `max_tokens` sub-option (default `4096`) limits the total token budget for source facts. +:::note +The budget is spent in result order, so when it runs out it is the lowest-ranked results that lose their source facts — the top results always keep theirs. `source_fact_ids` always lists every source, so an ID may have no entry in `source_facts`; the response sets `source_facts_truncated: true` when that is the budget's doing rather than a missing fact. Raise `max_tokens` (or set it to `-1`) if you need every source resolved. +::: + @@ -480,6 +484,10 @@ Each field is also a valid [`min_scores`](#min_scores) floor. A dict keyed by fact ID containing full `RecallResult` objects for the source facts that contributed to observation results. Only present when `include.source_facts` is enabled. Facts are deduplicated — if two observations share a source fact, it appears once. +### source_facts_truncated + +Whether the token budget cut the `source_facts` map short. When `true`, some IDs in `results[].source_fact_ids` have no entry in `source_facts` because the budget ran out — the references are not dangling. Only present when `include.source_facts` is enabled. + ### chunks A dict keyed by chunk ID containing the raw source text chunks associated with the returned facts. Only present when `include.chunks` is enabled. Each chunk has `id`, `text`, `chunk_index`, and `truncated` (whether the text was cut to fit the token budget). diff --git a/hindsight-docs/static/openapi.json b/hindsight-docs/static/openapi.json index f4622a0ee1..fcb1dc3651 100644 --- a/hindsight-docs/static/openapi.json +++ b/hindsight-docs/static/openapi.json @@ -13560,6 +13560,18 @@ ], "title": "Source Facts", "description": "Source facts for observation-type results, keyed by fact ID" + }, + "source_facts_truncated": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Source Facts Truncated", + "description": "Whether the source_facts map was cut short by the token budget. When true, some IDs in results[].source_fact_ids have no entry in source_facts \u2014 the budget ran out, the references are not dangling. Only set when source facts were requested." } }, "type": "object", diff --git a/skills/hindsight-docs/references/developer/api/recall.md b/skills/hindsight-docs/references/developer/api/recall.md index fa16e3c0c1..926c86bfb2 100644 --- a/skills/hindsight-docs/references/developer/api/recall.md +++ b/skills/hindsight-docs/references/developer/api/recall.md @@ -258,6 +258,9 @@ When `include_chunks` is enabled, chunks are fetched based on the top-scored rer When enabled and `types` includes `observation`, each observation result is accompanied by the original contributing facts it was synthesized from. Source facts are returned in a top-level `source_facts` dict keyed by fact ID, and each observation result carries a `source_fact_ids` list for cross-referencing. Facts are deduplicated across observations. The `max_tokens` sub-option (default `4096`) limits the total token budget for source facts. +> **📝 Note** +> +The budget is spent in result order, so when it runs out it is the lowest-ranked results that lose their source facts — the top results always keep theirs. `source_fact_ids` always lists every source, so an ID may have no entry in `source_facts`; the response sets `source_facts_truncated: true` when that is the budget's doing rather than a missing fact. Raise `max_tokens` (or set it to `-1`) if you need every source resolved. ### Python ```python @@ -724,6 +727,10 @@ Each field is also a valid [`min_scores`](#min_scores) floor. A dict keyed by fact ID containing full `RecallResult` objects for the source facts that contributed to observation results. Only present when `include.source_facts` is enabled. Facts are deduplicated — if two observations share a source fact, it appears once. +### source_facts_truncated + +Whether the token budget cut the `source_facts` map short. When `true`, some IDs in `results[].source_fact_ids` have no entry in `source_facts` because the budget ran out — the references are not dangling. Only present when `include.source_facts` is enabled. + ### chunks A dict keyed by chunk ID containing the raw source text chunks associated with the returned facts. Only present when `include.chunks` is enabled. Each chunk has `id`, `text`, `chunk_index`, and `truncated` (whether the text was cut to fit the token budget). diff --git a/skills/hindsight-docs/references/openapi.json b/skills/hindsight-docs/references/openapi.json index f4622a0ee1..fcb1dc3651 100644 --- a/skills/hindsight-docs/references/openapi.json +++ b/skills/hindsight-docs/references/openapi.json @@ -13560,6 +13560,18 @@ ], "title": "Source Facts", "description": "Source facts for observation-type results, keyed by fact ID" + }, + "source_facts_truncated": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Source Facts Truncated", + "description": "Whether the source_facts map was cut short by the token budget. When true, some IDs in results[].source_fact_ids have no entry in source_facts \u2014 the budget ran out, the references are not dangling. Only set when source facts were requested." } }, "type": "object",