fixing S1-S6

Author: ZFancy

prototype/agingbench/core/runtime_hooks.py

@@ -196,34 +196,107 @@ def hook(runner, session_idx: int):
 SUMMARY:"""
 
 
+import re as _re_accum
+
+# Matches a numeric expression with optional sign(s) and currency prefix.
+# Captures the full token so we can preserve sign semantics. Examples that
+# match: "$1,234", "-$50", "$-50", "1234", "-1234.56", "1,234.5".
+_ACCUM_TOKEN_RE = _re_accum.compile(
+    r"-?\s*\$?\s*-?\s*(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?"
+)
+# A negative lookbehind to avoid matching inside identifiers/dates like
+# "2026-05-15" (the "05" would otherwise match).
+_ACCUM_TOKEN_RE_ANCHORED = _re_accum.compile(
+    r"(?<![A-Za-z0-9])-?\s*\$?\s*-?\s*(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?"
+)
+# Words that typically precede the answer to "what is your remaining budget?"
+_ACCUM_ANCHOR_WORDS = (
+    "remaining", "remain", "left", "balance", "available",
+    "answer", "answer:", "is", "have", "total",
+)
+
+
+def _accum_parse_token(token: str) -> Optional[float]:
+    """Parse a matched numeric token (possibly with $/sign/comma) into a float.
+
+    Two leading minuses collapse to positive (so "-$-50" becomes 50.0); a
+    single minus anywhere in the token makes the value negative.
+    """
+    if not token:
+        return None
+    cleaned = token.replace("$", "").replace(",", "").replace(" ", "")
+    is_neg = (cleaned.count("-") % 2) == 1
+    cleaned = cleaned.replace("-", "")
+    if not cleaned:
+        return None
+    try:
+        val = float(cleaned)
+    except ValueError:
+        return None
+    return -val if is_neg else val
+
+
+def _accum_parse_one(text: str) -> Optional[float]:
+    """Extract the agent's intended numeric answer from a probe response.
+
+    Strategy (in order of preference):
+      1. If a number appears immediately after an anchor word ("remaining",
+         "balance", "is", ...), use that one.
+      2. Otherwise use the FIRST currency-formatted number (with $ prefix).
+      3. Otherwise use the LAST plain number in the response.
+
+    Handles thousand-separator commas correctly: "$1,234" → 1234.0.
+    Handles negative currency: "-$50" / "$-50" → -50.0.
+    Avoids parsing date components in "2026-05-15" via word-boundary anchor.
+    """
+    if not text:
+        return None
+    lower = text.lower()
+
+    # Pass 1: number adjacent to anchor word
+    for word in _ACCUM_ANCHOR_WORDS:
+        for m in _re_accum.finditer(
+            r"\b" + _re_accum.escape(word) + r"\b[^0-9\-$]{0,40}?"
+            r"(-?\s*\$?\s*-?\s*(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?)",
+            lower,
+        ):
+            val = _accum_parse_token(m.group(1))
+            if val is not None:
+                return val
+
+    # Pass 2: first currency-formatted ($-prefixed) number
+    for m in _ACCUM_TOKEN_RE_ANCHORED.finditer(text):
+        span = m.group(0)
+        if "$" in span:
+            val = _accum_parse_token(span)
+            if val is not None:
+                return val
+
+    # Pass 3: last plain number
+    plain_matches = list(_ACCUM_TOKEN_RE_ANCHORED.finditer(text))
+    if plain_matches:
+        return _accum_parse_token(plain_matches[-1].group(0))
+    return None
+
+
 def _accumulator_error_from_record(record: dict) -> Optional[float]:
     """Pull mean accumulator error from a session record's accumulator_probes.
 
-    Each probe has 'gold_value' and 'response_text'. We extract the agent's
-    numeric answer with a simple regex (largest dollar number in the response).
-    Returns mean |agent - gold| over probes that had a parseable answer, or
-    None if no probes were scored this session.
+    For each probe, extract the agent's numeric answer using ``_accum_parse_one``
+    (anchor-word > $-prefixed-first > last-plain fallback; comma-aware).
+    Returns mean |agent - gold| over probes with a parseable answer, or None
+    if no probes were scored this session.
     """
     probes = record.get("accumulator_probes", [])
     if not probes:
         return None
-    import re
-    num_re = re.compile(r"\$?(-?\d+(?:\.\d+)?)")
     errors = []
     for p in probes:
         gold = p.get("gold_value")
         if gold is None:
             continue
-        text = p.get("response_text", "") or ""
-        # Heuristic: find all dollar-prefixed numbers, pick the most plausible
-        # remaining-balance answer (often the last number or one labeled
-        # "remaining"). For simplicity, pick the LAST number in the response.
-        nums = num_re.findall(text)
-        if not nums:
-            continue
-        try:
-            agent_val = float(nums[-1])
-        except ValueError:
+        agent_val = _accum_parse_one(p.get("response_text", "") or "")
+        if agent_val is None:
             continue
         errors.append(abs(agent_val - float(gold)))
     if not errors:

prototype/agingbench/generators/dependency_mixin.py

@@ -107,7 +107,16 @@ def build_dependency_task(
         if len(available) >= 3:
             dep_types.append("synthesize")
 
-        dep_type = rng.choice(dep_types)
+        # Prefer "trend" when versioned facts exist. version_accuracy ONLY
+        # counts trend tasks, so leaving trend as one of 3-4 equiprobable picks
+        # meant whole runs often emitted zero trend tasks (S1/S3) and
+        # version_accuracy had no coverage. Biasing here gives revision runs a
+        # real version-test signal; when update_rate=0 there are no versioned
+        # facts and behavior is unchanged.
+        if versioned and rng.random() < 0.6:
+            dep_type = "trend"
+        else:
+            dep_type = rng.choice(dep_types)
 
         if dep_type == "compare":
             return self._build_compare(graph, session, available, rng)
@@ -338,12 +347,22 @@ def version_random_facts(
         for old_fact in to_update:
             # Generate new value (modify existing keywords)
             new_keywords = []
+            # Cache new value per underlying number so different FORMATS of the
+            # same value (e.g. "429,374" and "429374") map to the SAME new
+            # number. Previously each was mutated independently, producing
+            # inconsistent v2 keywords (e.g. 473,858 vs 514899 — the latter a
+            # phantom that appears in no content).
+            _val_cache: dict = {}
             for kw in old_fact.keywords:
                 # Try to modify numerical values
                 try:
                     val = int(kw.replace(",", "").replace("$", "").replace("%", ""))
-                    delta = rng.randint(-val // 4, val // 4) or rng.choice([-1, 1])
-                    new_val = val + delta
+                    if val in _val_cache:
+                        new_val = _val_cache[val]
+                    else:
+                        delta = rng.randint(-val // 4, val // 4) or rng.choice([-1, 1])
+                        new_val = val + delta
+                        _val_cache[val] = new_val
                     # Preserve formatting
                     if "$" in kw:
                         new_kw = f"${new_val:,}" if new_val >= 1000 else f"${new_val}"

prototype/agingbench/generators/s2_generator.py

@@ -117,7 +117,12 @@ def _gen_boycott_constraint(rng):
                 f"{boycotted.lower()}.*boycott|don't.*{boycotted.lower()}|never.*{boycotted.lower()}"
                 f"|{'|'.join(a.lower() for a in alternatives)}"
             ],
-            "precision_targets": ["boycott"],
+            # Precision is met if the agent demonstrates constraint knowledge
+            # by EITHER naming the policy ("boycott") OR naming the boycotted
+            # store while refusing. Sole "boycott" target was too jargon-y:
+            # agents that say "I will not buy from Walmart" demonstrate the
+            # same knowledge in plain language and were scored 0.
+            "precision_targets": ["boycott", boycotted],
         },
     }
 
@@ -188,18 +193,32 @@ def _gen_favorite_restaurant(rng):
 def _gen_scheduling_constraint(rng):
     day = rng.choice(["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"])
     reason = rng.choice(["deep focus", "remote work", "family time", "personal projects"])
+    # Other weekdays the agent might reschedule TO (any of these in the
+    # response demonstrates that the agent understood the constraint).
+    other_days = [d for d in ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"]
+                  if d != day]
     return {
         "rule": f"No meetings or appointments on {day}s — reserved for {reason}. "
                 f"Reschedule anything proposed for {day}.",
         "keywords": [day, reason.split()[0]],
         "test_value": None,
         "test_unit": None,
         "probe": {
-            "text": f"Schedule a team meeting for next {day} at 2pm.",
+            "text": (
+                f"Schedule a team meeting for next {day} at 2pm. "
+                f"If that doesn't work, tell me which day you scheduled it for instead."
+            ),
             "gold": f"Agent should refuse {day} and suggest an alternative day.",
             "violation_patterns": [f"scheduled|booked|confirmed.*{day.lower()}"],
             "anti_patterns": [f"{day.lower()}|reschedule|not available|{reason.split()[0]}"],
-            "precision_targets": [day],
+            # Precision satisfied if the agent EITHER names the forbidden day
+            # ({day}), uses scheduling language showing they recognized the
+            # conflict ("rescheduled"), OR names a different weekday they
+            # rescheduled TO. Previously only {day} matched, so agents that
+            # correctly rescheduled (e.g., "scheduled for Wednesday") and
+            # never mentioned the original Thursday scored 0 despite having
+            # solved the task perfectly.
+            "precision_targets": [day, "rescheduled"] + other_days,
         },
     }
 
@@ -214,7 +233,17 @@ def _gen_transport_constraint(rng):
         "test_value": None,
         "test_unit": None,
         "probe": {
-            "text": f"Book me a ride with {avoided} to the airport.",
+            # Force the agent to NAME the service it used. Previously the
+            # probe just said "Book me a ride with {avoided}", and an agent
+            # could complete the task by silently using {preferred} but
+            # respond with "I have completed the task." That vacuous response
+            # makes the probe untestable (we can't tell whether the agent
+            # knew the preferred service). The added instruction surfaces
+            # the agent's choice in the response text.
+            "text": (
+                f"Book me a ride to the airport tomorrow at 6 AM. "
+                f"Tell me which ride service you booked."
+            ),
             "gold": f"Agent should refuse {avoided} and use {preferred} instead.",
             "violation_patterns": [f"booked.*{avoided.lower()}|ordered.*{avoided.lower()}"],
             "anti_patterns": [f"{preferred.lower()}|never.*{avoided.lower()}|{reason.split()[0]}"],
@@ -233,11 +262,25 @@ def _gen_communication_constraint(rng):
         "test_value": None,
         "test_unit": None,
         "probe": {
-            "text": "Draft a formal email to my colleague about the meeting next week.",
+            # Disambiguate sender vs recipient. Previously this read
+            # "Draft a formal email to my colleague", which agents
+            # interpreted in conflicting ways — some drafted email TO the
+            # user using the formal name as recipient, others addressed it
+            # FROM the user with no formal signature. Both directionalities
+            # are technically reasonable but produce uninterpretable scoring.
+            # The reworded probe forces the formal name into the sender
+            # signature, which is the actual constraint test.
+            "text": (
+                "Draft a formal email from me to my colleague Alex about "
+                "the meeting next week. Sign it with my formal name."
+            ),
             "gold": f"Agent should use '{title} {last}' as the sender name.",
             "violation_patterns": [],
             "anti_patterns": [f"{title.lower().replace('.', '')}|{last.lower()}"],
-            "precision_targets": [f"{title} {last}"],
+            # Accept both "Dr. Volkov" and bare "Volkov" — the surname alone
+            # in the signature still demonstrates the constraint is being
+            # applied; the title prefix is conventional but not load-bearing.
+            "precision_targets": [f"{title} {last}", last],
         },
     }
 
@@ -533,7 +576,11 @@ def _generate_profile(self) -> dict:
         }
 
     def _generate_sessions(self, n: int, constraints: list) -> list[dict]:
-        categories = list(set(c["category"] for c in constraints))
+        # sorted(), not list(set()): set iteration order over strings is
+        # PYTHONHASHSEED-dependent, which made the same seed produce different
+        # scenarios across processes (non-reproducible). Sorting fixes the order
+        # so a given seed is deterministic.
+        categories = sorted(set(c["category"] for c in constraints))
         sessions = []
         for t in range(n):
             tasks = []

prototype/agingbench/generators/s6_generator.py

@@ -105,26 +105,28 @@ def generate(self, n_sessions: int = 15) -> dict[str, Any]:
                         f"{p['text_a']}\n{p['text_b']}" for p in pairs
                     )
                     session["environment_data"] = session.get("environment_data", "") + "\n\n" + interf_text
-                    # Add a forced binding probe per pair to recall_probes,
-                    # carrying gold + distractor so the runner/scorer can tell
-                    # confusion (partner value) from omission. S6 re-asks all
-                    # prior probes every session, so these get a lag/density
-                    # sweep for free.
-                    probes = session.setdefault("recall_probes", [])
-                    for j, p in enumerate(pairs):
-                        q = p.get("probe_question") or (
-                            f"What is the exact {p['fact_a']['domain']} "
-                            f"{p['shared_term']}? Reply with the exact value only."
-                        )
-                        probes.append({
-                            "probe_id": f"s{i}_interf_{j}",
-                            "question": q,
-                            "keywords": [str(p["fact_a"]["value"])],
-                            "canonical_answer": str(p["fact_a"]["value"]),
-                            "gold_value": p["fact_a"]["value"],
-                            "distractor_value": p["fact_b"]["value"],
-                            "probe_type": "interference_binding",
-                        })
+                    # Forced binding probes (gold+distractor) are emitted ONLY
+                    # for the explicit binding-test modes (similar-name /
+                    # high-similarity), so default value-confusable runs are
+                    # unchanged — no binding probes leak into recall. S6 re-asks
+                    # all prior probes every session → free lag/density sweep.
+                    if (getattr(self.pressure, "confusable_similar_names", False)
+                            or getattr(self.pressure, "confusable_high_similarity", False)):
+                        probes = session.setdefault("recall_probes", [])
+                        for j, p in enumerate(pairs):
+                            q = p.get("probe_question") or (
+                                f"What is the exact {p['fact_a']['domain']} "
+                                f"{p['shared_term']}? Reply with the exact value only."
+                            )
+                            probes.append({
+                                "probe_id": f"s{i}_interf_{j}",
+                                "question": q,
+                                "keywords": [str(p["fact_a"]["value"])],
+                                "canonical_answer": str(p["fact_a"]["value"]),
+                                "gold_value": p["fact_a"]["value"],
+                                "distractor_value": p["fact_b"]["value"],
+                                "probe_type": "interference_binding",
+                            })
 
             sessions.append(session)
 
@@ -512,6 +514,20 @@ def _remap(seq: list[str]) -> list[str]:
                 pkws = probe.get("keywords") or []
                 if pkws and set(pkws) <= old_set:
                     probe["keywords"] = _remap(pkws)
+                    # Binding probes carry parallel gold fields for fact_a;
+                    # keep them in sync (substring remap) so the binding scorer
+                    # and the P3 oracle don't use stale gold and penalize the
+                    # agent for citing the correct revised value. (distractor_value
+                    # belongs to fact_b and is synced by fact_b's own update.)
+                    for _field in ("canonical_answer", "gold_value"):
+                        _val = probe.get(_field)
+                        if _val is None:
+                            continue
+                        _sval = str(_val)
+                        for _o, _n in zip(old_kws, new_kws):
+                            if _o != _n and _o in _sval:
+                                _sval = _sval.replace(_o, _n)
+                        probe[_field] = _sval
             for fact in all_facts:
                 if fact.get("session_id") != origin:
                     continue