test(early-stop): cover second-review items (two-AgentStart, golden corpus, parity)

Add the still-applicable coverage from the second review; no scoring-logic
change (any-engagement was already restored by the prior commit):

- Golden-corpus regression test pinning observed_label AND score for the
  canonical skill_triggered trajectories (single-target, target-then-competitor
  recall/precision, wrong-then-target, negative false-positive, true-negative,
  file-read), so a future engagement-policy change fails loudly instead of
  silently shifting the activation suite's P/R/F1.
- Multi-turn + file-read parity tests: any-engagement holds across TurnRecords
  and for the file-read signal (Read of skills/<name>/...), not only the Claude
  Skill call, so the agent-agnostic parity is asserted for the new branch.
- Two-AgentStart watcher test: a retry's second AgentStart does not reset the
  wall-clock origin (the documented no-op branch in on_event).

Also clarifies the EarlyStopInfo.tool_call_index field doc: on a call-latched
stop it is the deciding in-flight call index (completed_tool_ends + 1), read as
"which call decided", not a count of fully-completed tool calls.

Author: mohsen-uipath

src/coder_eval/models/results.py

@@ -420,7 +420,12 @@ class EarlyStopInfo(BaseModel):
         description="SDK inner-turn count at the stop (watcher counts TurnStartEvents). NOT the "
         + "orchestrator iteration, which is always 1 in single-shot."
     )
-    tool_call_index: int = Field(description="1-based index of the tool call that decided the stop.")
+    tool_call_index: int = Field(
+        description="1-based index of the tool call that decided the stop. NOTE: because the "
+        + "stop latches on the tool CALL (ToolStartEvent), this is the index of the deciding "
+        + "call INCLUDING that in-flight call — i.e. completed_tool_ends + 1 for a call-latched "
+        + "stop. Read it as 'which call decided', not as a count of fully-completed tool calls."
+    )
     elapsed_seconds: float = Field(description="Wall-clock seconds from the first agent-start event to the stop.")
     turns_remaining_at_stop: int | None = Field(
         default=None,

tests/test_early_stop.py

@@ -1131,6 +1131,23 @@ def test_tool_call_index_counts_prior_resolved_calls(self) -> None:
         assert watcher.info is not None
         assert watcher.info.tool_call_index == 2
 
+    def test_second_agent_start_does_not_reset_origin(self) -> None:
+        # The wall-clock origin is stamped at the FIRST AgentStartEvent only; a
+        # retry's second AgentStart must NOT reset it (the documented no-op branch
+        # in on_event). Exercised deterministically via _started_monotonic rather
+        # than the time-based elapsed_seconds field.
+        watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_when="pass")])
+        _feed(watcher, [_agent_start()])
+        origin = watcher._started_monotonic
+        assert origin is not None
+        # A second AgentStart (as on a retry) must leave the origin untouched.
+        _feed(watcher, [_agent_start(), _turn_start()])
+        assert watcher._started_monotonic == origin
+        # The stop that follows anchors elapsed_seconds to that first origin.
+        _feed(watcher, [_skill_start("date-teller")])
+        assert watcher.info is not None
+        assert watcher.info.elapsed_seconds >= 0.0
+
 
 # --------------------------------------------------------------------------- #
 # Phase 3: Orchestrator wiring

tests/test_skill_triggered.py

@@ -158,6 +158,155 @@ def test_stacked_recall_and_precision_on_one_trajectory(self) -> None:
         assert precision.observed_label == "yes" and precision.score == 0.0  # precision: off-target engaged
 
 
+def _check_multi(
+    *, expected_skill: str, skill_name: str, turns: list[list[CommandTelemetry]]
+) -> ClassificationCriterionResult:
+    criterion = SkillTriggeredCriterion(
+        description="did agent invoke a skill?",
+        expected_skill=expected_skill,
+        skill_name=skill_name,
+    )
+    checker = SkillTriggeredChecker()
+    turn_records = [_turn(cmds) for cmds in turns]
+    result = checker.check(criterion, sandbox=None, turn_records=turn_records)  # type: ignore[arg-type]
+    assert isinstance(result, ClassificationCriterionResult)
+    return result
+
+
+class TestSkillTriggeredGoldenCorpus:
+    """Regression lock: pins observed_label AND score for the canonical
+    multi-skill trajectories through ``_check_impl``. Any future change to the
+    engagement policy (any- vs first-engagement) breaks these, forcing an
+    explicit acknowledgement of the methodology break — and a re-score/backfill
+    of historical activation P/R/F1 — before merge.
+    """
+
+    @pytest.mark.parametrize(
+        ("case", "expected_skill", "skill_name", "commands", "exp_observed", "exp_score"),
+        [
+            (
+                "single-target-recall",
+                "uipath-admin",
+                "uipath-admin",
+                [_cmd("Skill", {"skill": "uipath-admin"}, tool_id="s1")],
+                "yes",
+                1.0,
+            ),
+            (
+                "target-first-then-competitor-recall",
+                "uipath-admin",
+                "uipath-admin",
+                [
+                    _cmd("Skill", {"skill": "uipath-admin"}, tool_id="s1"),
+                    _cmd("Skill", {"skill": "uipath-platform"}, tool_id="s2"),
+                ],
+                "yes",
+                1.0,
+            ),
+            (
+                "target-first-then-competitor-precision",
+                "uipath-admin",
+                "uipath-platform",
+                [
+                    _cmd("Skill", {"skill": "uipath-admin"}, tool_id="s1"),
+                    _cmd("Skill", {"skill": "uipath-platform"}, tool_id="s2"),
+                ],
+                "yes",
+                0.0,
+            ),
+            (
+                "wrong-first-then-target-recall",
+                "uipath-admin",
+                "uipath-admin",
+                [
+                    _cmd("Skill", {"skill": "uipath-platform"}, tool_id="s1"),
+                    _cmd("Skill", {"skill": "uipath-admin"}, tool_id="s2"),
+                ],
+                "yes",
+                1.0,
+            ),
+            (
+                "negative-target-engaged-false-positive",
+                "",
+                "uipath-admin",
+                [_cmd("Skill", {"skill": "uipath-admin"}, tool_id="s1")],
+                "yes",
+                0.0,
+            ),
+            (
+                "negative-no-engagement-true-negative",
+                "",
+                "uipath-admin",
+                [_cmd("Read", {"file_path": "notes.txt"})],
+                "no",
+                1.0,
+            ),
+            (
+                "file-read-target-recall",
+                "uipath-admin",
+                "uipath-admin",
+                [_cmd("Read", {"file_path": "skills/uipath-admin/SKILL.md"})],
+                "yes",
+                1.0,
+            ),
+        ],
+    )
+    def test_golden_corpus_scores_are_pinned(
+        self,
+        case: str,
+        expected_skill: str,
+        skill_name: str,
+        commands: list[CommandTelemetry],
+        exp_observed: str,
+        exp_score: float,
+    ) -> None:
+        result = _check(expected_skill=expected_skill, skill_name=skill_name, commands=commands)
+        assert result.observed_label == exp_observed, case
+        assert result.score == exp_score, case
+
+
+class TestSkillTriggeredMultiTurnParity:
+    """Any-engagement holds across TurnRecords and for the file-read signal, so
+    the agent-agnostic parity CLAUDE.md stresses is asserted for the new branch.
+    """
+
+    def test_expected_skill_in_later_turn_still_recalls(self) -> None:
+        # Distractor engaged in turn 1, expected skill in turn 2. Recall must
+        # credit the GT criterion regardless of which turn the engagement lives in.
+        result = _check_multi(
+            expected_skill="uipath-admin",
+            skill_name="uipath-admin",
+            turns=[
+                [_cmd("Skill", {"skill": "uipath-platform"}, tool_id="s1")],
+                [_cmd("Skill", {"skill": "uipath-admin"}, tool_id="s2")],
+            ],
+        )
+        assert result.observed_label == "yes" and result.score == 1.0
+
+    def test_off_target_in_earlier_turn_is_precision_miss(self) -> None:
+        # The competitor engaged in an EARLIER turn than the GT skill still lands
+        # as a precision miss on its own criterion (no turn-order dependence).
+        turns = [
+            [_cmd("Skill", {"skill": "uipath-platform"}, tool_id="s1")],
+            [_cmd("Skill", {"skill": "uipath-admin"}, tool_id="s2")],
+        ]
+        precision = _check_multi(expected_skill="uipath-admin", skill_name="uipath-platform", turns=turns)
+        assert precision.observed_label == "yes" and precision.score == 0.0
+
+    def test_file_read_parity_across_turns(self) -> None:
+        # Off-Claude parity: the agent READS skill files across turns (platform's
+        # in turn 1, admin's in turn 2). Recall/precision score identically to the
+        # Skill-tool path, order- and turn-independent.
+        turns = [
+            [_cmd("Read", {"file_path": "skills/uipath-platform/SKILL.md"})],
+            [_cmd("Read", {"file_path": "skills/uipath-admin/reference.md"})],
+        ]
+        recall = _check_multi(expected_skill="uipath-admin", skill_name="uipath-admin", turns=turns)
+        precision = _check_multi(expected_skill="uipath-admin", skill_name="uipath-platform", turns=turns)
+        assert recall.observed_label == "yes" and recall.score == 1.0
+        assert precision.observed_label == "yes" and precision.score == 0.0
+
+
 class TestSkillTriggeredCodex:
     """Codex has no ``Skill`` tool — it engages a skill by reading its files via shell.