From 2de482f74e6ec468db11051cfb09220cf06b510c Mon Sep 17 00:00:00 2001 From: Ignazio De Santis Date: Sat, 25 Jul 2026 11:14:31 +0800 Subject: [PATCH] feat(llm_judge): multi-sample verdicts with median aggregation Add `samples` (1-9, default 1 = unchanged single-call behavior) to llm_judge. With N > 1 the judge grades the same rendered prompt N times; the criterion scores the median verdict, the sample closest to the median (earliest on ties) supplies rationale/findings/transcript, per-sample scores render in details, and token usage sums across samples. A sample that produces no verdict degrades to the median of the valid samples with a degraded note; when every sample fails, single-sample failure semantics apply unchanged (JudgeInfrastructureError still escalates, parse failures still score 0.0). agent_judge stays single-sample. --- docs/TASK_DEFINITION_GUIDE.md | 3 + src/coder_eval/criteria/llm_judge.py | 161 +++++++++++++++-- src/coder_eval/models/criteria.py | 21 +++ tests/test_llm_judge_criterion.py | 257 +++++++++++++++++++++++++++ 4 files changed, 428 insertions(+), 14 deletions(-) diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index c2de203a..60b7100c 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -834,8 +834,11 @@ Have an LLM grade the task against a rubric written in the task YAML. **Continuo | `model` | `anthropic.claude-sonnet-4-6` | Judge model id (vendor-prefixed; auto-translated per backend) | | `temperature` | `0.0` | Sampling temperature (0.0 = deterministic) | | `max_tokens` | `2000` | Maximum tokens in the judge's response | +| `samples` | `1` | Number of independent judge invocations (1–9). `1` = single call, unchanged behavior. With N > 1 the criterion scores the **median** of N sampled verdicts; rationale/findings come from the sample closest to the median (earliest on ties); per-sample scores land in `details`; token usage sums across samples. | | `max_file_chars` | `20000` | Per-file (and agent_output) truncation applied before building the prompt | +**Sampling the judge.** Even at `temperature: 0.0`, judge verdicts are not perfectly repeatable — the same artifacts can draw a strict or a lenient reading of the rubric on different calls. Set `samples: 3` (odd values recommended) to invoke the judge three times over the same rendered prompt and score the median verdict, damping a single outlier reading. Because every sample grades identical artifacts, the spread across samples is pure judge variance — unlike experiment-level `replicates`, which re-run the whole agent and mix agent variance into the same number. A sample that returns no verdict degrades to the median of the remaining valid samples (with a note in `details`) rather than scoring the criterion 0.0; only when every sample fails does the single-sample failure behavior apply. + **Transport selection.** The judge call is routed by the active `API_BACKEND`: | `API_BACKEND` | Credentials | Judge transport | diff --git a/src/coder_eval/criteria/llm_judge.py b/src/coder_eval/criteria/llm_judge.py index 73db9f4a..1270b375 100644 --- a/src/coder_eval/criteria/llm_judge.py +++ b/src/coder_eval/criteria/llm_judge.py @@ -1,9 +1,12 @@ """LLM-as-a-judge success criterion checker.""" import logging -from typing import TYPE_CHECKING +from collections.abc import Iterable +from statistics import median +from typing import TYPE_CHECKING, NamedTuple from coder_eval.criteria.base import BaseCriterion, CheckContext, register_criterion +from coder_eval.errors import JudgeInfrastructureError from coder_eval.evaluation.judge_anthropic import invoke_anthropic_judge from coder_eval.evaluation.judge_bedrock import invoke_bedrock_judge from coder_eval.evaluation.judge_context import ( @@ -113,6 +116,18 @@ def _check_impl( scrub_key = reference_code if criterion.include_reference else None + # Multi-sample grading (samples > 1) aggregates independent verdicts over + # the same rendered prompt; the default (1) stays on the single-call path + # below, unchanged. + if criterion.samples > 1: + return _grade_with_sampling( + criterion=criterion, + route=route, + user_msg=user_msg, + judge_ctx=judge_ctx, + scrub_key=scrub_key, + ) + # Attribute the judge's API call to ``JudgeCriterionResult.token_usage`` # from the usage the backend reported in its response. verdict, parse_error, raw_verdict_text, response_usage = _invoke_tool_channel( @@ -127,17 +142,6 @@ def _check_impl( # model could echo the reference back in an unparseable response, so we scrub it. scrubbed = scrub_reference(raw_verdict_text, scrub_key) - def _maybe_transcript() -> JudgeTranscript | None: - if not criterion.capture_transcript: - return None - return build_judge_transcript( - raw_verdict=raw_verdict_text, - max_chars=criterion.max_transcript_chars, - judge_system_prompt=_SYSTEM_PROMPT, - judge_prompt=user_msg, - scrub_key=scrub_key, - ) - if parse_error is not None: return JudgeCriterionResult( criterion_type=criterion.type, @@ -145,7 +149,7 @@ def _maybe_transcript() -> JudgeTranscript | None: score=0.0, details=scrubbed[:500], error=scrub_reference(parse_error, scrub_key), - transcript=_maybe_transcript(), + transcript=_build_transcript(criterion, raw_verdict_text, user_msg, scrub_key), token_usage=judge_usage, ) assert verdict is not None # parser contract: verdict is set iff parse_error is None @@ -157,7 +161,7 @@ def _maybe_transcript() -> JudgeTranscript | None: score=verdict.score, details=scrub_reference(details, scrub_key), findings=[scrub_reference(f, scrub_key) for f in verdict.findings], - transcript=_maybe_transcript(), + transcript=_build_transcript(criterion, raw_verdict_text, user_msg, scrub_key), token_usage=judge_usage, ) @@ -213,6 +217,135 @@ def _invoke_tool_channel( return None, err, f"(no verdict — {err})", response_usage +class _SampleOutcome(NamedTuple): + """One judge invocation's outcome, in ``_invoke_tool_channel`` return order.""" + + verdict: JudgeVerdict | None + parse_error: str | None + raw_verdict_text: str + response_usage: TokenUsage | None + + +def _grade_with_sampling( + *, + criterion: LLMJudgeCriterion, + route: "ApiRoute", + user_msg: str, + judge_ctx: JudgeContext, + scrub_key: str | None, +) -> JudgeCriterionResult: + """Invoke the judge ``criterion.samples`` times and score the median verdict. + + Every sample grades the SAME rendered prompt, so score spread across samples + is judge variance by construction — the median damps a single strict-or-lenient + outlier reading of the rubric. The representative sample (score closest to the + median, earliest on ties) supplies rationale/findings/transcript so the + persisted audit trail is a real verdict, never a synthetic blend. + + A sample that produces no verdict (a transport failure, or a response with no + usable ``submit_verdict`` call) degrades to the median of the remaining valid + samples with a note in ``details``. When NO sample produces a verdict, the + single-sample failure semantics apply: an infrastructure failure escalates + (``JudgeInfrastructureError`` propagates to ``FinalStatus.ERROR`` — judge infra + failure is not an agent failure), any other exception reaches + ``@handle_criterion_errors``, and all-parse-failures score 0.0 with the first + sample's diagnostic. + """ + outcomes: list[_SampleOutcome] = [] + infra_errors: list[JudgeInfrastructureError] = [] + unexpected_errors: list[Exception] = [] + for _ in range(criterion.samples): + try: + outcomes.append( + _SampleOutcome( + *_invoke_tool_channel( + criterion=criterion, + route=route, + system_msg=_SYSTEM_PROMPT, + user_msg=user_msg, + ) + ) + ) + except JudgeInfrastructureError as exc: + infra_errors.append(exc) + except Exception as exc: + unexpected_errors.append(exc) + + # Cost is real for every sample that returned a response, verdict or not. + token_usage = _sum_usage(o.response_usage for o in outcomes) + valid: list[tuple[JudgeVerdict, str]] = [(o.verdict, o.raw_verdict_text) for o in outcomes if o.verdict is not None] + + if not valid: + if infra_errors: + raise infra_errors[0] + if unexpected_errors: + raise unexpected_errors[0] + first = outcomes[0] + assert first.parse_error is not None # parser contract: verdict is set iff parse_error is None + scrubbed = scrub_reference(first.raw_verdict_text, scrub_key) + return JudgeCriterionResult( + criterion_type=criterion.type, + description=criterion.description, + score=0.0, + details=scrubbed[:500], + error=scrub_reference(first.parse_error, scrub_key), + transcript=_build_transcript(criterion, first.raw_verdict_text, user_msg, scrub_key), + token_usage=token_usage, + ) + + scores = [verdict.score for verdict, _ in valid] + median_score = float(median(scores)) + rep_verdict, rep_raw = min(valid, key=lambda pair: abs(pair[0].score - median_score)) + + degraded_notes = list(judge_ctx.degraded_notes) + failed_samples = criterion.samples - len(valid) + if failed_samples: + summary = f"median over {len(valid)} valid samples" + degraded_notes.append(f"{failed_samples}/{criterion.samples} judge samples produced no verdict; {summary}") + + details = format_details(median_score, rep_verdict.rationale, judge_ctx.missing_files, degraded_notes) + rendered_scores = ", ".join(f"{s:.3f}" for s in scores) + details = f"{details}\nsample_scores: [{rendered_scores}]" + return JudgeCriterionResult( + criterion_type=criterion.type, + description=criterion.description, + score=median_score, + details=scrub_reference(details, scrub_key), + findings=[scrub_reference(f, scrub_key) for f in rep_verdict.findings], + transcript=_build_transcript(criterion, rep_raw, user_msg, scrub_key), + token_usage=token_usage, + ) + + +def _sum_usage(usages: Iterable[TokenUsage | None]) -> TokenUsage | None: + """Field-wise sum of the reported usages; ``None`` when no sample reported any.""" + present = [u for u in usages if u is not None] + if not present: + return None + total = present[0] + for u in present[1:]: + total = total + u + return total + + +def _build_transcript( + criterion: LLMJudgeCriterion, + raw_verdict_text: str, + user_msg: str, + scrub_key: str | None, +) -> JudgeTranscript | None: + """Build the persisted judge transcript when ``capture_transcript`` is on.""" + if not criterion.capture_transcript: + return None + return build_judge_transcript( + raw_verdict=raw_verdict_text, + max_chars=criterion.max_transcript_chars, + judge_system_prompt=_SYSTEM_PROMPT, + judge_prompt=user_msg, + scrub_key=scrub_key, + ) + + def _render_user_message(prompt: str, context: JudgeContext) -> str: """Render the user-facing prompt envelope for the LLM judge.""" reference_block = "" diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 2605e1cf..108fe46f 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -657,6 +657,10 @@ class LLMJudgeCriterion(BaseSuccessCriterion): Continuous scoring. LLM error or a "did not call submit_verdict" diagnostic -> 0.0 with error. Score is clamped to [0.0, 1.0] by the ``JudgeVerdict`` validator. Non-numeric / non-finite score -> 0.0 with error. + + Set ``samples`` > 1 to invoke the judge several times over the same rendered + prompt and score the median verdict (see the field description for the + aggregation semantics). """ # Strict YAML-key validation: catch typos at load time rather than silently @@ -760,6 +764,23 @@ class LLMJudgeCriterion(BaseSuccessCriterion): "(score + rationale + a handful of findings) without runaway." ), ) + samples: int = Field( + default=1, + ge=1, + le=9, + description=( + "Number of independent judge invocations. The default (1) keeps the single-call " + "behavior. With N > 1, the judge grades the same rendered prompt N times and the " + "criterion scores the MEDIAN of the sampled scores; rationale/findings/transcript " + "come from the sample whose score is closest to the median (earliest on ties), so " + "the audit trail is a real verdict rather than a synthetic blend. Per-sample scores " + "are rendered in ``details`` and token usage is summed across samples. With N >= 2, " + "a sample that produces no verdict degrades to the median of the valid samples " + "(with a note in ``details``); when every sample fails, single-sample failure " + "semantics apply unchanged. Odd N recommended — an even-N median averages the two " + "middle scores." + ), + ) max_file_chars: int = Field( default=20_000, gt=0, diff --git a/tests/test_llm_judge_criterion.py b/tests/test_llm_judge_criterion.py index a18b9318..f5db5189 100644 --- a/tests/test_llm_judge_criterion.py +++ b/tests/test_llm_judge_criterion.py @@ -1080,3 +1080,260 @@ def test_usage_extractors_degrade_non_numeric_values_without_raising() -> None: # A nested object is also non-coercible → degrades to 0 (here all-zero → None). assert token_usage_from_anthropic_dict({"usage": {"input_tokens": {}, "output_tokens": "x"}}) is None + + +# --- multi-sample judging (samples > 1) --- + + +def _sample_resp( + score: float, rationale: str = "ok", findings: list[str] | None = None, usage: dict | None = None +) -> dict: + """An Anthropic-shaped submit_verdict response for one judge sample.""" + verdict_input: dict[str, Any] = {"score": score, "rationale": rationale} + if findings is not None: + verdict_input["findings"] = findings + resp: dict[str, Any] = {"content": [{"type": "tool_use", "name": "submit_verdict", "input": verdict_input}]} + if usage is not None: + resp["usage"] = usage + return resp + + +def _no_verdict_resp(usage: dict | None = None) -> dict: + """A text-only response — the model returned prose instead of the tool call.""" + resp: dict[str, Any] = {"content": [{"type": "text", "text": "no verdict here"}]} + if usage is not None: + resp["usage"] = usage + return resp + + +def test_judge_samples_defaults_to_single_invocation(sandbox: Sandbox) -> None: + """Default samples=1: exactly one judge call, no aggregation artifacts in details.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade") + assert criterion.samples == 1 + resp = _sample_resp(0.85, rationale="mostly correct") + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", return_value=resp) as m_anthropic: + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert m_anthropic.call_count == 1 + assert result.score == 0.85 + assert result.error is None + assert "sample_scores" not in (result.details or "") + + +def test_judge_samples_bounds_validation() -> None: + from pydantic import ValidationError + + with pytest.raises(ValidationError): + LLMJudgeCriterion(description="x", prompt="grade", samples=0) + with pytest.raises(ValidationError): + LLMJudgeCriterion(description="x", prompt="grade", samples=10) + + +def test_judge_multi_sample_scores_median_odd(sandbox: Sandbox) -> None: + """samples=3: three invocations, median score wins, per-sample scores in details.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_sample_resp(0.2, "low"), _sample_resp(0.9, "high"), _sample_resp(0.6, "mid")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses) as m_anthropic: + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert m_anthropic.call_count == 3 + assert result.score == 0.6 + assert result.error is None + details = result.details or "" + assert "rationale: mid" in details # representative = the median sample + assert "sample_scores: [0.200, 0.900, 0.600]" in details + # All samples succeeded — the degraded note must not appear. + assert "judge samples produced no verdict" not in details + + +def test_judge_multi_sample_reuses_identical_prompt(sandbox: Sandbox) -> None: + """Every sample grades the same rendered prompt with the same invocation params.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_sample_resp(0.5), _sample_resp(0.5), _sample_resp(0.5)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses) as m_anthropic: + SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + first_kwargs = m_anthropic.call_args_list[0].kwargs + for call in m_anthropic.call_args_list[1:]: + assert call.kwargs == first_kwargs + + +def test_judge_multi_sample_even_count_averages_middle(sandbox: Sandbox) -> None: + """Even N: median averages the two middle scores; representative is the closest sample.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=4) + responses = [_sample_resp(0.2, "r1"), _sample_resp(0.4, "r2"), _sample_resp(0.6, "r3"), _sample_resp(0.8, "r4")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == pytest.approx(0.5) + # 0.4 and 0.6 are equidistant from the 0.5 median — the earlier sample wins. + assert "rationale: r2" in (result.details or "") + + +def test_judge_multi_sample_tie_break_is_first_sample(sandbox: Sandbox) -> None: + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [_sample_resp(0.4, "first"), _sample_resp(0.6, "second")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == pytest.approx(0.5) + assert "rationale: first" in (result.details or "") + + +def test_judge_multi_sample_partial_no_verdict_degrades(sandbox: Sandbox) -> None: + """One sample returning no verdict must not zero the criterion — median of the valid ones.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_sample_resp(0.8, "a"), _no_verdict_resp(), _sample_resp(0.6, "b")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == pytest.approx(0.7) + assert result.error is None + details = result.details or "" + assert "1/3 judge samples produced no verdict" in details + assert "median over 2 valid samples" in details + assert "sample_scores: [0.800, 0.600]" in details + + +def test_judge_multi_sample_two_samples_one_failure_degrades(sandbox: Sandbox) -> None: + """N=2 (the minimal multi-sample count) with one failure: the single valid score stands.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [_no_verdict_resp(), _sample_resp(0.8, "only valid")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == 0.8 + assert result.error is None + details = result.details or "" + assert "rationale: only valid" in details + assert "1/2 judge samples produced no verdict" in details + assert "median over 1 valid samples" in details + + +def test_judge_multi_sample_transport_exception_degrades(sandbox: Sandbox) -> None: + """One sample's transport failure must not void the others.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_sample_resp(0.8), RuntimeError("gateway down"), _sample_resp(0.6)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == pytest.approx(0.7) + assert result.error is None + assert "1/3 judge samples produced no verdict" in (result.details or "") + + +def test_judge_multi_sample_infra_error_degrades_when_verdicts_exist(sandbox: Sandbox) -> None: + """An infra failure on one sample degrades (never scores 0.0) when other samples produced verdicts.""" + from coder_eval.errors import JudgeInfrastructureError + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [JudgeInfrastructureError("api error"), _sample_resp(0.5), _sample_resp(0.9)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == pytest.approx(0.7) + assert result.error is None + + +def test_judge_multi_sample_all_no_verdict_keeps_failure_path(sandbox: Sandbox) -> None: + """Every sample failing to call submit_verdict keeps the single-sample failure semantics.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [_no_verdict_resp(), _no_verdict_resp(), _no_verdict_resp()] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == 0.0 + assert result.error == "Judge did not call submit_verdict" + + +def test_judge_multi_sample_all_fail_reports_first_sample_diagnostic(sandbox: Sandbox) -> None: + """When every sample fails to produce a verdict, the FIRST sample's diagnostic is reported.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + bad_args = {"content": [{"type": "tool_use", "name": "submit_verdict", "input": {"score": "not numeric"}}]} + responses = [bad_args, _no_verdict_resp()] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == 0.0 + assert result.error is not None + assert "score field is not a number" in result.error # sample #1's diagnostic, not sample #2's + + +def test_judge_multi_sample_all_infra_errors_escalate(sandbox: Sandbox) -> None: + """All samples failing on infrastructure escalates — judge infra failure is not an agent failure.""" + from coder_eval.errors import JudgeInfrastructureError + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [JudgeInfrastructureError("api error"), JudgeInfrastructureError("api error")] + with ( + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses), + pytest.raises(JudgeInfrastructureError), + ): + SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + + +def test_judge_multi_sample_mixed_all_fail_prefers_infra_escalation(sandbox: Sandbox) -> None: + """No verdict anywhere + an infra failure in the mix: escalate rather than score 0.0.""" + from coder_eval.errors import JudgeInfrastructureError + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [_no_verdict_resp(), JudgeInfrastructureError("api error")] + with ( + patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses), + pytest.raises(JudgeInfrastructureError), + ): + SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + + +def test_judge_multi_sample_all_unexpected_errors_map_to_score_zero(sandbox: Sandbox) -> None: + """Non-infra exceptions on every sample reach @handle_criterion_errors, as with samples=1.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=2) + responses = [RuntimeError("gateway down"), RuntimeError("gateway down")] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == 0.0 + assert result.error is not None + assert "gateway down" in result.error + + +def test_judge_multi_sample_token_usage_summed(sandbox: Sandbox) -> None: + """Usage sums across every sample that returned a response — including no-verdict samples.""" + from coder_eval.models import JudgeCriterionResult + + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + usage = {"input_tokens": 100, "output_tokens": 10} + responses = [_sample_resp(0.8, usage=usage), _no_verdict_resp(usage=usage), _sample_resp(0.6, usage=usage)] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert isinstance(result, JudgeCriterionResult) + assert result.token_usage is not None + assert result.token_usage.uncached_input_tokens == 300 + assert result.token_usage.output_tokens == 30 + + +def test_judge_multi_sample_representative_supplies_findings_and_transcript(sandbox: Sandbox) -> None: + """Findings and the persisted transcript come from the representative sample, not a blend.""" + criterion = LLMJudgeCriterion(description="x", prompt="grade", samples=3) + responses = [ + _sample_resp(0.2, "low", findings=["low finding"]), + _sample_resp(0.6, "mid", findings=["mid finding"]), + _sample_resp(0.9, "high", findings=["high finding"]), + ] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check(criterion) + assert result.score == 0.6 + assert getattr(result, "findings", []) == ["mid finding"] + transcript = getattr(result, "transcript", None) + assert transcript is not None + assert '"score":0.6' in transcript.raw_verdict + + +def test_judge_multi_sample_scrubs_reference_everywhere(sandbox: Sandbox) -> None: + """Reference scrubbing covers details, findings, and the transcript with samples > 1.""" + sentinel = "REF_LEAK_MULTI_SAMPLE_321" + criterion = LLMJudgeCriterion(description="x", prompt="grade", include_reference=True, samples=3) + responses = [ + _sample_resp(0.4, f"echoed {sentinel}"), + _sample_resp(0.5, f"echoed {sentinel}", findings=[f"finding with {sentinel}"]), + _sample_resp(0.6, f"echoed {sentinel}"), + ] + with patch("coder_eval.criteria.llm_judge.invoke_anthropic_judge", side_effect=responses): + result = SuccessChecker(sandbox, init_registry=False, route=DirectRoute()).check( + criterion, reference_code=sentinel + ) + assert result.score == 0.5 + assert sentinel not in (result.details or "") + for finding in getattr(result, "findings", []) or []: + assert sentinel not in finding + transcript = getattr(result, "transcript", None) + assert transcript is not None + assert sentinel not in transcript.raw_verdict