Skip to content

Commit 30044ab

Browse files
Extend conformance validation and robot event contract for speculation events
Conformance and robot module changes to support speculative pipeline event contracts and strengthen segment validation: conformance.rs: - Add NaN/Infinity rejection for start_sec and end_sec timestamps using is_finite() checks before existing start > end validation, preventing nonsensical segments from passing conformance gates - Add negative timestamp rejection for start_sec and end_sec, catching segments with physically impossible negative time values - Simplify compare_shadow_run passes_gate logic: gate now depends solely on segment_comparison.within_tolerance() since text and timestamp violations are already captured by that method. Previous redundant checks for text_mismatches and timestamp_violations were removed. - Update compare_shadow_run docstring to clarify replay envelope comparison is informational only (different engines naturally produce different output hashes) - Add tests: NaN start/end rejection, Infinity start/end rejection, negative start/end rejection - Add IEEE 754 negative zero (-0.0) acceptance test documenting that -0.0 == 0.0 passes the negativity check - Add NativeEngineRolloutStage parse whitespace/mixed-case test - Add compare_shadow_run replay-pass-but-segment-fail gate test - Add None end_sec overlap-check non-regression test (seg with end_sec=None should not reset previous_end tracker) - Add compare_segments speaker mismatch ignored when not required test robot.rs: - Add TRANSCRIPT_CONFIRM_REQUIRED_FIELDS constant defining the required fields for transcript.confirm events (event, schema_version, run_id, seq, window_id, quality_model_id, drift, latency_ms, ts) - Add confirmations_emitted to SPECULATION_STATS_REQUIRED_FIELDS - Add transcript_confirm_value() and emit_transcript_confirm() functions for constructing and emitting confirm event payloads - Add confirmations_emitted field to speculation_stats_value() payload - Extend robot_schema_value() with full schema documentation for transcript.confirm, transcript.retract, transcript.correct, and transcript.speculation_stats event types including required fields and example payloads - Update schema event count from 7 to 11 in schema_has_expected_event_types test - Update test imports for new required-field constants - Add transcript_confirm_value required fields and drift test - Add transcript_retract_value required fields and reason test - Add transcript_correct_value required fields and segments test - Add speculation_stats_value required fields test - Add snapshot_resources memory field non-swap assertion test Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 847658d commit 30044ab

2 files changed

Lines changed: 502 additions & 12 deletions

File tree

src/conformance.rs

Lines changed: 216 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,38 @@ pub fn validate_segment_invariants_with_policy(
9595
let mut previous_end: Option<f64> = None;
9696

9797
for (index, segment) in segments.iter().enumerate() {
98+
if let Some(start) = segment.start_sec
99+
&& !start.is_finite()
100+
{
101+
return Err(FwError::Unsupported(format!(
102+
"conformance violation: segment {index} start_sec ({start}) is not finite"
103+
)));
104+
}
105+
106+
if let Some(end) = segment.end_sec
107+
&& !end.is_finite()
108+
{
109+
return Err(FwError::Unsupported(format!(
110+
"conformance violation: segment {index} end_sec ({end}) is not finite"
111+
)));
112+
}
113+
114+
if let Some(start) = segment.start_sec
115+
&& start < 0.0
116+
{
117+
return Err(FwError::Unsupported(format!(
118+
"conformance violation: segment {index} start_sec ({start}) is negative"
119+
)));
120+
}
121+
122+
if let Some(end) = segment.end_sec
123+
&& end < 0.0
124+
{
125+
return Err(FwError::Unsupported(format!(
126+
"conformance violation: segment {index} end_sec ({end}) is negative"
127+
)));
128+
}
129+
98130
if let (Some(start), Some(end)) = (segment.start_sec, segment.end_sec)
99131
&& end + policy.timestamp_epsilon_sec < start
100132
{
@@ -332,10 +364,11 @@ pub struct ShadowRunReport {
332364
/// Compare shadow-run results from a bridge adapter and native engine on the
333365
/// same input. Returns a `ShadowRunReport` summarizing parity.
334366
///
335-
/// `passes_gate` is true when:
336-
/// - segment comparison is `within_tolerance()`
337-
/// - no text mismatches
338-
/// - no timestamp violations
367+
/// `passes_gate` is true when segment comparison is `within_tolerance()`
368+
/// (no length mismatch, no text/speaker/timestamp violations).
369+
///
370+
/// Replay envelope comparison is informational only — different engines
371+
/// naturally produce different output hashes and backend identities.
339372
#[must_use]
340373
pub fn compare_shadow_run(
341374
primary_engine: &str,
@@ -350,9 +383,7 @@ pub fn compare_shadow_run(
350383
compare_segments_with_tolerance(primary_segments, shadow_segments, tolerance);
351384
let replay_comparison = compare_replay_envelopes(primary_replay, shadow_replay);
352385

353-
let passes_gate = segment_comparison.within_tolerance()
354-
&& segment_comparison.text_mismatches == 0
355-
&& segment_comparison.timestamp_violations == 0;
386+
let passes_gate = segment_comparison.within_tolerance();
356387

357388
ShadowRunReport {
358389
primary_engine: primary_engine.to_owned(),
@@ -806,6 +837,48 @@ mod tests {
806837
assert!(err.to_string().contains("confidence"));
807838
}
808839

840+
#[test]
841+
fn rejects_nan_start_sec() {
842+
let seg = segment(Some(f64::NAN), Some(1.0), "text");
843+
let err = validate_segment_invariants(&[seg]).expect_err("NaN start_sec should fail");
844+
assert!(err.to_string().contains("not finite"));
845+
}
846+
847+
#[test]
848+
fn rejects_nan_end_sec() {
849+
let seg = segment(Some(0.0), Some(f64::NAN), "text");
850+
let err = validate_segment_invariants(&[seg]).expect_err("NaN end_sec should fail");
851+
assert!(err.to_string().contains("not finite"));
852+
}
853+
854+
#[test]
855+
fn rejects_infinite_start_sec() {
856+
let seg = segment(Some(f64::INFINITY), Some(1.0), "text");
857+
let err = validate_segment_invariants(&[seg]).expect_err("infinite start_sec should fail");
858+
assert!(err.to_string().contains("not finite"));
859+
}
860+
861+
#[test]
862+
fn rejects_infinite_end_sec() {
863+
let seg = segment(Some(0.0), Some(f64::INFINITY), "text");
864+
let err = validate_segment_invariants(&[seg]).expect_err("infinite end_sec should fail");
865+
assert!(err.to_string().contains("not finite"));
866+
}
867+
868+
#[test]
869+
fn rejects_negative_start_sec() {
870+
let seg = segment(Some(-1.0), Some(1.0), "text");
871+
let err = validate_segment_invariants(&[seg]).expect_err("negative start_sec should fail");
872+
assert!(err.to_string().contains("negative"));
873+
}
874+
875+
#[test]
876+
fn rejects_negative_end_sec() {
877+
let seg = segment(Some(0.0), Some(-0.5), "text");
878+
let err = validate_segment_invariants(&[seg]).expect_err("negative end_sec should fail");
879+
assert!(err.to_string().contains("negative"));
880+
}
881+
809882
#[test]
810883
fn accepts_zero_length_segment() {
811884
// start == end is valid (a point in time).
@@ -1344,4 +1417,140 @@ mod tests {
13441417
"contract must document fallback policy"
13451418
);
13461419
}
1420+
1421+
// ── Task #205 — conformance edge-case tests ──────────────────────
1422+
1423+
#[test]
1424+
fn negative_zero_timestamps_accepted_by_validation() {
1425+
// IEEE 754: -0.0 == 0.0, so -0.0 < 0.0 is false.
1426+
// Document that -0.0 passes the negativity check.
1427+
let segments = vec![
1428+
TranscriptionSegment {
1429+
start_sec: Some(-0.0_f64),
1430+
end_sec: Some(-0.0_f64),
1431+
text: "neg zero".to_owned(),
1432+
speaker: None,
1433+
confidence: None,
1434+
},
1435+
TranscriptionSegment {
1436+
start_sec: Some(0.0),
1437+
end_sec: Some(1.0),
1438+
text: "normal".to_owned(),
1439+
speaker: None,
1440+
confidence: None,
1441+
},
1442+
];
1443+
assert!(
1444+
validate_segment_invariants(&segments).is_ok(),
1445+
"negative zero should be accepted (IEEE 754: -0.0 == 0.0)"
1446+
);
1447+
}
1448+
1449+
#[test]
1450+
fn rollout_stage_parse_whitespace_and_mixed_case() {
1451+
// parse() trims and lowercases before matching — exercise that path.
1452+
assert_eq!(
1453+
NativeEngineRolloutStage::parse(" SHADOW "),
1454+
Some(NativeEngineRolloutStage::Shadow)
1455+
);
1456+
assert_eq!(
1457+
NativeEngineRolloutStage::parse(" Primary "),
1458+
Some(NativeEngineRolloutStage::Primary)
1459+
);
1460+
assert_eq!(
1461+
NativeEngineRolloutStage::parse("\tSole\n"),
1462+
Some(NativeEngineRolloutStage::Sole)
1463+
);
1464+
// Whitespace-only → empty after trim → None.
1465+
assert_eq!(NativeEngineRolloutStage::parse(" "), None);
1466+
}
1467+
1468+
#[test]
1469+
fn compare_shadow_run_passes_replay_but_fails_segment_gate() {
1470+
use super::compare_shadow_run;
1471+
1472+
// Identical replays → replay_comparison.within_tolerance() is true.
1473+
let replay = ReplayEnvelope {
1474+
input_content_hash: Some("aaa".to_owned()),
1475+
backend_identity: Some("whisper-cpp".to_owned()),
1476+
backend_version: Some("1.0".to_owned()),
1477+
output_payload_hash: Some("bbb".to_owned()),
1478+
};
1479+
1480+
// Segments diverge in text → segment_comparison fails.
1481+
let primary = vec![segment(Some(0.0), Some(1.0), "hello world")];
1482+
let shadow = vec![segment(Some(0.0), Some(1.0), "goodbye world")];
1483+
1484+
let tolerance = SegmentCompatibilityTolerance {
1485+
require_text_exact: true,
1486+
require_speaker_exact: false,
1487+
timestamp_tolerance_sec: 0.5,
1488+
};
1489+
1490+
let report = compare_shadow_run(
1491+
"bridge", "native", &primary, &shadow, &replay, &replay, tolerance,
1492+
);
1493+
1494+
// Gate depends only on segment comparison, not replay comparison.
1495+
assert!(!report.passes_gate, "gate should fail on text mismatch");
1496+
assert!(
1497+
report.replay_comparison.within_tolerance(),
1498+
"replay comparison should pass (identical envelopes)"
1499+
);
1500+
}
1501+
1502+
#[test]
1503+
fn none_end_sec_does_not_reset_previous_end_for_overlap_check() {
1504+
// seg0 ends at 2.0, seg1 has end_sec=None (previous_end stays 2.0),
1505+
// seg2 starts at 2.0 — should NOT trigger overlap because
1506+
// 2.0 + epsilon < 2.0 is false.
1507+
let segments = vec![
1508+
segment(Some(0.0), Some(2.0), "first"),
1509+
TranscriptionSegment {
1510+
start_sec: Some(2.0),
1511+
end_sec: None,
1512+
text: "no end".to_owned(),
1513+
speaker: None,
1514+
confidence: None,
1515+
},
1516+
segment(Some(2.0), Some(3.0), "third"),
1517+
];
1518+
assert!(
1519+
validate_segment_invariants(&segments).is_ok(),
1520+
"seg2 starting at previous seg0's end should not be flagged as overlap"
1521+
);
1522+
}
1523+
1524+
#[test]
1525+
fn compare_segments_speaker_mismatch_ignored_when_not_required() {
1526+
// require_speaker_exact = false with genuinely different speakers
1527+
// → speaker_mismatches should be 0.
1528+
let expected = vec![TranscriptionSegment {
1529+
start_sec: Some(0.0),
1530+
end_sec: Some(1.0),
1531+
text: "hello".to_owned(),
1532+
speaker: Some("SPEAKER_00".to_owned()),
1533+
confidence: None,
1534+
}];
1535+
let observed = vec![TranscriptionSegment {
1536+
start_sec: Some(0.0),
1537+
end_sec: Some(1.0),
1538+
text: "hello".to_owned(),
1539+
speaker: Some("SPEAKER_01".to_owned()),
1540+
confidence: None,
1541+
}];
1542+
1543+
let tolerance = SegmentCompatibilityTolerance {
1544+
require_text_exact: true,
1545+
require_speaker_exact: false,
1546+
timestamp_tolerance_sec: 0.5,
1547+
};
1548+
1549+
let report = compare_segments_with_tolerance(&expected, &observed, tolerance);
1550+
assert_eq!(
1551+
report.speaker_mismatches, 0,
1552+
"speaker mismatch should not be counted when require_speaker_exact is false"
1553+
);
1554+
assert!(report.within_tolerance());
1555+
}
13471556
}

0 commit comments

Comments
 (0)