Skip to content

Commit a5acde7

Browse files
test(gauntlet-timing): recognize the unlimited_vision.hydrate stage
The gauntlet perf harness parses FOCR_TIMING=1 output into named stages so the ledger and A/B aggregator can fold seconds per pipeline stage. The Unlimited-OCR forward now emits an "unlimited_vision.hydrate(cached)" line on ordinary cached-tower runs and an "unlimited_vision.hydrate(batch-local)" line on the batch spine path, accounting the one-time hydration of the SAM/CLIP tower, projector, and widened embed tables that are cached on the loaded model and reused across pages and batches. Before this change the parser rejected those lines as unknown, failing any timing capture that included them. Add a dedicated `unlimited_vision_hydrate` pattern (ordered next to the existing batched `vision.hydrate(batch)` info stage) that matches both the cached and batch-local variants, and extend the synthetic mixed-precision and batch fixtures plus the self-test so both variants are exercised and their seconds asserted. This keeps the parser in lockstep with the runtime instrumentation without touching any source code; the full parser self-test stays green (95/95). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent f320f8a commit a5acde7

1 file changed

Lines changed: 17 additions & 0 deletions

File tree

scripts/gauntlet_timing.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,13 @@
9999
"vision_hydrate",
100100
re.compile(r"^vision\.hydrate\(batch\) (?P<s>\d+(?:\.\d+)?)s$"),
101101
),
102+
(
103+
"unlimited_vision_hydrate",
104+
re.compile(
105+
r"^unlimited_vision\.hydrate\((?:cached|batch-local)\) "
106+
r"(?P<s>\d+(?:\.\d+)?)s$"
107+
),
108+
),
102109
(
103110
"vision_sam_batch",
104111
re.compile(
@@ -1561,6 +1568,7 @@ def _cmd_aggregate_ab(args: argparse.Namespace) -> int:
15611568
_SYNTHETIC_UNLIMITED_MIXED = """\
15621569
[focr-timing] precision focr-mixed-ffn-int8
15631570
[focr-timing] preprocess 0.12s
1571+
[focr-timing] unlimited_vision.hydrate(cached) 0.11s
15641572
[focr-timing] vision.sam 1.20s
15651573
[focr-timing] vision.clip 0.40s
15661574
[focr-timing] vision.bridge 0.05s
@@ -1583,6 +1591,7 @@ def _cmd_aggregate_ab(args: argparse.Namespace) -> int:
15831591
# sam, single lines for hydrate/clip/bridge; the second sam group proves the
15841592
# same-stage fold sums seconds and view counts).
15851593
_SYNTHETIC_BATCH = """\
1594+
[focr-timing] unlimited_vision.hydrate(batch-local) 0.21s
15861595
[focr-timing] vision.hydrate(batch) 0.62s
15871596
[focr-timing] vision.sam(batch of 3, side 1024) 2.40s
15881597
[focr-timing] vision.sam(batch of 2, side 640) 0.80s
@@ -1615,6 +1624,10 @@ def check(name: str, ok: bool, **fields: object) -> None:
16151624
"unlimited-mixed-precision",
16161625
infer_precision([mixed_run]) == PRECISION_MIXED_FFN_INT8,
16171626
)
1627+
check(
1628+
"unlimited-mixed-vision-hydrate",
1629+
math.isclose(mixed_run["unlimited_vision_hydrate"]["s"], 0.11),
1630+
)
16181631
check(
16191632
"unlimited-mixed-phases",
16201633
math.isclose(mixed_run["decode_lm_head"]["s"], 1.6)
@@ -1688,6 +1701,10 @@ def check(name: str, ok: bool, **fields: object) -> None:
16881701
# bd-t6a batched-vision lines: hydrate is its own stage; sam/clip/bridge
16891702
# batch lines fold into the per-view stage names, summing seconds + views.
16901703
batch = parse_run(_SYNTHETIC_BATCH)
1704+
check(
1705+
"batch-unlimited-vision-hydrate",
1706+
math.isclose(batch["unlimited_vision_hydrate"]["s"], 0.21),
1707+
)
16911708
check("batch-hydrate", math.isclose(batch["vision_hydrate"]["s"], 0.62))
16921709
check("batch-sam-sum", math.isclose(batch["vision_sam"]["s"], 3.20))
16931710
check("batch-sam-views", batch["vision_sam"]["views"] == 5)

0 commit comments

Comments
 (0)