From 7a1ae43f00ee99c5b6a25deae8d8c0471908df7e Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Fri, 31 Jul 2026 11:31:29 +0200 Subject: [PATCH] fix(models): correct thinking config for gateway model IDs and new model generations - thinking.py now normalizes model IDs to their claude- core, so Bedrock (anthropic.claude-..., us.anthropic....-v1:0), LiteLLM (anthropic/claude-...), and Vertex (claude-...@date) IDs classify like bare IDs instead of falling into the budget_tokens path (400 on >=4.7 generation models). - Inverted classification: adaptive thinking is the default; only the frozen legacy budget_tokens families (2.x/3.x, Haiku 4.5, Sonnet 4/4.5, Opus 4/4.1/4.5) keep the integer budget, so unknown future Claude models work by default. - New accepts_sampling_params() / supports_disabled_thinking() helpers; AndroidAgent act defaults now use make_non_thinking_settings(), which drops temperature on models that reject sampling params (Opus 4.7+, Sonnet 5, Fable 5) and omits thinking "disabled" on always-on models (Fable 5 / Mythos 5). - EffortLevel gains "xhigh" (supported from the Opus 4.7 generation on). Co-Authored-By: Claude Fable 5 --- src/askui/android_agent.py | 8 +- .../models/shared/agent_message_param.py | 5 +- src/askui/models/shared/thinking.py | 148 ++++++++++++++++-- tests/unit/models/test_thinking.py | 80 ++++++++++ 4 files changed, 219 insertions(+), 22 deletions(-) diff --git a/src/askui/android_agent.py b/src/askui/android_agent.py index 956655e7..916b78ec 100644 --- a/src/askui/android_agent.py +++ b/src/askui/android_agent.py @@ -12,6 +12,7 @@ from askui.models.models import Point from askui.models.shared.secrets import Secret from askui.models.shared.settings import ActSettings, MessageSettings +from askui.models.shared.thinking import make_non_thinking_settings from askui.models.shared.tools import Tool from askui.models.shared.truncation_strategies import TruncationStrategy from askui.prompts.act_prompts import create_android_agent_prompt @@ -116,12 +117,13 @@ def __init__( image_scaler=self._vlm_provider.image_scaler, ) self.act_tool_collection.add_agent_os(self.act_agent_os_facade) - # Override default act settings with Android-specific settings + # Override default act settings with Android-specific settings: + # thinking disabled, temperature 0 — where the model still accepts + # those (newer generations reject one or both). self.act_settings = ActSettings( messages=MessageSettings( system=create_android_agent_prompt(), - thinking={"type": "disabled"}, - temperature=0.0, + **make_non_thinking_settings(self._vlm_provider.model_id), ), ) diff --git a/src/askui/models/shared/agent_message_param.py b/src/askui/models/shared/agent_message_param.py index 07e118e5..200c0624 100644 --- a/src/askui/models/shared/agent_message_param.py +++ b/src/askui/models/shared/agent_message_param.py @@ -136,8 +136,9 @@ class BetaRedactedThinkingBlock(BaseModel): # adaptive thinking (`thinking={"type": "adaptive"}`). It replaces the integer # `budget_tokens` used with `thinking={"type": "enabled", ...}` on older models. # Anthropic maps this to `output_config.effort`; providers that do not support -# it (e.g. OpenAI chat) ignore it. -EffortLevel = Literal["low", "medium", "high", "max"] +# it (e.g. OpenAI chat) ignore it. "xhigh" (between "high" and "max") is +# supported from the Opus 4.7 generation onward; the 4.6 generation rejects it. +EffortLevel = Literal["low", "medium", "high", "xhigh", "max"] class UsageParam(BaseModel): diff --git a/src/askui/models/shared/thinking.py b/src/askui/models/shared/thinking.py index a3ff58c4..b668a12b 100644 --- a/src/askui/models/shared/thinking.py +++ b/src/askui/models/shared/thinking.py @@ -15,6 +15,14 @@ rejected, and ``effort`` is a separate parameter sent via ``output_config.effort`` (not part of ``thinking``). +The budget-token generation is a closed set, so classification is inverted: +every Claude model *not* in the frozen legacy list is treated as adaptive, +which makes unknown future models work by default. Model IDs are matched on +their ``claude-...`` core, so gateway-prefixed identifiers (Bedrock +``anthropic.claude-opus-4-8`` or ``us.anthropic.claude-...-v1:0``, LiteLLM +``anthropic/claude-...``, Vertex ``claude-...@20260401``) resolve like the +bare model ID. + `make_thinking_settings()` returns the `MessageSettings` keyword arguments that enable thinking for a given ``model_id``, so agents can turn thinking on by default without knowing which generation they run on. Callers can still override @@ -27,33 +35,115 @@ _DEFAULT_BUDGET_TOKENS = 2048 -# Model-ID prefixes for Anthropic models that use adaptive thinking. These are -# the models where the integer `budget_tokens` is removed or deprecated in favour -# of adaptive thinking (`{"type": "adaptive"}`) plus the `effort` setting. -# `str.startswith` matches dated snapshots too (e.g. "claude-sonnet-4-6-20260401"). -# Note: "claude-sonnet-5" does not match the older "claude-sonnet-4-5". -_ADAPTIVE_THINKING_MODEL_PREFIXES = ( +# Model-ID prefixes (after normalization) of the Anthropic model families that +# take the fixed integer `budget_tokens`. This set is FROZEN: budget thinking +# was replaced by adaptive thinking with the 4.6 generation, so no future model +# will ever be added here. +_LEGACY_BUDGET_THINKING_MODEL_PREFIXES = ( + "claude-2", + "claude-instant", + "claude-3-", + "claude-haiku-4-5", + "claude-sonnet-4-0", + "claude-sonnet-4-1", + "claude-sonnet-4-2", # dated snapshots, e.g. "claude-sonnet-4-20250514" + "claude-sonnet-4-5", + "claude-opus-4-0", + "claude-opus-4-1", + "claude-opus-4-2", # dated snapshots, e.g. "claude-opus-4-20250514" + "claude-opus-4-5", +) + +# The one adaptive-thinking generation that still accepts sampling parameters +# (temperature/top_p/top_k). From Opus 4.7 / Sonnet 5 / Fable 5 onward the API +# rejects them with a 400. +_SAMPLING_CAPABLE_ADAPTIVE_MODEL_PREFIXES = ( "claude-sonnet-4-6", - "claude-sonnet-5", "claude-opus-4-6", - "claude-opus-4-7", - "claude-opus-4-8", - "claude-opus-5", +) + +# Models where thinking is always on: an explicit {"type": "disabled"} is +# rejected with a 400, so the thinking field must be omitted entirely. +_ALWAYS_ON_THINKING_MODEL_PREFIXES = ( "claude-fable-5", + "claude-mythos", ) +def _normalize(model_id: str) -> str | None: + """Extract the ``claude-...`` core of a model ID. + + Gateway wrappers then match like bare IDs (e.g. + ``"us.anthropic.claude-opus-4-8-v1:0"`` and ``"anthropic/claude-opus-4-8"`` + both normalize to ``"claude-opus-4-8..."``). + + Args: + model_id (str): The (possibly gateway-prefixed) model identifier. + + Returns: + str | None: The model ID from its ``claude-`` core onward, or ``None`` + if the ID does not reference a Claude model. + """ + index = model_id.find("claude-") + return None if index < 0 else model_id[index:] + + def uses_adaptive_thinking(model_id: str) -> bool: """Whether ``model_id`` uses adaptive thinking instead of a token budget. + True for every Claude model outside the frozen legacy budget families (so + unknown future models default to adaptive); False for non-Claude model IDs. + Args: - model_id (str): The Anthropic model identifier. + model_id (str): The model identifier (bare or gateway-prefixed). Returns: bool: ``True`` if the model expects ``{"type": "adaptive"}`` and the `effort` setting, ``False`` if it expects a fixed ``budget_tokens``. """ - return model_id.startswith(_ADAPTIVE_THINKING_MODEL_PREFIXES) + normalized = _normalize(model_id) + return normalized is not None and not normalized.startswith( + _LEGACY_BUDGET_THINKING_MODEL_PREFIXES + ) + + +def accepts_sampling_params(model_id: str) -> bool: + """Whether the model accepts sampling parameters such as ``temperature``. + + False for adaptive-thinking Claude models newer than the 4.6 generation + (Opus 4.7/4.8, Sonnet 5, Fable 5, and future models), which reject them + with a 400. True for older Claude models and non-Claude model IDs (other + providers manage their own sampling parameters). + + Args: + model_id (str): The model identifier (bare or gateway-prefixed). + + Returns: + bool: ``True`` if sampling parameters may be sent to the model. + """ + normalized = _normalize(model_id) + return normalized is None or normalized.startswith( + _LEGACY_BUDGET_THINKING_MODEL_PREFIXES + + _SAMPLING_CAPABLE_ADAPTIVE_MODEL_PREFIXES + ) + + +def supports_disabled_thinking(model_id: str) -> bool: + """Whether the model accepts an explicit ``{"type": "disabled"}`` thinking config. + + False for always-on-thinking models (Fable 5, Mythos 5), which reject it + with a 400 — omit the thinking field there. + + Args: + model_id (str): The model identifier (bare or gateway-prefixed). + + Returns: + bool: ``True`` if ``{"type": "disabled"}`` may be sent to the model. + """ + normalized = _normalize(model_id) + return normalized is None or not normalized.startswith( + _ALWAYS_ON_THINKING_MODEL_PREFIXES + ) def make_thinking_settings( @@ -75,11 +165,11 @@ def make_thinking_settings( ``thinking={"type": "enabled", "budget_tokens": 2048}`` and ignore ``effort``. Args: - model_id (str): The Anthropic model identifier. - effort (EffortLevel | None, optional): How much the model should think and - act (``"low"``, ``"medium"``, ``"high"`` or ``"max"``). Only applied - for models that support adaptive thinking. Default: None (the model - uses its own default). + model_id (str): The model identifier (bare or gateway-prefixed). + effort (EffortLevel | None, optional): How much the model should think + and act (``"low"``, ``"medium"``, ``"high"``, ``"xhigh"`` or + ``"max"``). Only applied for models that support adaptive thinking. + Default: None (the model uses its own default). Returns: dict[str, Any]: `MessageSettings` keyword arguments (``thinking`` and, @@ -91,3 +181,27 @@ def make_thinking_settings( settings["provider_options"] = {"output_config": {"effort": effort}} return settings return {"thinking": {"type": "enabled", "budget_tokens": _DEFAULT_BUDGET_TOKENS}} + + +def make_non_thinking_settings(model_id: str) -> dict[str, Any]: + """Return `MessageSettings` keyword arguments for thinking-off, deterministic runs. + + Used by device agents (Android) that historically pinned + ``thinking={"type": "disabled"}`` and ``temperature=0.0``. Each field is + included only where the model still accepts it: models from the Opus 4.7 + generation onward reject sampling parameters, and always-on-thinking models + (Fable 5) reject an explicit ``"disabled"``. + + Args: + model_id (str): The model identifier (bare or gateway-prefixed). + + Returns: + dict[str, Any]: `MessageSettings` keyword arguments (``thinking`` + and/or ``temperature``, possibly empty). + """ + settings: dict[str, Any] = {} + if supports_disabled_thinking(model_id): + settings["thinking"] = {"type": "disabled"} + if accepts_sampling_params(model_id): + settings["temperature"] = 0.0 + return settings diff --git a/tests/unit/models/test_thinking.py b/tests/unit/models/test_thinking.py index 689eedc9..58682494 100644 --- a/tests/unit/models/test_thinking.py +++ b/tests/unit/models/test_thinking.py @@ -3,7 +3,10 @@ import pytest from askui.models.shared.thinking import ( + accepts_sampling_params, + make_non_thinking_settings, make_thinking_settings, + supports_disabled_thinking, uses_adaptive_thinking, ) @@ -17,6 +20,15 @@ "claude-opus-4-8", "claude-opus-5", "claude-fable-5", + "claude-haiku-5", # unknown future model -> adaptive by default + # Gateway-prefixed IDs (Bedrock, LiteLLM, Vertex) classify like bare IDs. + "anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8-v1:0", + "eu.anthropic.claude-sonnet-5-v1:0", + "anthropic/claude-opus-4-8", + "anthropic/claude-fable-5", + "bedrock/us.anthropic.claude-opus-4-7-v1:0", + "vertex_ai/claude-sonnet-4-6", ] _BUDGET_MODELS = [ @@ -25,6 +37,11 @@ "claude-opus-4-5-20251101", "claude-opus-4-1-20250805", "claude-haiku-4-5-20251001", + "claude-3-5-sonnet-20241022", + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "anthropic/claude-haiku-4-5", + "claude-opus-4-5@20251101", # Vertex version separator "gpt-5.4", "some-unknown-model", ] @@ -62,3 +79,66 @@ def test_sonnet_5_is_not_confused_with_sonnet_4_5() -> None: # "claude-sonnet-5" must not match the older "claude-sonnet-4-5" prefix. assert uses_adaptive_thinking("claude-sonnet-5") is True assert uses_adaptive_thinking("claude-sonnet-4-5") is False + + +@pytest.mark.parametrize( + ("model_id", "expected"), + [ + ("claude-sonnet-4-5-20250929", True), # legacy: sampling params fine + ("claude-haiku-4-5", True), + ("claude-sonnet-4-6", True), # 4.6 generation still accepts them + ("claude-opus-4-6", True), + ("claude-opus-4-7", False), # removed from Opus 4.7 onward + ("claude-opus-4-8", False), + ("claude-sonnet-5", False), + ("claude-fable-5", False), + ("anthropic.claude-opus-4-8", False), # gateway IDs classified too + ("anthropic/claude-sonnet-5", False), + ("gpt-5.4", True), # non-Claude: other providers manage their own + ], +) +def test_accepts_sampling_params(model_id: str, expected: bool) -> None: + assert accepts_sampling_params(model_id) is expected + + +@pytest.mark.parametrize( + ("model_id", "expected"), + [ + ("claude-sonnet-4-6", True), + ("claude-opus-4-8", True), + ("claude-sonnet-5", True), + ("claude-fable-5", False), # always-on thinking rejects "disabled" + ("claude-mythos-5", False), + ("anthropic/claude-fable-5", False), + ("gpt-5.4", True), + ], +) +def test_supports_disabled_thinking(model_id: str, expected: bool) -> None: + assert supports_disabled_thinking(model_id) is expected + + +def test_non_thinking_settings_keep_parity_on_older_models() -> None: + assert make_non_thinking_settings("claude-sonnet-4-6") == { + "thinking": {"type": "disabled"}, + "temperature": 0.0, + } + + +def test_non_thinking_settings_drop_temperature_from_opus_4_7_on() -> None: + assert make_non_thinking_settings("claude-opus-4-8") == { + "thinking": {"type": "disabled"}, + } + assert make_non_thinking_settings("anthropic/claude-sonnet-5") == { + "thinking": {"type": "disabled"}, + } + + +def test_non_thinking_settings_omit_thinking_on_always_on_models() -> None: + assert make_non_thinking_settings("claude-fable-5") == {} + + +def test_effort_supports_xhigh() -> None: + assert make_thinking_settings("claude-opus-4-8", effort="xhigh") == { + "thinking": {"type": "adaptive"}, + "provider_options": {"output_config": {"effort": "xhigh"}}, + }