From d3ed57252cb2c3f459b4840c08f8efeb19bcf1eb Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 07:32:46 +0100 Subject: [PATCH 01/16] chore(platform): fix race condition in e2e test due to caching --- src/aignostics/platform/resources/runs.py | 3 +- tests/aignostics/application/cli_test.py | 51 ++++++++++++++++++++--- tests/aignostics/application/gui_test.py | 49 ++++++++++++---------- tests/aignostics/platform/e2e_test.py | 32 ++++++++------ tests/constants_test.py | 4 -- 5 files changed, 93 insertions(+), 46 deletions(-) diff --git a/src/aignostics/platform/resources/runs.py b/src/aignostics/platform/resources/runs.py index 8f4431f63..882e75cff 100644 --- a/src/aignostics/platform/resources/runs.py +++ b/src/aignostics/platform/resources/runs.py @@ -226,7 +226,7 @@ def download_to_folder( # incrementally check for available results application_run_state = self.details().state while application_run_state in {RunState.PROCESSING, RunState.PENDING}: - for item in self.results(): + for item in self.results(): # we accept this might be stale for a bit if item.state == ItemState.TERMINATED and item.output == ItemOutput.FULL: self.ensure_artifacts_downloaded(application_run_dir, item, checksum_attribute_key) sleep(5) @@ -234,6 +234,7 @@ def download_to_folder( print(self) # check if last results have been downloaded yet and report on errors + operation_cache_clear() # clear cache to get fresh results for item in self.results(): match item.output: case ItemOutput.FULL: diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py index 8edfd6cf9..f725145bc 100644 --- a/tests/aignostics/application/cli_test.py +++ b/tests/aignostics/application/cli_test.py @@ -21,6 +21,12 @@ MESSAGE_RUN_NOT_FOUND = "Warning: Run with ID '4711' not found" +TEST_APPLICATION_DEADLINE_SECONDS = 60 * 45 # 45 minutes +TEST_APPLICATION_DUE_DATE_SECONDS = 60 * 10 # 10 minutes + +HETA_APPLICATION_DUE_DATE_SECONDS = 60 * 60 * 1 # 1 hour +HETA_APPLICATION_DEADLINE_SECONDS = 60 * 60 * 3 # 3 hours + @pytest.mark.e2e @pytest.mark.timeout(timeout=60) @@ -157,7 +163,18 @@ def test_cli_run_submit_fails_on_application_not_found(runner: CliRunner, tmp_pa csv_path = tmp_path / "dummy.csv" csv_path.write_text(csv_content) - result = runner.invoke(cli, ["application", "run", "submit", "wrong", str(csv_path)]) + result = runner.invoke( + cli, + [ + "application", + "run", + "submit", + "wrong", + str(csv_path), + "--deadline", + (datetime.now(tz=UTC) + timedelta(minutes=10)).isoformat(), + ], + ) assert result.exit_code == 2 assert 'HTTP response body: {"detail":"application not found"}' in normalize_output(result.stdout) @@ -175,7 +192,18 @@ def test_cli_run_submit_fails_on_unsupported_cloud(runner: CliRunner, tmp_path: csv_path = tmp_path / "dummy.csv" csv_path.write_text(csv_content) - result = runner.invoke(cli, ["application", "run", "submit", HETA_APPLICATION_ID, str(csv_path)]) + result = runner.invoke( + cli, + [ + "application", + "run", + "submit", + HETA_APPLICATION_ID, + str(csv_path), + "--deadline", + (datetime.now(tz=UTC) + timedelta(minutes=10)).isoformat(), + ], + ) assert result.exit_code == 2 assert "Invalid platform bucket URL: 'aws://bucket/test'" in normalize_output(result.stdout) @@ -191,7 +219,18 @@ def test_cli_run_submit_fails_on_missing_url(runner: CliRunner, tmp_path: Path) csv_path = tmp_path / "dummy.csv" csv_path.write_text(csv_content) - result = runner.invoke(cli, ["application", "run", "submit", HETA_APPLICATION_ID, str(csv_path)]) + result = runner.invoke( + cli, + [ + "application", + "run", + "submit", + HETA_APPLICATION_ID, + str(csv_path), + "--deadline", + (datetime.now(tz=UTC) + timedelta(minutes=10)).isoformat(), + ], + ) assert result.exit_code == 2 assert "Invalid platform bucket URL: ''" in normalize_output(result.stdout) @@ -416,7 +455,7 @@ def test_cli_run_result_delete_fails_on_no_arg(runner: CliRunner) -> None: @pytest.mark.e2e @pytest.mark.very_long_running -@pytest.mark.timeout(timeout=60 * 60 * 5) +@pytest.mark.timeout(timeout=HETA_APPLICATION_DEADLINE_SECONDS + 60 * 30) def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: """Check run execution runs e2e.""" # Step 1: Download the sample file @@ -451,9 +490,9 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: ".*\\.tiff:staining_method=H&E,tissue=LUNG,disease=LUNG_CANCER", "--no-create-subdirectory-for-run", "--due-date", - (datetime.now(tz=UTC) + timedelta(hours=1)).isoformat(), + (datetime.now(tz=UTC) + timedelta(seconds=HETA_APPLICATION_DUE_DATE_SECONDS)).isoformat(), "--deadline", - (datetime.now(tz=UTC) + timedelta(hours=3)).isoformat(), + (datetime.now(tz=UTC) + timedelta(seconds=HETA_APPLICATION_DEADLINE_SECONDS)).isoformat(), "--validate-only", ], ) diff --git a/tests/aignostics/application/gui_test.py b/tests/aignostics/application/gui_test.py index e47c0387f..0b9f9a816 100644 --- a/tests/aignostics/application/gui_test.py +++ b/tests/aignostics/application/gui_test.py @@ -16,7 +16,7 @@ from aignostics.cli import cli from aignostics.utils import get_logger from tests.conftest import assert_notified, normalize_output, print_directory_structure -from tests.constants_test import HETA_APPLICATION_ID, HETA_APPLICATION_VERSION +from tests.constants_test import HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, HETA_SINGLE_SPOT_GS_URL if TYPE_CHECKING: from nicegui import ui @@ -251,6 +251,7 @@ async def test_gui_download_dataset_via_application_to_run_cancel( # noqa: PLR0 await user.should_see("Soft Due Date", retries=100) await user.should_see("The platform will try to complete the run before this time", retries=100) user.find(marker="BUTTON_SCHEDULING_NEXT").click() + # TODO(Helmut): Set short deadline via GUI await assert_notified(user, "Prepared upload UI.") # Now on Submission step @@ -300,32 +301,36 @@ async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, s "aignostics.application._gui._page_application_run_describe.get_user_data_directory", return_value=tmp_path, ): - application = Service().application(HETA_APPLICATION_ID) - latest_version_number = application.versions[0].number if application.versions else None - assert latest_version_number is not None, f"No versions found for application {HETA_APPLICATION_ID}" - # This assumes a successful HETA run is in the last 200 completed runs - runs = Service().application_runs(limit=200, has_output=True) - + # Find run + runs = Service().application_runs( + application_id=HETA_APPLICATION_ID, + application_version=HETA_APPLICATION_VERSION, + external_id=HETA_SINGLE_SPOT_GS_URL, + has_output=True, + limit=1, + ) if not runs: - pytest.fail("No completed runs found, please run other tests first.") - # Find a completed run with the latest application version ID - run = None - for potential_run in runs: - if ( - potential_run.application_id == application.application_id - and potential_run.version_number == latest_version_number - ): - run = potential_run - break - if not run: - pytest.skip(f"No completed runs found with {application.application_id} ({latest_version_number})") - + message = f"No matching runs found for application {HETA_APPLICATION_ID} ({HETA_APPLICATION_VERSION}). " + message += "This test requires the scheduled test test_application_runs_heta_version passing first." + pytest.skip(message) + + run_id = runs[0].run_id + + # Explore run + run = Service().application_run(run_id).details() + print( + f"Found existing run: {run.run_id}\n" + f"application: {run.application_id} ({run.version_number})\n" + f"status: {run.state}, output: {run.output}\n" + f"submitted at: {run.submitted_at}, terminated at: {run.terminated_at}\n" + f"statistics: {run.statistics!r}\n", + f"custom_metadata: {run.custom_metadata!r}\n", + ) # Step 1: Go to latest completed run - print(f"Found existing run: {run.run_id}, status: {run.state}") await user.open(f"/application/run/{run.run_id}") await user.should_see(f"Run {run.run_id}", retries=100) await user.should_see( - f"Run of {application.application_id} ({latest_version_number})", + f"Run of {run.application_id} ({run.version_number})", retries=100, ) diff --git a/tests/aignostics/platform/e2e_test.py b/tests/aignostics/platform/e2e_test.py index 7464f63d0..ad58c5c08 100644 --- a/tests/aignostics/platform/e2e_test.py +++ b/tests/aignostics/platform/e2e_test.py @@ -23,15 +23,19 @@ from aignostics.platform.resources.runs import Run from tests.constants_test import ( HETA_APPLICATION_ID, - HETA_APPLICATION_TIMEOUT_SECONDS, HETA_APPLICATION_VERSION, HETA_SINGLE_SPOT_GS_URL, TEST_APPLICATION_ID, - TEST_APPLICATION_TIMEOUT_SECONDS, TEST_APPLICATION_VERSION, TEST_THREE_SPOTS_GS_URLS, ) +TEST_APPLICATION_DEADLINE_SECONDS = 60 * 45 # 45 minutes +TEST_APPLICATION_DUE_DATE_SECONDS = 60 * 10 # 10 minutes + +HETA_APPLICATION_DUE_DATE_SECONDS = 60 * 60 * 1 # 1 hour +HETA_APPLICATION_DEADLINE_SECONDS = 60 * 60 * 3 # 3 hours + def _get_single_spot_payload_for_heta() -> list[platform.InputItem]: """Generates a payload using a single spot.""" @@ -43,7 +47,7 @@ def _get_single_spot_payload_for_heta() -> list[platform.InputItem]: name="whole_slide_image", download_url=platform.generate_signed_url( HETA_SINGLE_SPOT_GS_URL, - HETA_APPLICATION_TIMEOUT_SECONDS, + HETA_APPLICATION_DEADLINE_SECONDS, ), metadata={ "checksum_base64_crc32c": "5onqtA==", @@ -73,7 +77,7 @@ def _get_three_spots_payload_for_test() -> list[platform.InputItem]: name="whole_slide_image", download_url=platform.generate_signed_url( TEST_THREE_SPOTS_GS_URLS[0], - TEST_APPLICATION_TIMEOUT_SECONDS, + TEST_APPLICATION_DEADLINE_SECONDS, ), metadata={ "checksum_base64_crc32c": "9l3NNQ==", @@ -92,7 +96,7 @@ def _get_three_spots_payload_for_test() -> list[platform.InputItem]: name="whole_slide_image", download_url=platform.generate_signed_url( TEST_THREE_SPOTS_GS_URLS[1], - TEST_APPLICATION_TIMEOUT_SECONDS, + TEST_APPLICATION_DEADLINE_SECONDS, ), metadata={ "checksum_base64_crc32c": "w+ud3g==", @@ -111,7 +115,7 @@ def _get_three_spots_payload_for_test() -> list[platform.InputItem]: name="whole_slide_image", download_url=platform.generate_signed_url( TEST_THREE_SPOTS_GS_URLS[2], - TEST_APPLICATION_TIMEOUT_SECONDS, + TEST_APPLICATION_DEADLINE_SECONDS, ), metadata={ "checksum_base64_crc32c": "Zmx0wA==", @@ -130,9 +134,9 @@ def _run_application_test( # noqa: PLR0913, PLR0917 application_id: str, application_version: str, payload: list[platform.InputItem], + due_date_seconds: int, + deadline_seconds: int, checksum_attribute_key: str = "checksum_base64_crc32c", - due_date_seconds: int = 60 * 60, - deadline_seconds: int = 60 * 60 * 3, ) -> None: """Helper function to run an application test. @@ -142,9 +146,9 @@ def _run_application_test( # noqa: PLR0913, PLR0917 application_id (str): The application ID to use for the test. application_version (str): The application version to use for the test. payload (list[platform.InputItem]): The input items for the application run. - checksum_attribute_key (str): The key used to validate the checksum of the output artifacts. due_date_seconds (int): The due date in seconds from now for the application run. deadline_seconds (int): The deadline in seconds from now for the application run. + checksum_attribute_key (str): The key used to validate the checksum of the output artifacts. Raises: AssertionError: If any of the validation checks fail. @@ -173,7 +177,7 @@ def _run_application_test( # noqa: PLR0913, PLR0917 @pytest.mark.skip(reason="v0.0.4 on production balking on whole_slide_image input") @pytest.mark.e2e @pytest.mark.long_running -@pytest.mark.timeout(timeout=TEST_APPLICATION_TIMEOUT_SECONDS + 60 * 10) +@pytest.mark.timeout(timeout=TEST_APPLICATION_DEADLINE_SECONDS + 60 * 30) def test_application_runs_test_version() -> None: """Test application runs with the test application. @@ -188,14 +192,15 @@ def test_application_runs_test_version() -> None: application_id=TEST_APPLICATION_ID, application_version=TEST_APPLICATION_VERSION, payload=_get_three_spots_payload_for_test(), - deadline_seconds=TEST_APPLICATION_TIMEOUT_SECONDS, + deadline_seconds=TEST_APPLICATION_DEADLINE_SECONDS, + due_date_seconds=TEST_APPLICATION_DUE_DATE_SECONDS, ) @pytest.mark.e2e @pytest.mark.very_long_running @pytest.mark.scheduled_only -@pytest.mark.timeout(timeout=HETA_APPLICATION_TIMEOUT_SECONDS + 60 * 10) +@pytest.mark.timeout(timeout=HETA_APPLICATION_DEADLINE_SECONDS + 60 * 30) def test_application_runs_heta_version() -> None: """Test application runs with the HETA application. @@ -210,7 +215,8 @@ def test_application_runs_heta_version() -> None: application_id=HETA_APPLICATION_ID, application_version=HETA_APPLICATION_VERSION, payload=_get_single_spot_payload_for_heta(), - deadline_seconds=HETA_APPLICATION_TIMEOUT_SECONDS, + deadline_seconds=HETA_APPLICATION_DEADLINE_SECONDS, + due_date_seconds=HETA_APPLICATION_DUE_DATE_SECONDS, ) diff --git a/tests/constants_test.py b/tests/constants_test.py index 183bf4786..30dd3d280 100644 --- a/tests/constants_test.py +++ b/tests/constants_test.py @@ -20,19 +20,15 @@ case "production": TEST_APPLICATION_ID = "test-app" TEST_APPLICATION_VERSION = "0.0.4" - TEST_APPLICATION_TIMEOUT_SECONDS = 60 * 45 # 45 minutes HETA_APPLICATION_ID = "he-tme" HETA_APPLICATION_VERSION = "1.0.0-beta.8" - HETA_APPLICATION_TIMEOUT_SECONDS = 5 * 60 * 60 # 5 hours case "staging": TEST_APPLICATION_ID = "test-app" TEST_APPLICATION_VERSION = "0.0.5" - TEST_APPLICATION_TIMEOUT_SECONDS = 60 * 45 # 45 minutes HETA_APPLICATION_ID = "he-tme" HETA_APPLICATION_VERSION = "1.0.0-beta.8" - HETA_APPLICATION_TIMEOUT_SECONDS = 5 * 60 * 60 # 5 hours case _: message = f"Unsupported AIGNOSTICS_PLATFORM_ENVIRONMENT value: {os.getenv('AIGNOSTICS_PLATFORM_ENVIRONMENT')}" raise ValueError(message) From 81caf2bae1bb7afadd45f15a6662c0db030d8b5b Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 08:47:11 +0100 Subject: [PATCH 02/16] chore(application,platform): improve depth of tests feat(platform): support nocache=True on cached operations --- src/aignostics/platform/_client.py | 22 +- src/aignostics/platform/_operation_cache.py | 11 +- .../platform/resources/applications.py | 43 +- src/aignostics/platform/resources/runs.py | 30 +- tests/aignostics/application/cli_test.py | 64 +- tests/aignostics/application/gui_test.py | 69 +- tests/aignostics/platform/conftest.py | 15 + tests/aignostics/platform/e2e_test.py | 6 +- tests/aignostics/platform/nocache_test.py | 687 ++++++++++++++++++ tests/aignostics/qupath/gui_test.py | 85 ++- tests/constants_test.py | 13 + 11 files changed, 952 insertions(+), 93 deletions(-) create mode 100644 tests/aignostics/platform/nocache_test.py diff --git a/src/aignostics/platform/_client.py b/src/aignostics/platform/_client.py index 71af0b790..7aca7a60c 100644 --- a/src/aignostics/platform/_client.py +++ b/src/aignostics/platform/_client.py @@ -106,7 +106,7 @@ def __init__(self, cache_token: bool = True) -> None: logger.exception("Failed to initialize client.") raise - def me(self) -> Me: + def me(self, nocache: bool = False) -> Me: """Retrieves info about the current user and their organisation. Retries on network and server errors. @@ -115,6 +115,10 @@ def me(self) -> Me: - We are not using urllib3s retry class as it does not support fine grained definition when to retry, exponential backoff with jitter, logging before retry, and is difficult to configure. + Args: + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. + Returns: Me: User and organization information. @@ -136,15 +140,17 @@ def me_with_retry() -> Me: ) ) # Retryer will pass down arguments - return me_with_retry() + return me_with_retry(nocache=nocache) # type: ignore[call-arg] - def application(self, application_id: str) -> Application: + def application(self, application_id: str, nocache: bool = False) -> Application: """Find application by id. Retries on network and server errors. Args: application_id (str): The ID of the application. + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. Raises: NotFoundException: If the application with the given ID is not found. @@ -172,9 +178,11 @@ def application_with_retry(application_id: str) -> Application: ) ) - return application_with_retry(application_id) + return application_with_retry(application_id, nocache=nocache) # type: ignore[call-arg] - def application_version(self, application_id: str, version_number: str | None = None) -> ApplicationVersion: + def application_version( + self, application_id: str, version_number: str | None = None, nocache: bool = False + ) -> ApplicationVersion: """Find application version by id. Retries on network and server errors. @@ -183,6 +191,8 @@ def application_version(self, application_id: str, version_number: str | None = application_id (str): The ID of the application. version_number (str | None): The version number of the application. If None, the latest version will be retrieved. + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. Raises: NotFoundException: If the application with the given ID and version number is not found. @@ -227,7 +237,7 @@ def application_version_with_retry(application_id: str, version: str) -> Applica ) ) - return application_version_with_retry(application_id, version_number) + return application_version_with_retry(application_id, version_number, nocache=nocache) # type: ignore[call-arg] def run(self, run_id: str) -> Run: """Finds run by id. diff --git a/src/aignostics/platform/_operation_cache.py b/src/aignostics/platform/_operation_cache.py index a3281de0b..90b9cc8d7 100644 --- a/src/aignostics/platform/_operation_cache.py +++ b/src/aignostics/platform/_operation_cache.py @@ -108,10 +108,18 @@ def cached_operation( Returns: Callable: A decorator that caches the function result. + + Note: + The decorated function can accept a 'nocache' keyword argument (bool) to bypass + reading from the cache. When nocache=True, the function is executed directly + and the result is still cached for subsequent calls. """ def decorator(func: Callable[P, T]) -> Callable[P, T]: def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + # Check if nocache is requested and remove it from kwargs before passing to func + nocache = kwargs.pop("nocache", False) + # Build cache key components cache_args: tuple[object, ...] = args @@ -129,7 +137,8 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: else: key = cache_key(func_qualified_name, *cache_args, **kwargs) - if key in _operation_cache: + # If nocache=True, skip cache lookup but still cache the result + if not nocache and key in _operation_cache: result, expiry = _operation_cache[key] if time.time() < expiry: return t.cast("T", result) diff --git a/src/aignostics/platform/resources/applications.py b/src/aignostics/platform/resources/applications.py index 48050f787..e9704323c 100644 --- a/src/aignostics/platform/resources/applications.py +++ b/src/aignostics/platform/resources/applications.py @@ -57,13 +57,15 @@ def __init__(self, api: PublicApi) -> None: """ self._api = api - def list(self, application: Application | str) -> list[VersionTuple]: + def list(self, application: Application | str, nocache: bool = False) -> list[VersionTuple]: """Find all versions for a specific application. Retries on network and server errors. Args: application (Application | str): The application to find versions for, either object or id + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. Returns: list[VersionTuple]: List of the available application versions. @@ -91,10 +93,12 @@ def list_with_retry(app_id: str) -> Application: ) ) - app = list_with_retry(application_id) + app = list_with_retry(application_id, nocache=nocache) # type: ignore[call-arg] return app.versions if app.versions is not None else [] - def details(self, application_id: str, application_version: VersionTuple | str | None = None) -> ApplicationVersion: + def details( + self, application_id: str, application_version: VersionTuple | str | None = None, nocache: bool = False + ) -> ApplicationVersion: """Retrieves details for a specific application version. Retries on network and server errors. @@ -103,6 +107,8 @@ def details(self, application_id: str, application_version: VersionTuple | str | application_id (str): The ID of the application. application_version (VersionTuple | str | None): The version of the application. If None, the latest version will be retrieved. + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. Returns: ApplicationVersion: The version details. @@ -146,20 +152,22 @@ def details_with_retry(app_id: str, app_version: str) -> ApplicationVersion: ) ) - return details_with_retry(application_id, application_version) + return details_with_retry(application_id, application_version, nocache=nocache) # type: ignore[call-arg] # TODO(Helmut): Refactor given new API capabilities - def list_sorted(self, application: Application | str) -> builtins.list[VersionTuple]: + def list_sorted(self, application: Application | str, nocache: bool = False) -> builtins.list[VersionTuple]: """Get application versions sorted by semver, descending. Args: application (Application | str): The application to find versions for, either object or id + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. Returns: list[VersionTuple]: List of version objects sorted by semantic versioning (latest first), or empty list if no versions are found """ - versions = builtins.list(self.list(application=application)) + versions = builtins.list(self.list(application=application, nocache=nocache)) # If no versions available if not versions: @@ -184,16 +192,18 @@ def list_sorted(self, application: Application | str) -> builtins.list[VersionTu # If we couldn't parse any versions, return all versions as is return versions - def latest(self, application: Application | str) -> VersionTuple | None: + def latest(self, application: Application | str, nocache: bool = False) -> VersionTuple | None: """Get latest version. Args: application (Application | str): The application to find versions for, either object or id + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. Returns: VersionTuple | None: The latest version, or None if no versions found. """ - sorted_versions = self.list_sorted(application=application) + sorted_versions = self.list_sorted(application=application, nocache=nocache) return sorted_versions[0] if sorted_versions else None @@ -212,13 +222,15 @@ def __init__(self, api: PublicApi) -> None: self._api = api self.versions: Versions = Versions(self._api) - def details(self, application_id: str) -> Application: + def details(self, application_id: str, nocache: bool = False) -> Application: """Find application by id. Retries on network and server errors. Args: application_id (str): The ID of the application. + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. Returns: Application: The application object @@ -246,15 +258,17 @@ def details_with_retry(application_id: str) -> Application: ) ) - return details_with_retry(application_id) + return details_with_retry(application_id, nocache=nocache) # type: ignore[call-arg] - def list(self) -> t.Iterator[ApplicationSummary]: + def list(self, nocache: bool = False) -> t.Iterator[ApplicationSummary]: """Find all available applications. Retries on network and server errors for each page. Returns: Iterator[ApplicationSummary]: An iterator over the available applications. + notcache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. Raises: aignx.codegen.exceptions.ApiException: If the API request fails. @@ -280,4 +294,9 @@ def list_with_retry(**kwargs: object) -> list[ApplicationSummary]: ) ) - return paginate(list_with_retry) + return paginate( + lambda **kwargs: list_with_retry( + nocache=nocache, + **kwargs, + ) + ) diff --git a/src/aignostics/platform/resources/runs.py b/src/aignostics/platform/resources/runs.py index 882e75cff..865155681 100644 --- a/src/aignostics/platform/resources/runs.py +++ b/src/aignostics/platform/resources/runs.py @@ -108,11 +108,15 @@ def for_run_id(cls, run_id: str, cache_token: bool = True) -> "Run": return cls(Client.get_api_client(cache_token=cache_token), run_id) - def details(self) -> RunData: + def details(self, nocache: bool = False) -> RunData: """Retrieves the current status of the application run. Retries on network and server errors. + Args: + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. + Returns: RunData: The run data. @@ -136,7 +140,7 @@ def details_with_retry(run_id: str) -> RunData: ) ) - return details_with_retry(self.run_id) + return details_with_retry(self.run_id, nocache=nocache) # type: ignore[call-arg] # TODO(Andreas): Low Prio / existed prior to API migration: Please check if this still fails with # Internal Server Error if run was already canceled, should rather fail with 400 bad request in that state. @@ -168,11 +172,15 @@ def delete(self) -> None: _headers={"User-Agent": user_agent()}, ) - def results(self) -> t.Iterator[ItemResultData]: + def results(self, nocache: bool = False) -> t.Iterator[ItemResultData]: """Retrieves the results of all items in the run. Retries on network and server errors. + Args: + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. + Returns: list[ItemResultData]: A list of item results. @@ -199,7 +207,7 @@ def results_with_retry(run_id: str, **kwargs: object) -> list[ItemResultData]: ) ) - return paginate(lambda **kwargs: results_with_retry(self.run_id, **kwargs)) + return paginate(lambda **kwargs: results_with_retry(self.run_id, nocache=nocache, **kwargs)) def download_to_folder( self, download_base: Path | str, checksum_attribute_key: str = "checksum_base64_crc32c" @@ -234,8 +242,7 @@ def download_to_folder( print(self) # check if last results have been downloaded yet and report on errors - operation_cache_clear() # clear cache to get fresh results - for item in self.results(): + for item in self.results(nocache=True): # no cache to get fresh results match item.output: case ItemOutput.FULL: self.ensure_artifacts_downloaded(application_run_dir, item, checksum_attribute_key) @@ -388,7 +395,9 @@ def submit( ) return Run(self._api, str(res.run_id)) - def list(self, application_id: str | None = None, application_version: str | None = None) -> Iterator[Run]: + def list( + self, application_id: str | None = None, application_version: str | None = None, nocache: bool = False + ) -> Iterator[Run]: """Find application runs, optionally filtered by application id and/or version. Retries on network and server errors. @@ -396,6 +405,8 @@ def list(self, application_id: str | None = None, application_version: str | Non Args: application_id (str | None): Optional application ID to filter by. application_version (str | None): Optional application version to filter by. + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. Returns: Iterator[Run]: An iterator yielding application runs. @@ -424,6 +435,7 @@ def list_with_retry(**kwargs: object) -> list[RunData]: lambda **kwargs: list_with_retry( application_id=application_id, application_version=application_version, + nocache=nocache, **kwargs, ) ) @@ -437,6 +449,7 @@ def list_data( # noqa: PLR0913, PLR0917 custom_metadata: str | None = None, sort: str | None = None, page_size: int = LIST_APPLICATION_RUNS_MAX_PAGE_SIZE, + nocache: bool = False, ) -> t.Iterator[RunData]: """Fetch application runs, optionally filtered by application version. @@ -449,6 +462,8 @@ def list_data( # noqa: PLR0913, PLR0917 custom_metadata (str | None): Optional metadata filter in JSONPath format. sort (str | None): Optional field to sort by. Prefix with '-' for descending order. page_size (int): Number of items per page, defaults to max + nocache (bool): If True, skip reading from cache and fetch fresh data from the API. + The fresh result will still be cached for subsequent calls. Defaults to False. Returns: Iterator[RunData]: Iterator yielding application run data. @@ -486,6 +501,7 @@ def list_data_with_retry(**kwargs: object) -> list[RunData]: external_id=external_id, custom_metadata=custom_metadata, sort=[sort] if sort else None, + nocache=nocache, **kwargs, ), page_size=page_size, diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py index f725145bc..f369d0faa 100644 --- a/tests/aignostics/application/cli_test.py +++ b/tests/aignostics/application/cli_test.py @@ -15,6 +15,10 @@ from tests.constants_test import ( HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, + HETA_EXPECTED_RESULT_FILES, + HETA_SINGLE_SPOT_GS_FILENAME, + HETA_SINGLE_SPOT_GS_FILESIZE, + HETA_SINGLE_SPOT_GS_URL, TEST_APPLICATION_ID, TEST_APPLICATION_VERSION, ) @@ -453,6 +457,7 @@ def test_cli_run_result_delete_fails_on_no_arg(runner: CliRunner) -> None: assert result.exit_code == 2 +# TODO (Helmut): Schedule this run @pytest.mark.e2e @pytest.mark.very_long_running @pytest.mark.timeout(timeout=HETA_APPLICATION_DEADLINE_SECONDS + 60 * 30) @@ -465,17 +470,25 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: "dataset", "aignostics", "download", - "gs://aignx-storage-service-dev/sample_data_formatted/9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff", + HETA_SINGLE_SPOT_GS_URL, str(tmp_path), ], ) + + # Explore what was download print_directory_structure(tmp_path, "download") - assert result.exit_code == 0 + + # Validate what was downloaded assert "Successfully downloaded" in normalize_output(result.stdout) - assert "9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" in normalize_output(result.stdout) - expected_file = tmp_path / "9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" + assert HETA_SINGLE_SPOT_GS_FILENAME in normalize_output(result.stdout) + expected_file = tmp_path / HETA_SINGLE_SPOT_GS_FILENAME assert expected_file.exists(), f"Expected file {expected_file} not found" - assert expected_file.stat().st_size == 14681750 + assert expected_file.stat().st_size == HETA_SINGLE_SPOT_GS_FILESIZE, ( + f"Expected file size {HETA_SINGLE_SPOT_GS_FILESIZE}, but got {expected_file.stat().st_size}" + ) + + # Validate the download command exited successfully + assert result.exit_code == 0 # Step 2: Execute the run, i.e. prepare, amend, upload, submit and download the results result = runner.invoke( @@ -496,35 +509,31 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: "--validate-only", ], ) + + # Explore what was download print_directory_structure(tmp_path, "execute") - assert result.exit_code == 0 - item_out_dir = tmp_path / "9375e3ed-28d2-4cf3-9fb9-8df9d11a6627" - assert item_out_dir.is_dir(), f"Expected directory {item_out_dir} not found" - files_in_dir = list(item_out_dir.glob("*")) + + # Validate no input dir, given we used an external id pointing to a local file + input_file = tmp_path / "input" + assert not input_file.is_dir(), f"Expected input directory {input_file} not found" + + # Validate results generated and downloaded + results_dir = tmp_path / HETA_SINGLE_SPOT_GS_FILENAME.replace(".tiff", "") + assert results_dir.is_dir(), f"Expected directory {results_dir} not found" + files_in_dir = list(results_dir.glob("*")) assert len(files_in_dir) == 9, ( - f"Expected 9 files in {item_out_dir}, but found {len(files_in_dir)}: {[f.name for f in files_in_dir]}" + f"Expected 9 files in {results_dir}, but found {len(files_in_dir)}: {[f.name for f in files_in_dir]}" ) - expected_files = [ - ("tissue_segmentation_csv_class_information.csv", 342, 10), - ("cell_classification_geojson_polygons.json", 16054058, 10), - ("readout_generation_cell_readouts.csv", 2228907, 10), - ("tissue_qc_csv_class_information.csv", 232, 10), - ("tissue_segmentation_geojson_polygons.json", 270931, 10), - ("tissue_qc_geojson_polygons.json", 180522, 10), - ("tissue_qc_segmentation_map_image.tiff", 464908, 10), - ("readout_generation_slide_readouts.csv", 295268, 10), - ("tissue_segmentation_segmentation_map_image.tiff", 581258, 10), - ] - print(f"Found files in {item_out_dir}:") - for filename, expected_size, tolerance_percent in expected_files: - file_path = item_out_dir / filename + print(f"Found files in {results_dir}:") + for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + file_path = results_dir / filename if file_path.exists(): actual_size = file_path.stat().st_size print(f" {filename}: {actual_size} bytes (expected: {expected_size} ±{tolerance_percent}%)") else: print(f" {filename}: NOT FOUND") - for filename, expected_size, tolerance_percent in expected_files: - file_path = item_out_dir / filename + for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + file_path = results_dir / filename assert file_path.exists(), f"Expected file {filename} not found" actual_size = file_path.stat().st_size min_size = expected_size * (100 - tolerance_percent) // 100 @@ -533,3 +542,6 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: f"File size for {filename} ({actual_size} bytes) is outside allowed range " f"({min_size} to {max_size} bytes, ±{tolerance_percent}% of {expected_size})" ) + + # Validate the execute command exited successfully + assert result.exit_code == 0 diff --git a/tests/aignostics/application/gui_test.py b/tests/aignostics/application/gui_test.py index 0b9f9a816..28ce58b7f 100644 --- a/tests/aignostics/application/gui_test.py +++ b/tests/aignostics/application/gui_test.py @@ -16,7 +16,14 @@ from aignostics.cli import cli from aignostics.utils import get_logger from tests.conftest import assert_notified, normalize_output, print_directory_structure -from tests.constants_test import HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, HETA_SINGLE_SPOT_GS_URL +from tests.constants_test import ( + HETA_APPLICATION_ID, + HETA_APPLICATION_VERSION, + HETA_EXPECTED_RESULT_FILES, + HETA_SINGLE_SPOT_GS_FILENAME, + HETA_SINGLE_SPOT_GS_FILESIZE, + HETA_SINGLE_SPOT_GS_URL, +) if TYPE_CHECKING: from nicegui import ui @@ -354,19 +361,49 @@ async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, s # Check: Download completed await assert_notified(user, "Download completed.", 60 * 4) print_directory_structure(tmp_path, "execute") - run_out_dir = tmp_path / run.run_id - assert run_out_dir.is_dir(), f"Expected run directory {run_out_dir} not found" - # Find any subdirectory in the run_out_dir - subdirs = [d for d in run_out_dir.iterdir() if d.is_dir()] - assert len(subdirs) > 0, f"Expected at least one subdirectory in {run_out_dir}, but found none" - - # Take the first subdirectory found (item_out_dir) - item_out_dir = subdirs[0] - print(f"Found subdirectory: {item_out_dir.name}") - - # Check for files in the item directory - files_in_item_dir = list(item_out_dir.glob("*")) - assert len(files_in_item_dir) == 9, ( - f"Expected 9 files in {item_out_dir}, but found {len(files_in_item_dir)}: " - f"{[f.name for f in files_in_item_dir]}" + + # Check for directory layout as expected + run_dir = tmp_path / run.run_id + assert run_dir.is_dir(), f"Expected run directory {run_dir} not found" + + subdirs = [d for d in run_dir.iterdir() if d.is_dir()] + assert len(subdirs) == 2, f"Expected two subdirectories in {run_dir}, but found {len(subdirs)}" + + input_dir = tmp_path / "input" + assert input_dir.is_dir(), f"Expected input directory {input_dir} not found" + + results_dir = tmp_path / run.run_id / HETA_SINGLE_SPOT_GS_FILENAME.replace(".tiff", "") + assert results_dir.is_dir(), f"Expected run results directory {results_dir} not found" + + # Check for input file having been downloaded + input_file = tmp_path / run.run_id / "input" / HETA_SINGLE_SPOT_GS_FILENAME + assert input_file.is_file(), f"Expected input file {input_file} not found" + assert input_file.stat().st_size == HETA_SINGLE_SPOT_GS_FILESIZE, ( + f"Expected input file size {HETA_SINGLE_SPOT_GS_FILESIZE}, but got {input_file.stat().st_size}" + ) + + # Check for files in the results directory + files_in_results_dir = list(results_dir.glob("*")) + assert len(files_in_results_dir) == 9, ( + f"Expected 9 files in {results_dir}, but found {len(files_in_results_dir)}: " + f"{[f.name for f in files_in_results_dir]}" ) + + print(f"Found files in {results_dir}:") + for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + file_path = results_dir / filename + if file_path.exists(): + actual_size = file_path.stat().st_size + print(f" {filename}: {actual_size} bytes (expected: {expected_size} ±{tolerance_percent}%)") + else: + print(f" {filename}: NOT FOUND") + for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + file_path = results_dir / filename + assert file_path.exists(), f"Expected file {filename} not found" + actual_size = file_path.stat().st_size + min_size = expected_size * (100 - tolerance_percent) // 100 + max_size = expected_size * (100 + tolerance_percent) // 100 + assert min_size <= actual_size <= max_size, ( + f"File size for {filename} ({actual_size} bytes) is outside allowed range " + f"({min_size} to {max_size} bytes, ±{tolerance_percent}% of {expected_size})" + ) diff --git a/tests/aignostics/platform/conftest.py b/tests/aignostics/platform/conftest.py index fac64bc2e..4e47b5bdb 100644 --- a/tests/aignostics/platform/conftest.py +++ b/tests/aignostics/platform/conftest.py @@ -23,6 +23,21 @@ def mock_settings() -> MagicMock: settings.me_retry_wait_max = 5.0 settings.me_timeout = 10.0 settings.me_cache_ttl = 60 # 60 seconds for testing + settings.application_retry_attempts = 3 + settings.application_retry_wait_min = 0.1 + settings.application_retry_wait_max = 5.0 + settings.application_timeout = 10.0 + settings.application_cache_ttl = 300 # 5 minutes + settings.application_version_retry_attempts = 3 + settings.application_version_retry_wait_min = 0.1 + settings.application_version_retry_wait_max = 5.0 + settings.application_version_timeout = 10.0 + settings.application_version_cache_ttl = 300 # 5 minutes + settings.run_retry_attempts = 3 + settings.run_retry_wait_min = 0.1 + settings.run_retry_wait_max = 5.0 + settings.run_timeout = 10.0 + settings.run_cache_ttl = 15 # 15 seconds settings.api_root = "https://test.api.com" mock_settings.return_value = settings yield mock_settings diff --git a/tests/aignostics/platform/e2e_test.py b/tests/aignostics/platform/e2e_test.py index ad58c5c08..9cb86067f 100644 --- a/tests/aignostics/platform/e2e_test.py +++ b/tests/aignostics/platform/e2e_test.py @@ -1,7 +1,7 @@ -"""Scheduled integration tests for the Aignostics client. +"""Scheduled end-to-end (e2e) tests for the Aignostics client. -This module contains integration tests that run real application workflows -against the Aignostics platform. These tests verify end-to-end functionality +This module contains e2e tests that run real application workflows +against the Aignostics platform. These tests verify e2e functionality including creating runs, downloading results, and validating outputs. """ diff --git a/tests/aignostics/platform/nocache_test.py b/tests/aignostics/platform/nocache_test.py new file mode 100644 index 000000000..364b8260e --- /dev/null +++ b/tests/aignostics/platform/nocache_test.py @@ -0,0 +1,687 @@ +"""Tests for nocache parameter functionality across the platform module. + +This module tests that: +1. nocache=False uses cached values (default behavior) +2. nocache=True skips reading from cache but still writes to cache +3. All platform methods that support caching correctly handle nocache +4. The decorator properly intercepts and handles the nocache parameter +""" + +import time +from unittest.mock import MagicMock + +import pytest + +from aignostics.platform._client import Client +from aignostics.platform._operation_cache import _operation_cache, cached_operation, operation_cache_clear + + +class TestNocacheDecoratorBehavior: + """Test the nocache parameter handling in the cached_operation decorator.""" + + @pytest.mark.unit + @staticmethod + def test_decorator_without_nocache_uses_cache() -> None: + """Test that decorated function uses cache by default (nocache=False).""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # First call - should execute function + result1 = test_func() + assert result1 == 1 + assert call_count == 1 + + # Second call - should use cache + result2 = test_func() + assert result2 == 1 # Same as first call, from cache + assert call_count == 1 # Function not called again + + @pytest.mark.unit + @staticmethod + def test_decorator_with_nocache_false_uses_cache() -> None: + """Test that nocache=False explicitly uses cache.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # First call with nocache=False + result1 = test_func() # type: ignore[call-arg] + assert result1 == 1 + assert call_count == 1 + + # Second call with nocache=False - should use cache + result2 = test_func() # type: ignore[call-arg] + assert result2 == 1 + assert call_count == 1 + + @pytest.mark.unit + @staticmethod + def test_decorator_with_nocache_true_skips_reading_cache() -> None: + """Test that nocache=True skips reading from cache.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # First call - populates cache + result1 = test_func() # type: ignore[call-arg] + assert result1 == 1 + assert call_count == 1 + + # Second call with nocache=True - skips cache, executes function + result2 = test_func(nocache=True) # type: ignore[call-arg] + assert result2 == 2 # New value, not from cache + assert call_count == 2 # Function called again + + @pytest.mark.unit + @staticmethod + def test_decorator_with_nocache_true_still_writes_to_cache() -> None: + """Test that nocache=True still writes the result to cache.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # First call - populates cache + result1 = test_func() # type: ignore[call-arg] + assert result1 == 1 + assert call_count == 1 + + # Second call with nocache=True - skips cache read, writes new value + result2 = test_func(nocache=True) # type: ignore[call-arg] + assert result2 == 2 + assert call_count == 2 + + # Third call without nocache - should use the value cached by second call + result3 = test_func() # type: ignore[call-arg] + assert result3 == 2 # Uses value from second call + assert call_count == 2 # Function not called again + + @pytest.mark.unit + @staticmethod + def test_decorator_nocache_parameter_not_passed_to_function() -> None: + """Test that nocache parameter is intercepted and not passed to the decorated function.""" + received_kwargs = {} + + @cached_operation(ttl=60, use_token=False) + def test_func(**kwargs: bool) -> dict: + nonlocal received_kwargs + received_kwargs = kwargs + return {"called": True} + + # Call with nocache=True + test_func(nocache=True) # type: ignore[call-arg] + + # The decorated function should not receive nocache in kwargs + assert "nocache" not in received_kwargs + + @pytest.mark.unit + @staticmethod + def test_decorator_with_nocache_and_other_kwargs() -> None: + """Test that nocache works alongside other keyword arguments.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func(param1: str = "default", param2: int = 0) -> tuple: + nonlocal call_count + call_count += 1 + return (call_count, param1, param2) + + # First call with params + result1 = test_func(param1="value1", param2=123) # type: ignore[call-arg] + assert result1 == (1, "value1", 123) + assert call_count == 1 + + # Second call with same params - should use cache + result2 = test_func(param1="value1", param2=123) # type: ignore[call-arg] + assert result2 == (1, "value1", 123) + assert call_count == 1 + + # Third call with nocache=True and same params - should skip cache + result3 = test_func(param1="value1", param2=123, nocache=True) # type: ignore[call-arg] + assert result3 == (2, "value1", 123) + assert call_count == 2 + + @pytest.mark.unit + @staticmethod + def test_decorator_nocache_with_different_cache_keys() -> None: + """Test that nocache respects different cache keys (different args).""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func(key: str) -> tuple: + nonlocal call_count + call_count += 1 + return (call_count, key) + + # Call with key="A" + result1 = test_func("A") # type: ignore[call-arg] + assert result1 == (1, "A") + assert call_count == 1 + + # Call with key="B" + result2 = test_func("B") # type: ignore[call-arg] + assert result2 == (2, "B") + assert call_count == 2 + + # Call with key="A", nocache=True - should skip cache for key="A" + result3 = test_func("A", nocache=True) # type: ignore[call-arg] + assert result3 == (3, "A") + assert call_count == 3 + + # Call with key="B" again - should use cache for key="B" + result4 = test_func("B") # type: ignore[call-arg] + assert result4 == (2, "B") # Still has cached value + assert call_count == 3 + + +class TestClientMeNocache: + """Test nocache parameter for Client.me() method.""" + + @pytest.mark.unit + @staticmethod + def test_me_default_uses_cache( + mock_settings: MagicMock, client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Test that me() uses cache by default.""" + mock_me_response = {"user_id": "test-user", "org_id": "test-org"} + mock_api_client.get_me_v1_me_get.return_value = mock_me_response + + # First call + result1 = client_with_mock_api.me() + assert result1 == mock_me_response + assert mock_api_client.get_me_v1_me_get.call_count == 1 + + # Second call - should use cache + result2 = client_with_mock_api.me() + assert result2 == mock_me_response + assert mock_api_client.get_me_v1_me_get.call_count == 1 + + @pytest.mark.unit + @staticmethod + def test_me_nocache_false_uses_cache( + mock_settings: MagicMock, client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Test that me(nocache=False) uses cache.""" + mock_me_response = {"user_id": "test-user", "org_id": "test-org"} + mock_api_client.get_me_v1_me_get.return_value = mock_me_response + + # First call + result1 = client_with_mock_api.me(nocache=False) + assert result1 == mock_me_response + assert mock_api_client.get_me_v1_me_get.call_count == 1 + + # Second call with nocache=False - should use cache + result2 = client_with_mock_api.me(nocache=False) + assert result2 == mock_me_response + assert mock_api_client.get_me_v1_me_get.call_count == 1 + + @pytest.mark.unit + @staticmethod + def test_me_nocache_true_fetches_fresh_data( + mock_settings: MagicMock, client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Test that me(nocache=True) fetches fresh data from API.""" + mock_me_response_1 = {"user_id": "user-1", "org_id": "org-1"} + mock_me_response_2 = {"user_id": "user-2", "org_id": "org-2"} + + # First call - populates cache + mock_api_client.get_me_v1_me_get.return_value = mock_me_response_1 + result1 = client_with_mock_api.me() + assert result1 == mock_me_response_1 + assert mock_api_client.get_me_v1_me_get.call_count == 1 + + # Change API response + mock_api_client.get_me_v1_me_get.return_value = mock_me_response_2 + + # Second call with nocache=True - should fetch fresh data + result2 = client_with_mock_api.me(nocache=True) + assert result2 == mock_me_response_2 + assert mock_api_client.get_me_v1_me_get.call_count == 2 + + @pytest.mark.unit + @staticmethod + def test_me_nocache_true_updates_cache( + mock_settings: MagicMock, client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Test that me(nocache=True) updates the cache with fresh data.""" + mock_me_response_1 = {"user_id": "user-1", "org_id": "org-1"} + mock_me_response_2 = {"user_id": "user-2", "org_id": "org-2"} + + # First call - populates cache + mock_api_client.get_me_v1_me_get.return_value = mock_me_response_1 + result1 = client_with_mock_api.me() + assert result1 == mock_me_response_1 + assert mock_api_client.get_me_v1_me_get.call_count == 1 + + # Change API response + mock_api_client.get_me_v1_me_get.return_value = mock_me_response_2 + + # Second call with nocache=True - fetches and caches new data + result2 = client_with_mock_api.me(nocache=True) + assert result2 == mock_me_response_2 + assert mock_api_client.get_me_v1_me_get.call_count == 2 + + # Third call without nocache - should use updated cache + result3 = client_with_mock_api.me() + assert result3 == mock_me_response_2 # Uses new cached value + assert mock_api_client.get_me_v1_me_get.call_count == 2 # No additional API call + + +class TestClientApplicationNocache: + """Test nocache parameter for Client.application() method.""" + + @pytest.mark.unit + @staticmethod + def test_application_default_uses_cache( + mock_settings: MagicMock, client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Test that application() uses cache by default.""" + mock_app_response = {"application_id": "test-app", "name": "Test App"} + mock_api_client.read_application_by_id_v1_applications_application_id_get.return_value = mock_app_response + + # First call + result1 = client_with_mock_api.application("test-app") + assert result1 == mock_app_response + assert mock_api_client.read_application_by_id_v1_applications_application_id_get.call_count == 1 + + # Second call - should use cache + result2 = client_with_mock_api.application("test-app") + assert result2 == mock_app_response + assert mock_api_client.read_application_by_id_v1_applications_application_id_get.call_count == 1 + + @pytest.mark.unit + @staticmethod + def test_application_nocache_true_fetches_fresh_data( + mock_settings: MagicMock, client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Test that application(nocache=True) fetches fresh data.""" + mock_app_response_1 = {"application_id": "test-app", "name": "App v1"} + mock_app_response_2 = {"application_id": "test-app", "name": "App v2"} + + # First call + mock_api_client.read_application_by_id_v1_applications_application_id_get.return_value = mock_app_response_1 + result1 = client_with_mock_api.application("test-app") + assert result1 == mock_app_response_1 + assert mock_api_client.read_application_by_id_v1_applications_application_id_get.call_count == 1 + + # Change response + mock_api_client.read_application_by_id_v1_applications_application_id_get.return_value = mock_app_response_2 + + # Second call with nocache=True + result2 = client_with_mock_api.application("test-app", nocache=True) + assert result2 == mock_app_response_2 + assert mock_api_client.read_application_by_id_v1_applications_application_id_get.call_count == 2 + + @pytest.mark.unit + @staticmethod + def test_application_nocache_with_different_app_ids( + mock_settings: MagicMock, client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Test that nocache works correctly with different application IDs.""" + mock_app_response_a = {"application_id": "app-a", "name": "App A"} + mock_app_response_b = {"application_id": "app-b", "name": "App B"} + + def side_effect(*args, **kwargs): + app_id = kwargs.get("application_id") + if app_id == "app-a": + return mock_app_response_a + return mock_app_response_b + + mock_api_client.read_application_by_id_v1_applications_application_id_get.side_effect = side_effect + + # Call for app-a + result1 = client_with_mock_api.application("app-a") + assert result1 == mock_app_response_a + + # Call for app-b + result2 = client_with_mock_api.application("app-b") + assert result2 == mock_app_response_b + + # Both should be cached + assert mock_api_client.read_application_by_id_v1_applications_application_id_get.call_count == 2 + + # Call app-a with nocache=True + result3 = client_with_mock_api.application("app-a", nocache=True) + assert result3 == mock_app_response_a + assert mock_api_client.read_application_by_id_v1_applications_application_id_get.call_count == 3 + + # Call app-b without nocache - should use cache + result4 = client_with_mock_api.application("app-b") + assert result4 == mock_app_response_b + assert mock_api_client.read_application_by_id_v1_applications_application_id_get.call_count == 3 + + +class TestClientApplicationVersionNocache: + """Test nocache parameter for Client.application_version() method.""" + + @pytest.mark.unit + @staticmethod + def test_application_version_default_uses_cache( + mock_settings: MagicMock, client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Test that application_version() uses cache by default.""" + mock_version_response = {"application_id": "test-app", "version": "1.0.0"} + mock_api_client.application_version_details_v1_applications_application_id_versions_version_get.return_value = ( + mock_version_response + ) + + # First call + result1 = client_with_mock_api.application_version("test-app", "1.0.0") + assert result1 == mock_version_response + assert ( + mock_api_client.application_version_details_v1_applications_application_id_versions_version_get.call_count + == 1 + ) + + # Second call - should use cache + result2 = client_with_mock_api.application_version("test-app", "1.0.0") + assert result2 == mock_version_response + assert ( + mock_api_client.application_version_details_v1_applications_application_id_versions_version_get.call_count + == 1 + ) + + @pytest.mark.unit + @staticmethod + def test_application_version_nocache_true_fetches_fresh_data( + mock_settings: MagicMock, client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Test that application_version(nocache=True) fetches fresh data.""" + mock_version_response_1 = {"application_id": "test-app", "version": "1.0.0", "updated": "2024-01-01"} + mock_version_response_2 = {"application_id": "test-app", "version": "1.0.0", "updated": "2024-01-02"} + + # First call + mock_api_client.application_version_details_v1_applications_application_id_versions_version_get.return_value = ( + mock_version_response_1 + ) + result1 = client_with_mock_api.application_version("test-app", "1.0.0") + assert result1 == mock_version_response_1 + assert ( + mock_api_client.application_version_details_v1_applications_application_id_versions_version_get.call_count + == 1 + ) + + # Change response + mock_api_client.application_version_details_v1_applications_application_id_versions_version_get.return_value = ( + mock_version_response_2 + ) + + # Second call with nocache=True + result2 = client_with_mock_api.application_version("test-app", "1.0.0", nocache=True) + assert result2 == mock_version_response_2 + assert ( + mock_api_client.application_version_details_v1_applications_application_id_versions_version_get.call_count + == 2 + ) + + +class TestRunDetailsNocache: + """Test nocache parameter for Run.details() method - simplified tests.""" + + @pytest.mark.unit + @staticmethod + def test_run_details_supports_nocache_parameter() -> None: + """Test that Run.details() method signature supports nocache parameter.""" + from inspect import signature + + from aignostics.platform.resources.runs import Run + + # Verify the method has nocache parameter + sig = signature(Run.details) + assert "nocache" in sig.parameters + param = sig.parameters["nocache"] + assert param.default is False + assert param.annotation is bool + + +class TestRunsListNocache: + """Test nocache parameter for Runs.list() method - simplified tests.""" + + @pytest.mark.unit + @staticmethod + def test_runs_list_supports_nocache_parameter() -> None: + """Test that Runs.list() method signature supports nocache parameter.""" + from inspect import signature + + from aignostics.platform.resources.runs import Runs + + # Verify the method has nocache parameter + sig = signature(Runs.list) + assert "nocache" in sig.parameters + param = sig.parameters["nocache"] + assert param.default is False + assert param.annotation is bool + + +class TestApplicationsResourcesNocache: + """Test nocache parameter for Applications and Versions resources - simplified tests.""" + + @pytest.mark.unit + @staticmethod + def test_versions_list_supports_nocache_parameter() -> None: + """Test that Versions.list() method signature supports nocache parameter.""" + from inspect import signature + + from aignostics.platform.resources.applications import Versions + + # Verify the method has nocache parameter + sig = signature(Versions.list) + assert "nocache" in sig.parameters + param = sig.parameters["nocache"] + assert param.default is False + assert param.annotation is bool + + @pytest.mark.unit + @staticmethod + def test_applications_details_supports_nocache_parameter() -> None: + """Test that Applications.details() method signature supports nocache parameter.""" + from inspect import signature + + from aignostics.platform.resources.applications import Applications + + # Verify the method has nocache parameter + sig = signature(Applications.details) + assert "nocache" in sig.parameters + param = sig.parameters["nocache"] + assert param.default is False + assert param.annotation is bool + + +class TestNocacheEdgeCases: + """Test edge cases and special scenarios for nocache functionality.""" + + @pytest.mark.unit + @staticmethod + def test_nocache_with_expired_cache_entry() -> None: + """Test nocache behavior when cache entry has expired.""" + call_count = 0 + + @cached_operation(ttl=1, use_token=False) # 1 second TTL + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # First call - populates cache + result1 = test_func() # type: ignore[call-arg] + assert result1 == 1 + assert call_count == 1 + + # Wait for cache to expire + time.sleep(1.1) + + # Second call with nocache=True on expired entry + result2 = test_func(nocache=True) # type: ignore[call-arg] + assert result2 == 2 + assert call_count == 2 + + @pytest.mark.unit + @staticmethod + def test_nocache_clears_expired_entry_before_writing_new() -> None: + """Test that nocache properly handles expired entries.""" + call_count = 0 + + @cached_operation(ttl=1, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # First call + result1 = test_func() # type: ignore[call-arg] + assert result1 == 1 + + # Wait for expiry + time.sleep(1.1) + + # Call with nocache=True - should write new value + result2 = test_func(nocache=True) # type: ignore[call-arg] + assert result2 == 2 + + # Subsequent call should use new cached value + result3 = test_func() # type: ignore[call-arg] + assert result3 == 2 + assert call_count == 2 + + @pytest.mark.unit + @staticmethod + def test_multiple_consecutive_nocache_calls() -> None: + """Test multiple consecutive calls with nocache=True.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # Multiple calls with nocache=True + result1 = test_func(nocache=True) # type: ignore[call-arg] + assert result1 == 1 + + result2 = test_func(nocache=True) # type: ignore[call-arg] + assert result2 == 2 + + result3 = test_func(nocache=True) # type: ignore[call-arg] + assert result3 == 3 + + assert call_count == 3 + + # Last call without nocache should use cached value from third call + result4 = test_func() # type: ignore[call-arg] + assert result4 == 3 + assert call_count == 3 + + @pytest.mark.unit + @staticmethod + def test_nocache_interleaved_with_normal_calls() -> None: + """Test interleaving nocache=True with normal cached calls.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # Normal call - populates cache + result1 = test_func() # type: ignore[call-arg] + assert result1 == 1 + assert call_count == 1 + + # Normal call - uses cache + result2 = test_func() # type: ignore[call-arg] + assert result2 == 1 + assert call_count == 1 + + # Nocache call - skips cache, updates it + result3 = test_func(nocache=True) # type: ignore[call-arg] + assert result3 == 2 + assert call_count == 2 + + # Normal call - uses updated cache + result4 = test_func() # type: ignore[call-arg] + assert result4 == 2 + assert call_count == 2 + + # Another nocache call + result5 = test_func(nocache=True) # type: ignore[call-arg] + assert result5 == 3 + assert call_count == 3 + + # Final normal call - uses latest cached value + result6 = test_func() # type: ignore[call-arg] + assert result6 == 3 + assert call_count == 3 + + +class TestNocacheWithClearCache: + """Test interaction between nocache and cache clearing.""" + + @pytest.mark.unit + @staticmethod + def test_nocache_after_cache_clear() -> None: + """Test that nocache works correctly after cache has been cleared.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # Populate cache + result1 = test_func() # type: ignore[call-arg] + assert result1 == 1 + + # Clear cache + operation_cache_clear() + assert len(_operation_cache) == 0 + + # Call with nocache=True - should work normally + result2 = test_func(nocache=True) # type: ignore[call-arg] + assert result2 == 2 + + # Verify cache was populated + assert len(_operation_cache) == 1 + + @pytest.mark.unit + @staticmethod + def test_cache_clear_removes_nocache_populated_entries() -> None: + """Test that cache clear removes entries populated with nocache=True.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # Populate cache with nocache=True + result1 = test_func(nocache=True) # type: ignore[call-arg] + assert result1 == 1 + assert len(_operation_cache) == 1 + + # Clear cache + operation_cache_clear() + assert len(_operation_cache) == 0 + + # Subsequent call should fetch fresh data + result2 = test_func() # type: ignore[call-arg] + assert result2 == 2 diff --git a/tests/aignostics/qupath/gui_test.py b/tests/aignostics/qupath/gui_test.py index d9ca5b984..8502299eb 100644 --- a/tests/aignostics/qupath/gui_test.py +++ b/tests/aignostics/qupath/gui_test.py @@ -19,7 +19,14 @@ from aignostics.qupath import QUPATH_LAUNCH_MAX_WAIT_TIME, QUPATH_VERSION from aignostics.utils import __project_name__ from tests.conftest import assert_notified, normalize_output, print_directory_structure -from tests.constants_test import HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, HETA_SINGLE_SPOT_GS_URL +from tests.constants_test import ( + HETA_APPLICATION_ID, + HETA_APPLICATION_VERSION, + HETA_EXPECTED_RESULT_FILES, + HETA_SINGLE_SPOT_GS_FILENAME, + HETA_SINGLE_SPOT_GS_FILESIZE, + HETA_SINGLE_SPOT_GS_URL, +) if TYPE_CHECKING: from nicegui import ui @@ -137,11 +144,11 @@ async def test_gui_qupath_install_and_launch( @pytest.mark.long_running @pytest.mark.skipif( (platform.system() == "Linux" and platform.machine() in {"aarch64", "arm64"}) or platform.system() == "Windows", - reason="QuPath is not supported on ARM64 Linux; Windows support is not implemented yet", + reason="QuPath is not supported on ARM64 Linux; Windows support is not fully tested yet", ) @pytest.mark.timeout(timeout=60 * 15) @pytest.mark.sequential -async def test_gui_run_qupath_install_to_inspect( # noqa: PLR0914, PLR0915 +async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR0914, PLR0915 user: User, runner: CliRunner, tmp_path: Path, silent_logging: None, qupath_teardown: None ) -> None: """Test installing QuPath, downloading run results, creating QuPath project from it, and inspecting results.""" @@ -224,23 +231,56 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: PLR0914, PLR0915 # Step 6: Check download completes, QuPath project created, and QuPath launched await assert_notified(user, "Download and QuPath project creation completed.", 60 * 5) print_directory_structure(tmp_path, "execute") - run_out_dir = tmp_path / run.run_id - assert run_out_dir.is_dir(), f"Expected run directory {run_out_dir} not found" - # Find any subdirectory in the run_out_dir that is not qupath - subdirs = [d for d in run_out_dir.iterdir() if d.is_dir() and d.name != "qupath"] - assert len(subdirs) > 0, f"Expected at least one non-qupath subdirectory in {run_out_dir}, but found none" - - # Take the first subdirectory found (item_out_dir) - item_out_dir = subdirs[0] - print(f"Found subdirectory: {item_out_dir.name}") - - # Check for files in the item directory - files_in_item_dir = list(item_out_dir.glob("*")) - assert len(files_in_item_dir) == 9, ( - f"Expected 9 files in {item_out_dir}, but found {len(files_in_item_dir)}: " - f"{[f.name for f in files_in_item_dir]}" + + # Check for directory layout as expected + run_dir = tmp_path / run.run_id + assert run_dir.is_dir(), f"Expected run directory {run_dir} not found" + + subdirs = [d for d in run_dir.iterdir() if d.is_dir()] + assert len(subdirs) == 3, f"Expected three subdirectories in {run_dir}, but found {len(subdirs)}" + + input_dir = tmp_path / "input" + assert input_dir.is_dir(), f"Expected input directory {input_dir} not found" + + results_dir = tmp_path / run.run_id / HETA_SINGLE_SPOT_GS_FILENAME.replace(".tiff", "") + assert results_dir.is_dir(), f"Expected run results directory {results_dir} not found" + + qupath_dir = tmp_path / "qupath" + assert qupath_dir.is_dir(), f"Expected QuPath directory {qupath_dir} not found" + + # Check for input file having been downloaded + input_file = tmp_path / run.run_id / "input" / HETA_SINGLE_SPOT_GS_FILENAME + assert input_file.is_file(), f"Expected input file {input_file} not found" + assert input_file.stat().st_size == HETA_SINGLE_SPOT_GS_FILESIZE, ( + f"Expected input file size {HETA_SINGLE_SPOT_GS_FILESIZE}, but got {input_file.stat().st_size}" ) + # Check for files in the results directory + files_in_results_dir = list(results_dir.glob("*")) + assert len(files_in_results_dir) == 9, ( + f"Expected 9 files in {results_dir}, but found {len(files_in_results_dir)}: " + f"{[f.name for f in files_in_results_dir]}" + ) + + print(f"Found files in {results_dir}:") + for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + file_path = results_dir / filename + if file_path.exists(): + actual_size = file_path.stat().st_size + print(f" {filename}: {actual_size} bytes (expected: {expected_size} ±{tolerance_percent}%)") + else: + print(f" {filename}: NOT FOUND") + for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + file_path = results_dir / filename + assert file_path.exists(), f"Expected file {filename} not found" + actual_size = file_path.stat().st_size + min_size = expected_size * (100 - tolerance_percent) // 100 + max_size = expected_size * (100 + tolerance_percent) // 100 + assert min_size <= actual_size <= max_size, ( + f"File size for {filename} ({actual_size} bytes) is outside allowed range " + f"({min_size} to {max_size} bytes, ±{tolerance_percent}% of {expected_size})" + ) + # Check QuPath is running notification = await assert_notified(user, "QuPath opened successfully", 30) pid_match = re.search(r"process id '(\d+)'", notification) @@ -255,10 +295,9 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: PLR0914, PLR0915 pytest.fail(f"Failed to kill QuPath process: {e}") # Step 7: Inspect QuPath results - result = runner.invoke(cli, ["qupath", "inspect", str(run_out_dir / "qupath")]) + result = runner.invoke(cli, ["qupath", "inspect", str(run_dir / "qupath")]) output = normalize_output(result.output, strip_ansi=True) print(repr(output)) - assert result.exit_code == 0, f"QuPath inspect command failed with exit code {result.exit_code}" try: project_info = json.loads(output) @@ -268,10 +307,12 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: PLR0914, PLR0915 total = hierarchy.get("total", 0) if total > 0: annotations_total += total - # TODO(Helmut): More detailed checks on the annotations when improved above - assert annotations_total >= 0, "Expected at least 0 annotations in the QuPath results" + assert annotations_total >= 10000, "Expected at least 10000 annotations in the QuPath results" except json.JSONDecodeError as e: pytest.fail(f"Failed to parse QuPath inspect output as JSON: {e}\nOutput: {output!r}\n") + # Validate the inspect command exited successfully + assert result.exit_code == 0, f"QuPath inspect command failed with exit code {result.exit_code}" + if not was_installed: result = runner.invoke(cli, ["qupath", "uninstall"]) diff --git a/tests/constants_test.py b/tests/constants_test.py index 30dd3d280..63c38f557 100644 --- a/tests/constants_test.py +++ b/tests/constants_test.py @@ -9,6 +9,19 @@ HETA_SINGLE_SPOT_GS_URL = ( "gs://platform-api-application-test-data/heta/slides/8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" ) +HETA_SINGLE_SPOT_GS_FILENAME = "8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" +HETA_SINGLE_SPOT_GS_FILESIZE = 14681750 # in bytes +HETA_EXPECTED_RESULT_FILES = [ + ("tissue_segmentation_csv_class_information.csv", 342, 10), + ("cell_classification_geojson_polygons.json", 16054058, 10), + ("readout_generation_cell_readouts.csv", 2228907, 10), + ("tissue_qc_csv_class_information.csv", 232, 10), + ("tissue_segmentation_geojson_polygons.json", 270931, 10), + ("tissue_qc_geojson_polygons.json", 180522, 10), + ("tissue_qc_segmentation_map_image.tiff", 464908, 10), + ("readout_generation_slide_readouts.csv", 295268, 10), + ("tissue_segmentation_segmentation_map_image.tiff", 581258, 10), +] TEST_THREE_SPOTS_GS_URLS = [ "gs://aignx-storage-service-dev/sample_data_formatted/9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff", From 70f28805693151ee23c560038a36021679638033 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 09:05:34 +0100 Subject: [PATCH 03/16] testing --- tests/aignostics/application/cli_test.py | 24 ++++++++++++------------ tests/aignostics/application/gui_test.py | 20 ++++++++++---------- tests/aignostics/qupath/gui_test.py | 18 +++++++++--------- tests/constants_test.py | 23 ++++++++++++++++++++--- 4 files changed, 51 insertions(+), 34 deletions(-) diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py index f369d0faa..20904ecf5 100644 --- a/tests/aignostics/application/cli_test.py +++ b/tests/aignostics/application/cli_test.py @@ -13,12 +13,12 @@ from aignostics.utils import sanitize_path from tests.conftest import normalize_output, print_directory_structure from tests.constants_test import ( + HETA_ANOTHER_SPOT_EXPECTED_RESULT_FILES, + HETA_ANOTHER_SPOT_FILENAME, + HETA_ANOTHER_SPOT_FILESIZE, + HETA_ANOTHER_SPOT_GS_URL, HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, - HETA_EXPECTED_RESULT_FILES, - HETA_SINGLE_SPOT_GS_FILENAME, - HETA_SINGLE_SPOT_GS_FILESIZE, - HETA_SINGLE_SPOT_GS_URL, TEST_APPLICATION_ID, TEST_APPLICATION_VERSION, ) @@ -470,7 +470,7 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: "dataset", "aignostics", "download", - HETA_SINGLE_SPOT_GS_URL, + HETA_ANOTHER_SPOT_GS_URL, str(tmp_path), ], ) @@ -480,11 +480,11 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: # Validate what was downloaded assert "Successfully downloaded" in normalize_output(result.stdout) - assert HETA_SINGLE_SPOT_GS_FILENAME in normalize_output(result.stdout) - expected_file = tmp_path / HETA_SINGLE_SPOT_GS_FILENAME + assert HETA_ANOTHER_SPOT_FILENAME in normalize_output(result.stdout) + expected_file = tmp_path / HETA_ANOTHER_SPOT_FILENAME assert expected_file.exists(), f"Expected file {expected_file} not found" - assert expected_file.stat().st_size == HETA_SINGLE_SPOT_GS_FILESIZE, ( - f"Expected file size {HETA_SINGLE_SPOT_GS_FILESIZE}, but got {expected_file.stat().st_size}" + assert expected_file.stat().st_size == HETA_ANOTHER_SPOT_FILESIZE, ( + f"Expected file size {HETA_ANOTHER_SPOT_FILESIZE}, but got {expected_file.stat().st_size}" ) # Validate the download command exited successfully @@ -518,21 +518,21 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: assert not input_file.is_dir(), f"Expected input directory {input_file} not found" # Validate results generated and downloaded - results_dir = tmp_path / HETA_SINGLE_SPOT_GS_FILENAME.replace(".tiff", "") + results_dir = tmp_path / HETA_ANOTHER_SPOT_FILENAME.replace(".tiff", "") assert results_dir.is_dir(), f"Expected directory {results_dir} not found" files_in_dir = list(results_dir.glob("*")) assert len(files_in_dir) == 9, ( f"Expected 9 files in {results_dir}, but found {len(files_in_dir)}: {[f.name for f in files_in_dir]}" ) print(f"Found files in {results_dir}:") - for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in HETA_ANOTHER_SPOT_EXPECTED_RESULT_FILES: file_path = results_dir / filename if file_path.exists(): actual_size = file_path.stat().st_size print(f" {filename}: {actual_size} bytes (expected: {expected_size} ±{tolerance_percent}%)") else: print(f" {filename}: NOT FOUND") - for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in HETA_ANOTHER_SPOT_EXPECTED_RESULT_FILES: file_path = results_dir / filename assert file_path.exists(), f"Expected file {filename} not found" actual_size = file_path.stat().st_size diff --git a/tests/aignostics/application/gui_test.py b/tests/aignostics/application/gui_test.py index 28ce58b7f..4f5ec29e0 100644 --- a/tests/aignostics/application/gui_test.py +++ b/tests/aignostics/application/gui_test.py @@ -19,9 +19,9 @@ from tests.constants_test import ( HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, - HETA_EXPECTED_RESULT_FILES, - HETA_SINGLE_SPOT_GS_FILENAME, - HETA_SINGLE_SPOT_GS_FILESIZE, + HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES, + HETA_SINGLE_SPOT_FILENAME, + HETA_SINGLE_SPOT_FILESIZE, HETA_SINGLE_SPOT_GS_URL, ) @@ -302,7 +302,7 @@ async def test_gui_download_dataset_via_application_to_run_cancel( # noqa: PLR0 @pytest.mark.flaky(retries=1, delay=5) @pytest.mark.timeout(timeout=60 * 5) @pytest.mark.sequential # Helps on Linux with image analysis step otherwise timing out -async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, silent_logging: None) -> None: +async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, silent_logging: None) -> None: # noqa: PLR0915 """Test that the user can download a run result via the GUI.""" with patch( "aignostics.application._gui._page_application_run_describe.get_user_data_directory", @@ -372,14 +372,14 @@ async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, s input_dir = tmp_path / "input" assert input_dir.is_dir(), f"Expected input directory {input_dir} not found" - results_dir = tmp_path / run.run_id / HETA_SINGLE_SPOT_GS_FILENAME.replace(".tiff", "") + results_dir = tmp_path / run.run_id / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") assert results_dir.is_dir(), f"Expected run results directory {results_dir} not found" # Check for input file having been downloaded - input_file = tmp_path / run.run_id / "input" / HETA_SINGLE_SPOT_GS_FILENAME + input_file = tmp_path / run.run_id / "input" / HETA_SINGLE_SPOT_FILENAME assert input_file.is_file(), f"Expected input file {input_file} not found" - assert input_file.stat().st_size == HETA_SINGLE_SPOT_GS_FILESIZE, ( - f"Expected input file size {HETA_SINGLE_SPOT_GS_FILESIZE}, but got {input_file.stat().st_size}" + assert input_file.stat().st_size == HETA_SINGLE_SPOT_FILESIZE, ( + f"Expected input file size {HETA_SINGLE_SPOT_FILESIZE}, but got {input_file.stat().st_size}" ) # Check for files in the results directory @@ -390,14 +390,14 @@ async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, s ) print(f"Found files in {results_dir}:") - for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES: file_path = results_dir / filename if file_path.exists(): actual_size = file_path.stat().st_size print(f" {filename}: {actual_size} bytes (expected: {expected_size} ±{tolerance_percent}%)") else: print(f" {filename}: NOT FOUND") - for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES: file_path = results_dir / filename assert file_path.exists(), f"Expected file {filename} not found" actual_size = file_path.stat().st_size diff --git a/tests/aignostics/qupath/gui_test.py b/tests/aignostics/qupath/gui_test.py index 8502299eb..0e5abd9da 100644 --- a/tests/aignostics/qupath/gui_test.py +++ b/tests/aignostics/qupath/gui_test.py @@ -22,9 +22,9 @@ from tests.constants_test import ( HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, - HETA_EXPECTED_RESULT_FILES, - HETA_SINGLE_SPOT_GS_FILENAME, - HETA_SINGLE_SPOT_GS_FILESIZE, + HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES, + HETA_SINGLE_SPOT_FILENAME, + HETA_SINGLE_SPOT_FILESIZE, HETA_SINGLE_SPOT_GS_URL, ) @@ -242,17 +242,17 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 input_dir = tmp_path / "input" assert input_dir.is_dir(), f"Expected input directory {input_dir} not found" - results_dir = tmp_path / run.run_id / HETA_SINGLE_SPOT_GS_FILENAME.replace(".tiff", "") + results_dir = tmp_path / run.run_id / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") assert results_dir.is_dir(), f"Expected run results directory {results_dir} not found" qupath_dir = tmp_path / "qupath" assert qupath_dir.is_dir(), f"Expected QuPath directory {qupath_dir} not found" # Check for input file having been downloaded - input_file = tmp_path / run.run_id / "input" / HETA_SINGLE_SPOT_GS_FILENAME + input_file = tmp_path / run.run_id / "input" / HETA_SINGLE_SPOT_FILENAME assert input_file.is_file(), f"Expected input file {input_file} not found" - assert input_file.stat().st_size == HETA_SINGLE_SPOT_GS_FILESIZE, ( - f"Expected input file size {HETA_SINGLE_SPOT_GS_FILESIZE}, but got {input_file.stat().st_size}" + assert input_file.stat().st_size == HETA_SINGLE_SPOT_FILESIZE, ( + f"Expected input file size {HETA_SINGLE_SPOT_FILESIZE}, but got {input_file.stat().st_size}" ) # Check for files in the results directory @@ -263,14 +263,14 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 ) print(f"Found files in {results_dir}:") - for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES: file_path = results_dir / filename if file_path.exists(): actual_size = file_path.stat().st_size print(f" {filename}: {actual_size} bytes (expected: {expected_size} ±{tolerance_percent}%)") else: print(f" {filename}: NOT FOUND") - for filename, expected_size, tolerance_percent in HETA_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES: file_path = results_dir / filename assert file_path.exists(), f"Expected file {filename} not found" actual_size = file_path.stat().st_size diff --git a/tests/constants_test.py b/tests/constants_test.py index 63c38f557..bdee47f36 100644 --- a/tests/constants_test.py +++ b/tests/constants_test.py @@ -9,9 +9,9 @@ HETA_SINGLE_SPOT_GS_URL = ( "gs://platform-api-application-test-data/heta/slides/8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" ) -HETA_SINGLE_SPOT_GS_FILENAME = "8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" -HETA_SINGLE_SPOT_GS_FILESIZE = 14681750 # in bytes -HETA_EXPECTED_RESULT_FILES = [ +HETA_SINGLE_SPOT_FILENAME = "8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" +HETA_SINGLE_SPOT_FILESIZE = 14681750 # in bytes +HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES = [ ("tissue_segmentation_csv_class_information.csv", 342, 10), ("cell_classification_geojson_polygons.json", 16054058, 10), ("readout_generation_cell_readouts.csv", 2228907, 10), @@ -23,6 +23,23 @@ ("tissue_segmentation_segmentation_map_image.tiff", 581258, 10), ] +HETA_ANOTHER_SPOT_GS_URL = ( + "gs://aignx-storage-service-dev/sample_data_formatted/9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" +) +HETA_ANOTHER_SPOT_FILENAME = "9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" +HETA_ANOTHER_SPOT_FILESIZE = 10562338 # in bytes +HETA_ANOTHER_SPOT_EXPECTED_RESULT_FILES = [ + ("tissue_segmentation_csv_class_information.csv", 361, 10), + ("cell_classification_geojson_polygons.json", 9915953, 10), + ("readout_generation_cell_readouts.csv", 1470036, 10), + ("tissue_qc_csv_class_information.csv", 236, 10), + ("tissue_segmentation_geojson_polygons.json", 927599, 10), + ("tissue_qc_geojson_polygons.json", 315019, 10), + ("tissue_segmentation_segmentation_map_image.tiff", 2989980, 10), + ("readout_generation_slide_readouts.csv", 299865, 10), + ("tissue_segmentation_segmentation_map_image.tiff", 581258, 10), +] + TEST_THREE_SPOTS_GS_URLS = [ "gs://aignx-storage-service-dev/sample_data_formatted/9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff", "gs://aignx-storage-service-dev/sample_data_formatted/8c7b079e-8b8a-4036-bfde-5818352b503a.tiff", From 31b3a7e2236796e7beebdecfd8cbc9e03401adfd Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 10:04:23 +0100 Subject: [PATCH 04/16] chore(platform): start with two-legged e2e tests --- src/aignostics/platform/resources/runs.py | 125 +++++++---- tests/aignostics/application/cli_test.py | 4 +- tests/aignostics/application/gui_test.py | 8 +- tests/aignostics/platform/e2e_test.py | 250 ++++++++++++++++++---- tests/aignostics/qupath/gui_test.py | 6 +- 5 files changed, 306 insertions(+), 87 deletions(-) diff --git a/src/aignostics/platform/resources/runs.py b/src/aignostics/platform/resources/runs.py index 865155681..073409f4e 100644 --- a/src/aignostics/platform/resources/runs.py +++ b/src/aignostics/platform/resources/runs.py @@ -6,6 +6,7 @@ import builtins import logging +import time import typing as t from collections.abc import Iterator from pathlib import Path @@ -77,6 +78,10 @@ LIST_APPLICATION_RUNS_MIN_PAGE_SIZE = 5 +class DownloadTimeoutError(RuntimeError): + """Exception raised when the download operation exceeds its timeout.""" + + class Run: """Represents a single application run. @@ -209,8 +214,13 @@ def results_with_retry(run_id: str, **kwargs: object) -> list[ItemResultData]: return paginate(lambda **kwargs: results_with_retry(self.run_id, nocache=nocache, **kwargs)) - def download_to_folder( - self, download_base: Path | str, checksum_attribute_key: str = "checksum_base64_crc32c" + def download_to_folder( # noqa: C901 + self, + download_base: Path | str, + checksum_attribute_key: str = "checksum_base64_crc32c", + sleep_interval: int = 5, + timeout_seconds: int | None = None, + print_status: bool = True, ) -> None: """Downloads all result artifacts to a folder. @@ -219,43 +229,73 @@ def download_to_folder( Args: download_base (Path | str): Base directory to download results to. checksum_attribute_key (str): The key used to validate the checksum of the output artifacts. + sleep_interval (int): Time in seconds to wait between checks for new results. + timeout_seconds (int | None): Optional timeout in seconds for the entire download operation. + print_status (bool): If True, prints status updates to the console, otherwise just logs. Raises: ValueError: If the provided path is not a directory. - Exception: If downloads or API requests fail. + DownloadTimeoutError: If the timeout is exceeded while waiting for the run to terminate. + RuntimeError: If downloads or API requests fail. """ - # create application run base folder - download_base = Path(download_base) - if not download_base.is_dir(): - msg = f"{download_base} is not a directory" - raise ValueError(msg) - application_run_dir = Path(download_base) / self.run_id - - # incrementally check for available results - application_run_state = self.details().state - while application_run_state in {RunState.PROCESSING, RunState.PENDING}: - for item in self.results(): # we accept this might be stale for a bit - if item.state == ItemState.TERMINATED and item.output == ItemOutput.FULL: - self.ensure_artifacts_downloaded(application_run_dir, item, checksum_attribute_key) - sleep(5) - application_run_state = self.details().state - print(self) - - # check if last results have been downloaded yet and report on errors - for item in self.results(nocache=True): # no cache to get fresh results - match item.output: - case ItemOutput.FULL: - self.ensure_artifacts_downloaded(application_run_dir, item, checksum_attribute_key) - case ItemOutput.NONE: - print( - f"{item.external_id} failed with `{item.state.value}`.\n" - f"Termination reason `{item.termination_reason}`, " - f"error_code:`{item.error_code}`, message `{item.error_message}`." - ) + try: + # create application run base folder + download_base = Path(download_base) + if not download_base.is_dir(): + msg = f"{download_base} is not a directory" + raise ValueError(msg) # noqa: TRY301 + application_run_dir = Path(download_base) / self.run_id + + # track timeout if specified + start_time = time.time() if timeout_seconds is not None else None + + # incrementally check for available results + application_run_state = self.details(nocache=True).state # no cache to get fresh results + while application_run_state in {RunState.PROCESSING, RunState.PENDING}: + # check timeout + if start_time is not None and timeout_seconds is not None: + elapsed = time.time() - start_time + if elapsed >= timeout_seconds: + msg = ( + f"Timeout of {timeout_seconds} seconds exceeded while waiting for run {self.run_id} " + f"to terminate. Run state: {application_run_state.value}" + ) + raise DownloadTimeoutError(msg) # noqa: TRY301 + for item in self.results(nocache=True): + if item.state == ItemState.TERMINATED and item.output == ItemOutput.FULL: + self.ensure_artifacts_downloaded(application_run_dir, item, checksum_attribute_key) + sleep(sleep_interval) + application_run_state = self.details(nocache=True).state + logger.debug("Continuing to wait for run %s, current state: %r", self.run_id, self) + print(self) if print_status else None + + # check if last results have been downloaded yet and report on errors + for item in self.results(nocache=True): + match item.output: + case ItemOutput.FULL: + self.ensure_artifacts_downloaded(application_run_dir, item, checksum_attribute_key) + case ItemOutput.NONE: + message = ( + f"{item.external_id} failed with `{item.state.value}`.\n" + f"Termination reason `{item.termination_reason}`, " + f"error_code:`{item.error_code}`, message `{item.error_message}`." + ) + logger.error(message) + print(message) if print_status else None + except (ValueError, DownloadTimeoutError): + # Re-raise ValueError and DownloadTimeoutError as-is + raise + except Exception as e: + # Wrap all other exceptions in RuntimeError + msg = f"Download operation failed for run {self.run_id}: {e}" + raise RuntimeError(msg) from e @staticmethod def ensure_artifacts_downloaded( - base_folder: Path, item: ItemResultReadResponse, checksum_attribute_key: str = "checksum_base64_crc32c" + base_folder: Path, + item: ItemResultReadResponse, + checksum_attribute_key: str = "checksum_base64_crc32c", + print_status: bool = True, ) -> None: """Ensures all artifacts for an item are downloaded. @@ -265,6 +305,7 @@ def ensure_artifacts_downloaded( base_folder (Path): Base directory to download artifacts to. item (ItemResultReadResponse): The item result containing the artifacts to download. checksum_attribute_key (str): The key used to validate the checksum of the output artifacts. + print_status (bool): If True, prints status updates to the console, otherwise just logs. Raises: ValueError: If checksums don't match. @@ -279,28 +320,36 @@ def ensure_artifacts_downloaded( file_ending = mime_type_to_file_ending(get_mime_type_for_artifact(artifact)) file_path = item_dir / f"{artifact.name}{file_ending}" if not artifact.metadata: - message = f"Skipping artifact {artifact.name} for item {item.external_id}, no metadata present" - logger.error(message) + logger.error( + "Skipping artifact %s for item %s, no metadata present", artifact.name, item.external_id + ) + print( + f"> Skipping artifact {artifact.name} for item {item.external_id}, no metadata present" + ) if print_status else None continue checksum = artifact.metadata[checksum_attribute_key] if file_path.exists(): file_checksum = calculate_file_crc32c(file_path) if file_checksum != checksum: - print(f"> Resume download for {artifact.name} to {file_path}") + logger.debug("Resume download for %s to %s", artifact.name, file_path) + print(f"> Resume download for {artifact.name} to {file_path}") if print_status else None else: continue else: downloaded_at_least_one_artifact = True - print(f"> Download for {artifact.name} to {file_path}") + logger.debug("Download for %s to %s", artifact.name, file_path) + print(f"> Download for {artifact.name} to {file_path}") if print_status else None # if file is not there at all or only partially downloaded yet download_file(artifact.download_url, str(file_path), checksum) if downloaded_at_least_one_artifact: - print(f"Downloaded results for item: {item.external_id} to {item_dir}") + logger.debug("Downloaded results for item: %s to %s", item.external_id, item_dir) + print(f"Downloaded results for item: {item.external_id} to {item_dir}") if print_status else None else: - print(f"Results for item: {item.external_id} already present in {item_dir}") + logger.debug("Results for item: %s already present in %s", item.external_id, item_dir) + print(f"Results for item: {item.external_id} already present in {item_dir}") if print_status else None def __str__(self) -> str: """Returns a string representation of the application run. diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py index 20904ecf5..e7b107b6f 100644 --- a/tests/aignostics/application/cli_test.py +++ b/tests/aignostics/application/cli_test.py @@ -514,8 +514,8 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: print_directory_structure(tmp_path, "execute") # Validate no input dir, given we used an external id pointing to a local file - input_file = tmp_path / "input" - assert not input_file.is_dir(), f"Expected input directory {input_file} not found" + input_dir = tmp_path / "input" + assert not input_dir.is_dir(), f"Expected input directory {input_dir} not found" # Validate results generated and downloaded results_dir = tmp_path / HETA_ANOTHER_SPOT_FILENAME.replace(".tiff", "") diff --git a/tests/aignostics/application/gui_test.py b/tests/aignostics/application/gui_test.py index 4f5ec29e0..6203c31e9 100644 --- a/tests/aignostics/application/gui_test.py +++ b/tests/aignostics/application/gui_test.py @@ -360,7 +360,7 @@ async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, s # Check: Download completed await assert_notified(user, "Download completed.", 60 * 4) - print_directory_structure(tmp_path, "execute") + print_directory_structure(tmp_path, "downloaded_run") # Check for directory layout as expected run_dir = tmp_path / run.run_id @@ -369,14 +369,14 @@ async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, s subdirs = [d for d in run_dir.iterdir() if d.is_dir()] assert len(subdirs) == 2, f"Expected two subdirectories in {run_dir}, but found {len(subdirs)}" - input_dir = tmp_path / "input" + input_dir = run_dir / "input" assert input_dir.is_dir(), f"Expected input directory {input_dir} not found" - results_dir = tmp_path / run.run_id / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") + results_dir = run_dir / run.run_id / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") assert results_dir.is_dir(), f"Expected run results directory {results_dir} not found" # Check for input file having been downloaded - input_file = tmp_path / run.run_id / "input" / HETA_SINGLE_SPOT_FILENAME + input_file = input_dir / HETA_SINGLE_SPOT_FILENAME assert input_file.is_file(), f"Expected input file {input_file} not found" assert input_file.stat().st_size == HETA_SINGLE_SPOT_FILESIZE, ( f"Expected input file size {HETA_SINGLE_SPOT_FILESIZE}, but got {input_file.stat().st_size}" diff --git a/tests/aignostics/platform/e2e_test.py b/tests/aignostics/platform/e2e_test.py index 9cb86067f..dfe296b87 100644 --- a/tests/aignostics/platform/e2e_test.py +++ b/tests/aignostics/platform/e2e_test.py @@ -3,6 +3,7 @@ This module contains e2e tests that run real application workflows against the Aignostics platform. These tests verify e2e functionality including creating runs, downloading results, and validating outputs. + """ import tempfile @@ -30,14 +31,20 @@ TEST_THREE_SPOTS_GS_URLS, ) -TEST_APPLICATION_DEADLINE_SECONDS = 60 * 45 # 45 minutes -TEST_APPLICATION_DUE_DATE_SECONDS = 60 * 10 # 10 minutes +TEST_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS = 60 * 45 # 45 minutes +TEST_APPLICATION_SUBMIT_AND_WAIT_DUE_DATE_SECONDS = 60 * 10 # 10 minutes + +TEST_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS = 60 * 60 * 24 # 24 hours +TEST_APPLICATION_SUBMIT_AND_FIND_DUE_DATE_SECONDS = 60 * 60 * 24 # 24 hours -HETA_APPLICATION_DUE_DATE_SECONDS = 60 * 60 * 1 # 1 hour -HETA_APPLICATION_DEADLINE_SECONDS = 60 * 60 * 3 # 3 hours +HETA_APPLICATION_SUBMIT_AND_WAIT_DUE_DATE_SECONDS = 60 * 60 * 1 # 1 hour +HETA_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS = 60 * 60 * 3 # 3 hours +HETA_APPLICATION_SUBMIT_AND_FIND_DUE_DATE_SECONDS = 60 * 60 * 24 # 24 hours +HETA_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS = 60 * 60 * 24 # 24 hours -def _get_single_spot_payload_for_heta() -> list[platform.InputItem]: + +def _get_single_spot_payload_for_heta(expires_seconds: int) -> list[platform.InputItem]: """Generates a payload using a single spot.""" return [ platform.InputItem( @@ -46,8 +53,8 @@ def _get_single_spot_payload_for_heta() -> list[platform.InputItem]: platform.InputArtifact( name="whole_slide_image", download_url=platform.generate_signed_url( - HETA_SINGLE_SPOT_GS_URL, - HETA_APPLICATION_DEADLINE_SECONDS, + url=HETA_SINGLE_SPOT_GS_URL, + expires_seconds=expires_seconds, ), metadata={ "checksum_base64_crc32c": "5onqtA==", @@ -67,7 +74,7 @@ def _get_single_spot_payload_for_heta() -> list[platform.InputItem]: ] -def _get_three_spots_payload_for_test() -> list[platform.InputItem]: +def _get_three_spots_payload_for_test(expires_seconds: int) -> list[platform.InputItem]: """Generates a payload using three spots.""" return [ platform.InputItem( @@ -76,8 +83,8 @@ def _get_three_spots_payload_for_test() -> list[platform.InputItem]: platform.InputArtifact( name="whole_slide_image", download_url=platform.generate_signed_url( - TEST_THREE_SPOTS_GS_URLS[0], - TEST_APPLICATION_DEADLINE_SECONDS, + url=TEST_THREE_SPOTS_GS_URLS[0], + expires_seconds=expires_seconds, ), metadata={ "checksum_base64_crc32c": "9l3NNQ==", @@ -95,8 +102,8 @@ def _get_three_spots_payload_for_test() -> list[platform.InputItem]: platform.InputArtifact( name="whole_slide_image", download_url=platform.generate_signed_url( - TEST_THREE_SPOTS_GS_URLS[1], - TEST_APPLICATION_DEADLINE_SECONDS, + url=TEST_THREE_SPOTS_GS_URLS[1], + expires_seconds=expires_seconds, ), metadata={ "checksum_base64_crc32c": "w+ud3g==", @@ -114,8 +121,8 @@ def _get_three_spots_payload_for_test() -> list[platform.InputItem]: platform.InputArtifact( name="whole_slide_image", download_url=platform.generate_signed_url( - TEST_THREE_SPOTS_GS_URLS[2], - TEST_APPLICATION_DEADLINE_SECONDS, + url=TEST_THREE_SPOTS_GS_URLS[2], + expires_seconds=expires_seconds, ), metadata={ "checksum_base64_crc32c": "Zmx0wA==", @@ -130,17 +137,14 @@ def _get_three_spots_payload_for_test() -> list[platform.InputItem]: ] -def _run_application_test( # noqa: PLR0913, PLR0917 +def _submit_and_validate( application_id: str, application_version: str, payload: list[platform.InputItem], due_date_seconds: int, deadline_seconds: int, - checksum_attribute_key: str = "checksum_base64_crc32c", -) -> None: - """Helper function to run an application test. - - This function creates an application run, downloads results, and validates outputs. +) -> Run: + """Submit application run and validate its details. Args: application_id (str): The application ID to use for the test. @@ -148,13 +152,12 @@ def _run_application_test( # noqa: PLR0913, PLR0917 payload (list[platform.InputItem]): The input items for the application run. due_date_seconds (int): The due date in seconds from now for the application run. deadline_seconds (int): The deadline in seconds from now for the application run. - checksum_attribute_key (str): The key used to validate the checksum of the output artifacts. Raises: AssertionError: If any of the validation checks fail. """ client = platform.Client() - application_run = client.runs.submit( + run = client.runs.submit( application_id=application_id, application_version=application_version, items=payload, @@ -168,55 +171,222 @@ def _run_application_test( # noqa: PLR0913, PLR0917 } }, ) + details = run.details() + assert details.run_id == run.run_id, "Run ID mismatch after submission" + assert details.application_id == application_id, "Application ID mismatch after submission" + assert details.application_version == application_version, "Application version mismatch after submission" + assert details.state in {RunState.PENDING, RunState.PROCESSING}, ( + f"Unexpected run state `{details.state}` after submission" + ) + return run + + +def _submit_and_wait( # noqa: PLR0913, PLR0917 + application_id: str, + application_version: str, + payload: list[platform.InputItem], + due_date_seconds: int, + deadline_seconds: int, + checksum_attribute_key: str = "checksum_base64_crc32c", +) -> None: + """Helper function to run an application test. + + This function creates an application run, waits for results to become available, + downloads results, and validates outputs. + + Args: + application_id (str): The application ID to use for the test. + application_version (str): The application version to use for the test. + payload (list[platform.InputItem]): The input items for the application run. + due_date_seconds (int): The due date in seconds from now for the application run. + deadline_seconds (int): The deadline in seconds from now for the application run. + checksum_attribute_key (str): The key used to validate the checksum of the output artifacts. + + Raises: + AssertionError: If any of the validation checks fail. + """ + run = _submit_and_validate( + application_id=application_id, + application_version=application_version, + payload=payload, + due_date_seconds=due_date_seconds, + deadline_seconds=deadline_seconds, + ) with tempfile.TemporaryDirectory() as temp_dir: - application_run.download_to_folder(temp_dir, checksum_attribute_key) - _validate_output(application_run, Path(temp_dir), checksum_attribute_key) + run.download_to_folder(temp_dir, checksum_attribute_key, timeout_seconds=deadline_seconds) + _validate_output(run, Path(temp_dir), checksum_attribute_key) + + +def _find_and_validate( + application_id: str, + application_version: str, + payload: list[platform.InputItem], + due_date_seconds: int, + deadline_seconds: int, +) -> Run: + """Find application run submitted earlier and validate its details. + + Args: + application_id (str): The application ID to use for the test. + application_version (str): The application version to use for the test. + payload (list[platform.InputItem]): The input items for the application run. + due_date_seconds (int): The due date in seconds from now for the application run. + deadline_seconds (int): The deadline in seconds from now for the application run. + + Raises: + AssertionError: If any of the validation checks fail. + """ + client = platform.Client() + assert client is not None, "Failed to create platform client" + # TODO(Helmut): Build logic to find the run based on metadata once supported -@pytest.mark.skip(reason="v0.0.4 on production balking on whole_slide_image input") +@pytest.mark.skip( + reason="v0.0.4 on production balking on whole_slide_image input while identical version accepting on staging" +) @pytest.mark.e2e @pytest.mark.long_running -@pytest.mark.timeout(timeout=TEST_APPLICATION_DEADLINE_SECONDS + 60 * 30) -def test_application_runs_test_version() -> None: +@pytest.mark.timeout(timeout=TEST_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS + 60 * 5) +def test_platform_test_app_submit_and_wait() -> None: """Test application runs with the test application. This test creates an application run using the test application and three spots. - It then downloads the results to a temporary directory and performs various checks to ensure - the application run completed successfully and the results are valid. + It then waits for results to become available, downloads the results to a temporary directory + and performs various checks to ensure the application run completed successfully and the results are valid. Raises: AssertionError: If any of the validation checks fail. """ - _run_application_test( + _submit_and_wait( application_id=TEST_APPLICATION_ID, application_version=TEST_APPLICATION_VERSION, payload=_get_three_spots_payload_for_test(), - deadline_seconds=TEST_APPLICATION_DEADLINE_SECONDS, - due_date_seconds=TEST_APPLICATION_DUE_DATE_SECONDS, + deadline_seconds=TEST_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS, + due_date_seconds=TEST_APPLICATION_SUBMIT_AND_FIND_DUE_DATE_SECONDS, ) @pytest.mark.e2e @pytest.mark.very_long_running @pytest.mark.scheduled_only -@pytest.mark.timeout(timeout=HETA_APPLICATION_DEADLINE_SECONDS + 60 * 30) -def test_application_runs_heta_version() -> None: +@pytest.mark.timeout(timeout=HETA_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS + 60 * 5) +def test_platform_heta_app_submit_and_wait() -> None: """Test application runs with the HETA application. This test creates an application run using the HETA application and a single spot. - It then downloads the results to a temporary directory and performs various checks to ensure - the application run completed successfully and the results are valid. + It then waits for the results to become available, downloads the results to a + temporary directory and performs various checks to ensure the application run completed successfully + and the results are valid. + + Raises: + AssertionError: If any of the validation checks fail. + """ + _submit_and_wait( + application_id=HETA_APPLICATION_ID, + application_version=HETA_APPLICATION_VERSION, + payload=_get_single_spot_payload_for_heta( + expires_seconds=HETA_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS + 60 * 5 + ), + deadline_seconds=HETA_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS, + due_date_seconds=HETA_APPLICATION_SUBMIT_AND_WAIT_DUE_DATE_SECONDS, + ) + + +@pytest.mark.skip(reason="Waits for change in scheduler") +@pytest.mark.e2e +@pytest.mark.long_running +@pytest.mark.timeout(timeout=TEST_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS + 60 * 5) +def test_platform_test_app_submit() -> None: + """Test application submission with the test application. + + This test submits an application run with the test application and validates the submission. Raises: AssertionError: If any of the validation checks fail. """ - _run_application_test( + _submit_and_validate( + application_id=TEST_APPLICATION_ID, + application_version=TEST_APPLICATION_VERSION, + payload=_get_three_spots_payload_for_test( + expires_seconds=TEST_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS + 60 * 5 + ), + deadline_seconds=TEST_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS, + due_date_seconds=TEST_APPLICATION_SUBMIT_AND_WAIT_DUE_DATE_SECONDS, + ) + + +@pytest.mark.skip(reason="Waits for change in scheduler") +@pytest.mark.e2e +@pytest.mark.very_long_running +@pytest.mark.scheduled_only +@pytest.mark.timeout(timeout=TEST_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS + 60 * 5) +def test_platform_test_app_find() -> None: + """Test application runs with the test application. + + This test finds an application run with the test application submitted earlier and + validates it completed successfully and in time. + + Raises: + AssertionError: If any of the validation checks fail. + """ + _find_and_validate( + application_id=TEST_APPLICATION_ID, + application_version=TEST_APPLICATION_VERSION, + payload=_get_three_spots_payload_for_test( + expires_seconds=TEST_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS + 60 * 5 + ), + deadline_seconds=TEST_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS, + due_date_seconds=TEST_APPLICATION_SUBMIT_AND_FIND_DUE_DATE_SECONDS, + ) + + +@pytest.mark.skip(reason="Waits for change in scheduler") +@pytest.mark.e2e +@pytest.mark.very_long_running +@pytest.mark.scheduled_only +@pytest.mark.timeout(timeout=HETA_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS + 60 * 5) +def test_platform_heta_app_submit() -> None: + """Test application runs with the HETA application. + + This test submits an application run with the HETA application and validates the submission. + + Raises: + AssertionError: If any of the validation checks fail. + """ + _submit_and_validate( + application_id=HETA_APPLICATION_ID, + application_version=HETA_APPLICATION_VERSION, + payload=_get_single_spot_payload_for_heta( + expires_seconds=HETA_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS + 60 * 5 + ), + deadline_seconds=HETA_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS, + due_date_seconds=HETA_APPLICATION_SUBMIT_AND_FIND_DUE_DATE_SECONDS, + ) + + +@pytest.mark.skip(reason="Waits for change in scheduler") +@pytest.mark.e2e +@pytest.mark.very_long_running +@pytest.mark.scheduled_only +@pytest.mark.timeout(timeout=HETA_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS + 60 * 5) +def test_platform_heta_app_find() -> None: + """Test application runs with the HETA application. + + This test finds an application run with the HETA application submitted earlier and + validates it completed successfully and in time. + + Raises: + AssertionError: If any of the validation checks fail. + """ + _find_and_validate( application_id=HETA_APPLICATION_ID, application_version=HETA_APPLICATION_VERSION, - payload=_get_single_spot_payload_for_heta(), - deadline_seconds=HETA_APPLICATION_DEADLINE_SECONDS, - due_date_seconds=HETA_APPLICATION_DUE_DATE_SECONDS, + payload=_get_single_spot_payload_for_heta( + expires_seconds=HETA_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS + 60 * 5 + ), + deadline_seconds=HETA_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS, + due_date_seconds=HETA_APPLICATION_SUBMIT_AND_FIND_DUE_DATE_SECONDS, ) diff --git a/tests/aignostics/qupath/gui_test.py b/tests/aignostics/qupath/gui_test.py index 0e5abd9da..180f364d9 100644 --- a/tests/aignostics/qupath/gui_test.py +++ b/tests/aignostics/qupath/gui_test.py @@ -239,17 +239,17 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 subdirs = [d for d in run_dir.iterdir() if d.is_dir()] assert len(subdirs) == 3, f"Expected three subdirectories in {run_dir}, but found {len(subdirs)}" - input_dir = tmp_path / "input" + input_dir = run_dir / "input" assert input_dir.is_dir(), f"Expected input directory {input_dir} not found" - results_dir = tmp_path / run.run_id / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") + results_dir = run_dir / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") assert results_dir.is_dir(), f"Expected run results directory {results_dir} not found" qupath_dir = tmp_path / "qupath" assert qupath_dir.is_dir(), f"Expected QuPath directory {qupath_dir} not found" # Check for input file having been downloaded - input_file = tmp_path / run.run_id / "input" / HETA_SINGLE_SPOT_FILENAME + input_file = input_dir / HETA_SINGLE_SPOT_FILENAME assert input_file.is_file(), f"Expected input file {input_file} not found" assert input_file.stat().st_size == HETA_SINGLE_SPOT_FILESIZE, ( f"Expected input file size {HETA_SINGLE_SPOT_FILESIZE}, but got {input_file.stat().st_size}" From f147a778d90cc31ff62c08dbbc0ad25f84986040 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 10:05:10 +0100 Subject: [PATCH 05/16] chore(platform): start with two-legged e2e tests --- tests/aignostics/qupath/gui_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aignostics/qupath/gui_test.py b/tests/aignostics/qupath/gui_test.py index 180f364d9..835a47789 100644 --- a/tests/aignostics/qupath/gui_test.py +++ b/tests/aignostics/qupath/gui_test.py @@ -245,7 +245,7 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 results_dir = run_dir / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") assert results_dir.is_dir(), f"Expected run results directory {results_dir} not found" - qupath_dir = tmp_path / "qupath" + qupath_dir = run_dir / "qupath" assert qupath_dir.is_dir(), f"Expected QuPath directory {qupath_dir} not found" # Check for input file having been downloaded From c7be57d9a5b8787b8dc271d895efbdb536154ea8 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 12:36:02 +0100 Subject: [PATCH 06/16] feat(platform): support for tags in custom sdk metadata feat(application): allow to submit tags via CLI and find back runs via tags --- src/aignostics/application/_cli.py | 28 +- .../_gui/_page_application_describe.py | 2 +- src/aignostics/application/_service.py | 96 ++++- src/aignostics/platform/_sdk_metadata.py | 8 +- src/aignostics/platform/_utils.py | 29 ++ src/aignostics/platform/resources/runs.py | 60 ++-- tests/aignostics/application/cli_test.py | 126 ++++++- tests/aignostics/application/gui_test.py | 2 +- tests/aignostics/platform/e2e_test.py | 33 +- .../platform/resources/runs_test.py | 330 +++++++++++++++++- .../aignostics/platform/sdk_metadata_test.py | 188 ++++++++++ tests/aignostics/platform/utils_test.py | 154 ++++++++ tests/aignostics/qupath/gui_test.py | 2 +- tests/constants_test.py | 4 +- 14 files changed, 989 insertions(+), 73 deletions(-) diff --git a/src/aignostics/application/_cli.py b/src/aignostics/application/_cli.py index b2e004234..501c03dd1 100644 --- a/src/aignostics/application/_cli.py +++ b/src/aignostics/application/_cli.py @@ -624,6 +624,10 @@ def run_submit( # noqa: PLR0913, PLR0917 str | None, typer.Option(help="Optional note to include with the run submission via custom metadata."), ] = None, + tags: Annotated[ + str | None, + typer.Option(help="Optional comma-separated list of tags to attach to the run for filtering."), + ] = None, due_date: Annotated[ str | None, typer.Option( @@ -703,6 +707,7 @@ def run_submit( # noqa: PLR0913, PLR0917 application_version=application_version, custom_metadata=None, # TODO(Helmut): Add support for custom metadata note=note, + tags={tag.strip() for tag in tags.split(",") if tag.strip()} if tags else None, due_date=due_date, deadline=deadline, onboard_to_aignostics_portal=onboard_to_aignostics_portal, @@ -740,12 +745,31 @@ def run_submit( # noqa: PLR0913, PLR0917 def run_list( verbose: Annotated[bool, typer.Option(help="Show application details")] = False, limit: Annotated[int | None, typer.Option(help="Maximum number of runs to display")] = None, + tags: Annotated[ + str | None, + typer.Option(help="Optional comma-separated list of tags to filter runs. All tags must match."), + ] = None, + note_regex: Annotated[ + str | None, + typer.Option(help="Optional regex pattern to filter runs by note metadata."), + ] = None, + note_case_insensitive: Annotated[bool, typer.Option(help="Make note regex search case-insensitive.")] = True, ) -> None: """List runs.""" try: - runs = Service().application_runs(limit=limit) + runs = Service().application_runs( + limit=limit, + tags={tag.strip() for tag in tags.split(",") if tag.strip()} if tags else None, + note_regex=note_regex, + note_query_case_insensitive=note_case_insensitive, + ) if len(runs) == 0: - message = "You did not yet create a run." + if tags: + message = f"You did not yet create a run matching tags: {tags!r}." + elif note_regex: + message = f"You did not yet create a run matching note pattern: {note_regex!r}." + else: + message = "You did not yet create a run." logger.warning(message) console.print(message, style="warning") else: diff --git a/src/aignostics/application/_gui/_page_application_describe.py b/src/aignostics/application/_gui/_page_application_describe.py index 63ddd1007..54fde70be 100644 --- a/src/aignostics/application/_gui/_page_application_describe.py +++ b/src/aignostics/application/_gui/_page_application_describe.py @@ -591,7 +591,7 @@ class ThumbnailRenderer { submit_form.metadata_exclude_button.disable() ui.button("Back", on_click=stepper.previous).props("flat") - with ui.step("Leave Note"): + with ui.step("Leave Note and Tags"): with ui.column(align_items="start").classes("w-full"): ui.textarea( label="Note (optional)", diff --git a/src/aignostics/application/_service.py b/src/aignostics/application/_service.py index 9fe06399b..e431d120f 100644 --- a/src/aignostics/application/_service.py +++ b/src/aignostics/application/_service.py @@ -574,7 +574,7 @@ def application_runs_static( # noqa: PLR0913, PLR0917 ) ] - def application_runs( # noqa: PLR0913, PLR0917 + def application_runs( # noqa: C901, PLR0912, PLR0913, PLR0917 self, application_id: str | None = None, application_version: str | None = None, @@ -582,6 +582,7 @@ def application_runs( # noqa: PLR0913, PLR0917 has_output: bool = False, note_regex: str | None = None, note_query_case_insensitive: bool = True, + tags: set[str] | None = None, limit: int | None = None, ) -> list[RunData]: """Get a list of all application runs. @@ -594,6 +595,8 @@ def application_runs( # noqa: PLR0913, PLR0917 has_output (bool): If True, only runs with partial or full output are retrieved. note_regex (str | None): Optional regex to filter runs by note metadata. If None, no filtering is applied. note_query_case_insensitive (bool): If True, the note_regex is case insensitive. Default is True. + tags (set[str] | None): Optional set of tags to filter runs. All tags must match. + If None, no filtering is applied. limit (int | None): The maximum number of runs to retrieve. If None, all runs are retrieved. Returns: @@ -607,12 +610,30 @@ def application_runs( # noqa: PLR0913, PLR0917 runs = [] page_size = LIST_APPLICATION_RUNS_MAX_PAGE_SIZE try: + custom_metadata = None + client_side_note_filter = None + + # Handle note_regex filter if note_regex: flag_case_insensitive = ' flag "i"' if note_query_case_insensitive else "" - custom_metadata = f'$.sdk.note ? (@ like_regex "{note_regex}"{flag_case_insensitive})' - else: - custom_metadata = None - + # If we also have tags, we'll need to do note filtering client-side + if tags: + # Store for client-side filtering + client_side_note_filter = (note_regex, note_query_case_insensitive) + else: + # No tags, so we can filter note on backend + custom_metadata = f'$.sdk.note ? (@ like_regex "{note_regex}"{flag_case_insensitive})' + + # Handle tags filter + if tags: + # JSONPath filter to match all of the provided tags in the sdk.tags array + # PostgreSQL limitation: Cannot use && between separate path expressions as backend crashes with 500 + # Workaround: Filter on backend for ANY tag match, then filter client-side for ALL + # Use regex alternation to match any of the tags + escaped_tags = [tag.replace('"', '\\"').replace("\\", "\\\\") for tag in tags] + # Create regex pattern: ^(tag1|tag2|tag3)$ + regex_pattern = "^(" + "|".join(escaped_tags) + ")$" + custom_metadata = f'$.sdk.tags ? (@ like_regex "{regex_pattern}")' run_iterator = self._get_platform_client().runs.list_data( application_id=application_id, application_version=application_version, @@ -624,6 +645,38 @@ def application_runs( # noqa: PLR0913, PLR0917 for run in run_iterator: if has_output and run.output == RunOutput.NONE: continue + + # Client-side filtering when combining multiple criteria + # 1. If multiple tags specified, ensure ALL are present + if tags and len(tags) > 1: + # Backend filter with regex alternation matches ANY tag + # Now verify ALL tags are present in run metadata + run_tags = set() + if run.custom_metadata and "sdk" in run.custom_metadata: + sdk_metadata = run.custom_metadata.get("sdk", {}) + if "tags" in sdk_metadata: + run_tags = set(sdk_metadata.get("tags", [])) + + # Check if all required tags are present + if not tags.issubset(run_tags): + continue # Skip this run, not all tags match + + # 2. If note filter is applied client-side (when combined with tags) + if client_side_note_filter: + note_pattern, case_insensitive = client_side_note_filter + run_note = None + if run.custom_metadata and "sdk" in run.custom_metadata: + sdk_metadata = run.custom_metadata.get("sdk", {}) + run_note = sdk_metadata.get("note") + + # Check if note matches the regex pattern + if run_note: + flags = re.IGNORECASE if case_insensitive else 0 + if not re.search(note_pattern, run_note, flags): + continue # Skip this run, note doesn't match + else: + continue # Skip this run, no note present + runs.append(run) if limit is not None and len(runs) >= limit: break @@ -659,6 +712,7 @@ def application_run_submit_from_metadata( # noqa: PLR0913, PLR0917 application_version: str | None = None, custom_metadata: dict[str, Any] | None = None, note: str | None = None, + tags: set[str] | None = None, due_date: str | None = None, deadline: str | None = None, onboard_to_aignostics_portal: bool = False, @@ -671,6 +725,7 @@ def application_run_submit_from_metadata( # noqa: PLR0913, PLR0917 metadata (list[dict[str, Any]]): The metadata for the run. custom_metadata (dict[str, Any] | None): Optional custom metadata to attach to the run. note (str | None): An optional note for the run. + tags (set[str] | None): Optional set of tags to attach to the run for filtering. due_date (str | None): An optional requested completion time for the run, ISO8601 format. The scheduler will try to complete the run before this time, taking the subscription tier and available GPU resources into account. @@ -773,6 +828,7 @@ def application_run_submit_from_metadata( # noqa: PLR0913, PLR0917 application_version=app_version.version_number, custom_metadata=custom_metadata, note=note, + tags=tags, due_date=due_date, deadline=deadline, onboard_to_aignostics_portal=onboard_to_aignostics_portal, @@ -807,6 +863,7 @@ def application_run_submit( # noqa: PLR0913, PLR0917 application_version: str | None = None, custom_metadata: dict[str, Any] | None = None, note: str | None = None, + tags: set[str] | None = None, due_date: str | None = None, deadline: str | None = None, onboard_to_aignostics_portal: bool = False, @@ -820,6 +877,7 @@ def application_run_submit( # noqa: PLR0913, PLR0917 application_version (str | None): The version of the application to run. custom_metadata (dict[str, Any] | None): Optional custom metadata to attach to the run. note (str | None): An optional note for the run. + tags (set[str] | None): Optional set of tags to attach to the run for filtering. due_date (str | None): An optional requested completion time for the run, ISO8601 format. The scheduler will try to complete the run before this time, taking the subscription tier and available GPU resources into account. @@ -844,18 +902,26 @@ def application_run_submit( # noqa: PLR0913, PLR0917 try: if custom_metadata is None: custom_metadata = {} - custom_metadata["sdk"] = { - "note": note, - "workflow": { + + sdk_metadata: dict[str, Any] = {} + if note: + sdk_metadata["note"] = note + if tags: + sdk_metadata["tags"] = tags + if onboard_to_aignostics_portal or validate_only: + sdk_metadata["workflow"] = { "onboard_to_aignostics_portal": onboard_to_aignostics_portal, "validate_only": validate_only, - }, - "scheduling": { - "due_date": due_date, - "deadline": deadline, - }, - } - custom_metadata["sdk"]["note"] = note + } + if due_date or deadline: + sdk_metadata["scheduling"] = {} + if due_date: + sdk_metadata["scheduling"]["due_date"] = due_date + if deadline: + sdk_metadata["scheduling"]["deadline"] = deadline + + custom_metadata["sdk"] = sdk_metadata + return self._get_platform_client().runs.submit( application_id=application_id, items=items, diff --git a/src/aignostics/platform/_sdk_metadata.py b/src/aignostics/platform/_sdk_metadata.py index 89fba4de4..7d84aba6b 100644 --- a/src/aignostics/platform/_sdk_metadata.py +++ b/src/aignostics/platform/_sdk_metadata.py @@ -15,8 +15,8 @@ logger = get_logger(__name__) -SDK_METADATA_SCHEMA_VERSION = "0.0.2" -ITEM_SDK_METADATA_SCHEMA_VERSION = "0.0.1" +SDK_METADATA_SCHEMA_VERSION = "0.0.3" +ITEM_SDK_METADATA_SCHEMA_VERSION = "0.0.2" class SubmissionMetadata(BaseModel): @@ -111,6 +111,8 @@ class RunSdkMetadata(BaseModel): schema_version: str = Field( ..., description="Schema version for this metadata format", pattern=r"^\d+\.\d+\.\d+-?.*$" ) + + tags: set[str] | None = Field(None, description="Optional list of tags associated with the run") submission: SubmissionMetadata = Field(..., description="Submission context metadata") user_agent: str = Field(..., description="User agent string for the SDK client") user: UserMetadata | None = Field(None, description="User information (when authenticated)") @@ -141,6 +143,8 @@ class ItemSdkMetadata(BaseModel): schema_version: str = Field( ..., description="Schema version for this metadata format", pattern=r"^\d+\.\d+\.\d+-?.*$" ) + + tags: set[str] | None = Field(None, description="Optional list of tags associated with the item") platform_bucket: PlatformBucketMetadata | None = Field(None, description="Platform bucket storage information") model_config = {"extra": "forbid"} # Reject unknown fields diff --git a/src/aignostics/platform/_utils.py b/src/aignostics/platform/_utils.py index e788c46ec..97bd88da7 100644 --- a/src/aignostics/platform/_utils.py +++ b/src/aignostics/platform/_utils.py @@ -33,6 +33,35 @@ SIGNED_DOWNLOAD_URL_EXPIRES_SECONDS_DEFAULT = 6 * 60 * 60 # 6 hours +def convert_to_json_serializable(obj: object) -> object: + """Recursively convert non-JSON-serializable types to serializable equivalents. + + Handles common Python types that are not directly JSON-serializable: + - set → sorted list (for consistency and deterministic output) + - Recursively processes nested dict, list, and tuple structures + + Args: + obj: The object to convert. + + Returns: + The converted object with all non-serializable types replaced. + + Examples: + >>> convert_to_json_serializable({"tags": {"a", "c", "b"}}) + {"tags": ["a", "b", "c"]} + + >>> convert_to_json_serializable({"nested": {"items": {1, 2, 3}}}) + {"nested": {"items": [1, 2, 3]}} + """ + if isinstance(obj, set): + return sorted(obj) # Convert set to sorted list for consistency + if isinstance(obj, dict): + return {key: convert_to_json_serializable(value) for key, value in obj.items()} + if isinstance(obj, (list, tuple)): + return [convert_to_json_serializable(item) for item in obj] + return obj + + def mime_type_to_file_ending(mime_type: str) -> str: """Converts a MIME type to an appropriate file extension. diff --git a/src/aignostics/platform/resources/runs.py b/src/aignostics/platform/resources/runs.py index 073409f4e..0b19706a6 100644 --- a/src/aignostics/platform/resources/runs.py +++ b/src/aignostics/platform/resources/runs.py @@ -55,6 +55,7 @@ from aignostics.platform._settings import settings from aignostics.platform._utils import ( calculate_file_crc32c, + convert_to_json_serializable, download_file, get_mime_type_for_artifact, mime_type_to_file_ending, @@ -66,7 +67,7 @@ logger = get_logger(__name__) RETRYABLE_EXCEPTIONS = ( - ServiceException, + ServiceException, # TODO(Helmut): Do we want this down the road? Urllib3TimeoutError, PoolError, IncompleteRead, @@ -155,13 +156,12 @@ def cancel(self) -> None: Raises: Exception: If the API request fails. """ - # Clear all caches since run state is changing - operation_cache_clear() self._api.cancel_run_v1_runs_run_id_cancel_post( self.run_id, _request_timeout=settings().run_cancel_timeout, _headers={"User-Agent": user_agent()}, ) + operation_cache_clear() # Clear all caches since we added a new run def delete(self) -> None: """Delete the application run. @@ -169,13 +169,12 @@ def delete(self) -> None: Raises: Exception: If the API request fails. """ - # Clear all caches since run is being deleted - operation_cache_clear() self._api.delete_run_items_v1_runs_run_id_artifacts_delete( self.run_id, _request_timeout=settings().run_delete_timeout, _headers={"User-Agent": user_agent()}, ) + operation_cache_clear() # Clear all caches since we added a new run def results(self, nocache: bool = False) -> t.Iterator[ItemResultData]: """Retrieves the results of all items in the run. @@ -431,21 +430,27 @@ def submit( payload = RunCreationRequest( application_id=application_id, version_number=application_version, - custom_metadata=custom_metadata, + custom_metadata=cast("dict[str, Any]", convert_to_json_serializable(custom_metadata)), items=items, ) self._validate_input_items(payload) - # Clear all caches since we added a new run - operation_cache_clear() res: RunCreationResponse = self._api.create_run_v1_runs_post( payload, _request_timeout=settings().run_submit_timeout, _headers={"User-Agent": user_agent()}, ) + operation_cache_clear() # Clear all caches since we added a new run return Run(self._api, str(res.run_id)) - def list( - self, application_id: str | None = None, application_version: str | None = None, nocache: bool = False + def list( # noqa: PLR0913, PLR0917 + self, + application_id: str | None = None, + application_version: str | None = None, + external_id: str | None = None, + custom_metadata: str | None = None, + sort: str | None = None, + page_size: int = LIST_APPLICATION_RUNS_MAX_PAGE_SIZE, + nocache: bool = False, ) -> Iterator[Run]: """Find application runs, optionally filtered by application id and/or version. @@ -454,41 +459,32 @@ def list( Args: application_id (str | None): Optional application ID to filter by. application_version (str | None): Optional application version to filter by. + external_id (str | None): The external ID to filter runs. If None, no filtering is applied. + custom_metadata (str | None): Optional metadata filter in JSONPath format. + sort (str | None): Optional field to sort by. Prefix with '-' for descending order. + page_size (int): Number of items per page, defaults to max nocache (bool): If True, skip reading from cache and fetch fresh data from the API. The fresh result will still be cached for subsequent calls. Defaults to False. Returns: - Iterator[Run]: An iterator yielding application runs. + Iterator[Run]: An iterator yielding application run handles. Raises: + ValueError: If page_size is greater than 100. Exception: If the API request fails. """ - - @cached_operation(ttl=settings().run_cache_ttl, use_token=True) - def list_with_retry(**kwargs: object) -> list[RunData]: - return Retrying( - retry=retry_if_exception_type(exception_types=RETRYABLE_EXCEPTIONS), - stop=stop_after_attempt(settings().run_retry_attempts), - wait=wait_exponential_jitter(initial=settings().run_retry_wait_min, max=settings().run_retry_wait_max), - before_sleep=before_sleep_log(logger, logging.WARNING), - reraise=True, - )( - lambda: self._api.list_runs_v1_runs_get( - _request_timeout=settings().run_timeout, - _headers={"User-Agent": user_agent()}, - **kwargs, # pyright: ignore[reportArgumentType] - ) - ) - - res = paginate( - lambda **kwargs: list_with_retry( + return ( + Run(self._api, response.run_id) + for response in self.list_data( application_id=application_id, application_version=application_version, + external_id=external_id, + custom_metadata=custom_metadata, + sort=sort, + page_size=page_size, nocache=nocache, - **kwargs, ) ) - return (Run(self._api, response.run_id) for response in res) def list_data( # noqa: PLR0913, PLR0917 self, diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py index e7b107b6f..9e1ed13ee 100644 --- a/tests/aignostics/application/cli_test.py +++ b/tests/aignostics/application/cli_test.py @@ -243,7 +243,7 @@ def test_cli_run_submit_fails_on_missing_url(runner: CliRunner, tmp_path: Path) @pytest.mark.e2e @pytest.mark.long_running @pytest.mark.timeout(timeout=60 * 10) -def test_cli_run_submit_and_describe_and_cancel_and_download_and_delete( +def test_cli_run_submit_and_describe_and_cancel_and_download_and_delete( # noqa: PLR0915 runner: CliRunner, tmp_path: Path, silent_logging ) -> None: """Check run submit command runs successfully.""" @@ -263,6 +263,8 @@ def test_cli_run_submit_and_describe_and_cancel_and_download_and_delete( str(csv_path), "--note", "test_cli_run_submit_and_describe_and_cancel_and_download_and_delete", + "--tags", + "cli-test,test_cli_run_submit_and_describe_and_cancel_and_download_and_delete,further-tag", "--deadline", (datetime.now(tz=UTC) + timedelta(minutes=10)).isoformat(), "--validate-only", @@ -280,6 +282,128 @@ def test_cli_run_submit_and_describe_and_cancel_and_download_and_delete( assert run_id_match, f"Failed to extract run ID from output '{output}'" run_id = run_id_match.group(1) + # Test that we can find this run by it's note + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--note-regex", + "test_cli_run_submit_and_describe_and_cancel_and_download_and_delete", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id in list_output, f"Run ID '{run_id}' not found when filtering by note" + + # but not another note + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--note-regex", + "other_note", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id not in list_output, f"Run ID '{run_id}' found when filtering by other note" + + # Test that we can find this run by one of its tags + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--tags", + "test_cli_run_submit_and_describe_and_cancel_and_download_and_delete", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id in list_output, f"Run ID '{run_id}' not found when filtering by one tag" + + # but not another tag + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--tags", + "other-tag", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id not in list_output, f"Run ID '{run_id}' found when filtering by other tag" + + # Test that we can find this run by two of its tags + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--tags", + "cli-test,test_cli_run_submit_and_describe_and_cancel_and_download_and_delete", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id in list_output, f"Run ID '{run_id}' not found when filtering by two tags" + + # Test that we can find this run by all of its tags + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--tags", + "cli-test,test_cli_run_submit_and_describe_and_cancel_and_download_and_delete,further-tag", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id in list_output, f"Run ID '{run_id}' not found when filtering by all tags" + + # Test that we cannot find this run by all of its tags and a non-existent tag + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--tags", + "cli-test,test_cli_run_submit_and_describe_and_cancel_and_download_and_delete,further-tag,non-existing-tag", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id not in list_output, f"Run ID '{run_id}' found when filtering by all tags" + + # Test that we can find this run by all of its tags and it's note + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--note-regex", + "test_cli_run_submit_and_describe_and_cancel_and_download_and_delete", + "--tags", + "cli-test,test_cli_run_submit_and_describe_and_cancel_and_download_and_delete,further-tag", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id in list_output, f"Run ID '{run_id}' not found when filtering by all tags and note" + # Test the describe command with the extracted run ID describe_result = runner.invoke(cli, ["application", "run", "describe", run_id]) assert describe_result.exit_code == 0 diff --git a/tests/aignostics/application/gui_test.py b/tests/aignostics/application/gui_test.py index 6203c31e9..36190f5d3 100644 --- a/tests/aignostics/application/gui_test.py +++ b/tests/aignostics/application/gui_test.py @@ -372,7 +372,7 @@ async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, s input_dir = run_dir / "input" assert input_dir.is_dir(), f"Expected input directory {input_dir} not found" - results_dir = run_dir / run.run_id / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") + results_dir = run_dir / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") assert results_dir.is_dir(), f"Expected run results directory {results_dir} not found" # Check for input file having been downloaded diff --git a/tests/aignostics/platform/e2e_test.py b/tests/aignostics/platform/e2e_test.py index dfe296b87..48b639d16 100644 --- a/tests/aignostics/platform/e2e_test.py +++ b/tests/aignostics/platform/e2e_test.py @@ -137,12 +137,13 @@ def _get_three_spots_payload_for_test(expires_seconds: int) -> list[platform.Inp ] -def _submit_and_validate( +def _submit_and_validate( # noqa: PLR0913, PLR0917 application_id: str, application_version: str, payload: list[platform.InputItem], due_date_seconds: int, deadline_seconds: int, + tags: set[str] | None = None, ) -> Run: """Submit application run and validate its details. @@ -152,6 +153,7 @@ def _submit_and_validate( payload (list[platform.InputItem]): The input items for the application run. due_date_seconds (int): The due date in seconds from now for the application run. deadline_seconds (int): The deadline in seconds from now for the application run. + tags (set[str] | None): A set of tags to attach to the application run. Raises: AssertionError: If any of the validation checks fail. @@ -163,7 +165,7 @@ def _submit_and_validate( items=payload, custom_metadata={ "sdk": { - "note": "_run_application_test", + "tags": tags or set(), "scheduling": { "due_date": (datetime.now(tz=UTC) + timedelta(seconds=due_date_seconds)).isoformat(), "deadline": (datetime.now(tz=UTC) + timedelta(seconds=deadline_seconds)).isoformat(), @@ -178,6 +180,26 @@ def _submit_and_validate( assert details.state in {RunState.PENDING, RunState.PROCESSING}, ( f"Unexpected run state `{details.state}` after submission" ) + + # Build JSONPath filter for tags if provided + tags_filter = None + if tags: + # JSONPath filter to match all of the provided tags in the sdk.tags array + # PostgreSQL JSONPath: Use equality operator for exact match + for tag in tags: + tag_condition = f'$.sdk.tags ? (@ == "{tag}")' + tags_filter = f"({tags_filter}) && ({tag_condition})" if tags_filter else tag_condition + + runs = client.runs.list( + application_id=application_id, + application_version=application_version, + custom_metadata=tags_filter, + ) + + # Find the submitted run in the list + matched_runs = [r for r in runs if r.run_id == run.run_id] + assert len(matched_runs) == 1, f"Submitted run `{run.run_id}` not found in run listing" + return run @@ -187,6 +209,7 @@ def _submit_and_wait( # noqa: PLR0913, PLR0917 payload: list[platform.InputItem], due_date_seconds: int, deadline_seconds: int, + tags: set[str] | None = None, checksum_attribute_key: str = "checksum_base64_crc32c", ) -> None: """Helper function to run an application test. @@ -200,6 +223,7 @@ def _submit_and_wait( # noqa: PLR0913, PLR0917 payload (list[platform.InputItem]): The input items for the application run. due_date_seconds (int): The due date in seconds from now for the application run. deadline_seconds (int): The deadline in seconds from now for the application run. + tags (set[str] | None): A set of tags to attach to the application run. checksum_attribute_key (str): The key used to validate the checksum of the output artifacts. Raises: @@ -211,6 +235,7 @@ def _submit_and_wait( # noqa: PLR0913, PLR0917 payload=payload, due_date_seconds=due_date_seconds, deadline_seconds=deadline_seconds, + tags=tags, ) with tempfile.TemporaryDirectory() as temp_dir: @@ -264,6 +289,7 @@ def test_platform_test_app_submit_and_wait() -> None: payload=_get_three_spots_payload_for_test(), deadline_seconds=TEST_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS, due_date_seconds=TEST_APPLICATION_SUBMIT_AND_FIND_DUE_DATE_SECONDS, + tags=["test_platform_test_app_submit_and_wait"], ) @@ -290,6 +316,7 @@ def test_platform_heta_app_submit_and_wait() -> None: ), deadline_seconds=HETA_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS, due_date_seconds=HETA_APPLICATION_SUBMIT_AND_WAIT_DUE_DATE_SECONDS, + tags=["test_platform_heta_app_submit_and_wait"], ) @@ -313,6 +340,7 @@ def test_platform_test_app_submit() -> None: ), deadline_seconds=TEST_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS, due_date_seconds=TEST_APPLICATION_SUBMIT_AND_WAIT_DUE_DATE_SECONDS, + tags=["test_platform_heta_app_submit_and_wait"], ) @@ -362,6 +390,7 @@ def test_platform_heta_app_submit() -> None: ), deadline_seconds=HETA_APPLICATION_SUBMIT_AND_FIND_DEADLINE_SECONDS, due_date_seconds=HETA_APPLICATION_SUBMIT_AND_FIND_DUE_DATE_SECONDS, + tags=["test_platform_heta_app_submit_and_find"], ) diff --git a/tests/aignostics/platform/resources/runs_test.py b/tests/aignostics/platform/resources/runs_test.py index dc61f9ae9..1bcf037d8 100644 --- a/tests/aignostics/platform/resources/runs_test.py +++ b/tests/aignostics/platform/resources/runs_test.py @@ -16,7 +16,11 @@ RunReadResponse, ) -from aignostics.platform.resources.runs import Run, Runs +from aignostics.platform.resources.runs import ( + LIST_APPLICATION_RUNS_MAX_PAGE_SIZE, + Run, + Runs, +) from aignostics.platform.resources.utils import PAGE_SIZE @@ -68,24 +72,25 @@ def test_runs_list_with_pagination(runs, mock_api) -> None: mock_api: Mock ExternalsApi instance. """ # Arrange - page1 = [Mock(spec=RunReadResponse, run_id=f"run-{i}") for i in range(PAGE_SIZE)] - page2 = [Mock(spec=RunReadResponse, run_id=f"run-{i + PAGE_SIZE}") for i in range(5)] + # Since list() now uses LIST_APPLICATION_RUNS_MAX_PAGE_SIZE, adjust expectations + page1 = [Mock(spec=RunReadResponse, run_id=f"run-{i}") for i in range(LIST_APPLICATION_RUNS_MAX_PAGE_SIZE)] + page2 = [Mock(spec=RunReadResponse, run_id=f"run-{i + LIST_APPLICATION_RUNS_MAX_PAGE_SIZE}") for i in range(5)] mock_api.list_runs_v1_runs_get.side_effect = [page1, page2] # Act result = list(runs.list()) # Assert - assert len(result) == PAGE_SIZE + 5 + assert len(result) == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE + 5 assert all(isinstance(run, Run) for run in result) assert mock_api.list_runs_v1_runs_get.call_count == 2 # Check that the calls were made with the expected parameters (ignoring _request_timeout and _headers) assert mock_api.list_runs_v1_runs_get.call_args_list[0][1]["page"] == 1 - assert mock_api.list_runs_v1_runs_get.call_args_list[0][1]["page_size"] == PAGE_SIZE + assert mock_api.list_runs_v1_runs_get.call_args_list[0][1]["page_size"] == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE assert mock_api.list_runs_v1_runs_get.call_args_list[0][1]["application_id"] is None assert mock_api.list_runs_v1_runs_get.call_args_list[0][1]["application_version"] is None assert mock_api.list_runs_v1_runs_get.call_args_list[1][1]["page"] == 2 - assert mock_api.list_runs_v1_runs_get.call_args_list[1][1]["page_size"] == PAGE_SIZE + assert mock_api.list_runs_v1_runs_get.call_args_list[1][1]["page_size"] == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE assert mock_api.list_runs_v1_runs_get.call_args_list[1][1]["application_id"] is None assert mock_api.list_runs_v1_runs_get.call_args_list[1][1]["application_version"] is None @@ -115,7 +120,7 @@ def test_runs_list_with_application_version_filter(runs, mock_api) -> None: assert call_kwargs["application_id"] == app_id assert call_kwargs["application_version"] == app_version assert call_kwargs["page"] == 1 - assert call_kwargs["page_size"] == PAGE_SIZE + assert call_kwargs["page_size"] == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE @pytest.mark.unit @@ -234,7 +239,7 @@ def test_paginate_with_not_found_exception_on_first_page(runs, mock_api) -> None mock_api.list_runs_v1_runs_get.assert_called_once() call_kwargs = mock_api.list_runs_v1_runs_get.call_args[1] assert call_kwargs["page"] == 1 - assert call_kwargs["page_size"] == PAGE_SIZE + assert call_kwargs["page_size"] == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE assert call_kwargs["application_id"] is None assert call_kwargs["application_version"] is None @@ -243,7 +248,7 @@ def test_paginate_with_not_found_exception_on_first_page(runs, mock_api) -> None def test_paginate_with_not_found_exception_after_full_page(runs, mock_api) -> None: """Test that paginate handles NotFoundException after a full page. - This test verifies that when we get exactly PAGE_SIZE items on the first page + This test verifies that when we get exactly LIST_APPLICATION_RUNS_MAX_PAGE_SIZE items on the first page and then a NotFoundException on the second page, we correctly return just the first page's items. @@ -254,22 +259,319 @@ def test_paginate_with_not_found_exception_after_full_page(runs, mock_api) -> No # Arrange from aignx.codegen.exceptions import NotFoundException - # Return exactly PAGE_SIZE items for first page, then throw NotFoundException - full_page = [Mock(spec=RunReadResponse, run_id=f"run-{i}") for i in range(PAGE_SIZE)] + # Return exactly LIST_APPLICATION_RUNS_MAX_PAGE_SIZE items for first page, then throw NotFoundException + full_page = [Mock(spec=RunReadResponse, run_id=f"run-{i}") for i in range(LIST_APPLICATION_RUNS_MAX_PAGE_SIZE)] mock_api.list_runs_v1_runs_get.side_effect = [full_page, NotFoundException()] # Act result = list(runs.list()) # Assert - assert len(result) == PAGE_SIZE + assert len(result) == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE assert mock_api.list_runs_v1_runs_get.call_count == 2 # Check that the calls were made with the expected parameters (ignoring _request_timeout and _headers) assert mock_api.list_runs_v1_runs_get.call_args_list[0][1]["page"] == 1 - assert mock_api.list_runs_v1_runs_get.call_args_list[0][1]["page_size"] == PAGE_SIZE + assert mock_api.list_runs_v1_runs_get.call_args_list[0][1]["page_size"] == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE assert mock_api.list_runs_v1_runs_get.call_args_list[0][1]["application_id"] is None assert mock_api.list_runs_v1_runs_get.call_args_list[0][1]["application_version"] is None assert mock_api.list_runs_v1_runs_get.call_args_list[1][1]["page"] == 2 - assert mock_api.list_runs_v1_runs_get.call_args_list[1][1]["page_size"] == PAGE_SIZE + assert mock_api.list_runs_v1_runs_get.call_args_list[1][1]["page_size"] == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE assert mock_api.list_runs_v1_runs_get.call_args_list[1][1]["application_id"] is None assert mock_api.list_runs_v1_runs_get.call_args_list[1][1]["application_version"] is None + + +@pytest.mark.unit +def test_runs_list_with_external_id_filter(runs, mock_api) -> None: + """Test that Runs.list() correctly filters by external ID. + + This test verifies that the external_id filter parameter is + correctly passed to the API client. + + Args: + runs: Runs instance with mock API. + mock_api: Mock ExternalsApi instance. + """ + # Arrange + external_id = "test-external-id" + mock_api.list_runs_v1_runs_get.return_value = [] + + # Act + list(runs.list(external_id=external_id)) + + # Assert + mock_api.list_runs_v1_runs_get.assert_called_once() + call_kwargs = mock_api.list_runs_v1_runs_get.call_args[1] + assert call_kwargs["external_id"] == external_id + assert call_kwargs["page"] == 1 + assert call_kwargs["page_size"] == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE + + +@pytest.mark.unit +def test_runs_list_with_custom_metadata_filter(runs, mock_api) -> None: + """Test that Runs.list() correctly filters by custom metadata. + + This test verifies that the custom_metadata filter parameter in JSONPath format + is correctly passed to the API client. + + Args: + runs: Runs instance with mock API. + mock_api: Mock ExternalsApi instance. + """ + # Arrange + custom_metadata = "$.experiment_id=='exp-123'" + mock_api.list_runs_v1_runs_get.return_value = [] + + # Act + list(runs.list(custom_metadata=custom_metadata)) + + # Assert + mock_api.list_runs_v1_runs_get.assert_called_once() + call_kwargs = mock_api.list_runs_v1_runs_get.call_args[1] + assert call_kwargs["custom_metadata"] == custom_metadata + assert call_kwargs["page"] == 1 + assert call_kwargs["page_size"] == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE + + +@pytest.mark.unit +def test_runs_list_with_sort_ascending(runs, mock_api) -> None: + """Test that Runs.list() correctly applies ascending sort. + + This test verifies that the sort parameter for ascending order + is correctly passed to the API client as a list. + + Args: + runs: Runs instance with mock API. + mock_api: Mock ExternalsApi instance. + """ + # Arrange + sort_field = "created_at" + mock_api.list_runs_v1_runs_get.return_value = [] + + # Act + list(runs.list(sort=sort_field)) + + # Assert + mock_api.list_runs_v1_runs_get.assert_called_once() + call_kwargs = mock_api.list_runs_v1_runs_get.call_args[1] + assert call_kwargs["sort"] == [sort_field] + assert call_kwargs["page"] == 1 + assert call_kwargs["page_size"] == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE + + +@pytest.mark.unit +def test_runs_list_with_descending_sort(runs, mock_api) -> None: + """Test that Runs.list() correctly applies descending sort. + + This test verifies that the sort parameter with '-' prefix for descending order + is correctly passed to the API client as a list. + + Args: + runs: Runs instance with mock API. + mock_api: Mock ExternalsApi instance. + """ + # Arrange + sort_field = "-created_at" + mock_api.list_runs_v1_runs_get.return_value = [] + + # Act + list(runs.list(sort=sort_field)) + + # Assert + mock_api.list_runs_v1_runs_get.assert_called_once() + call_kwargs = mock_api.list_runs_v1_runs_get.call_args[1] + assert call_kwargs["sort"] == [sort_field] + assert call_kwargs["page"] == 1 + assert call_kwargs["page_size"] == LIST_APPLICATION_RUNS_MAX_PAGE_SIZE + + +@pytest.mark.unit +def test_runs_list_with_custom_page_size(runs, mock_api) -> None: + """Test that Runs.list() correctly uses custom page size. + + This test verifies that a custom page_size parameter is correctly + passed to the paginate function and API client. + + Args: + runs: Runs instance with mock API. + mock_api: Mock ExternalsApi instance. + """ + # Arrange + custom_page_size = 50 + mock_api.list_runs_v1_runs_get.return_value = [] + + # Act + list(runs.list(page_size=custom_page_size)) + + # Assert + mock_api.list_runs_v1_runs_get.assert_called_once() + call_kwargs = mock_api.list_runs_v1_runs_get.call_args[1] + assert call_kwargs["page_size"] == custom_page_size + assert call_kwargs["page"] == 1 + + +@pytest.mark.unit +def test_runs_list_with_page_size_exceeding_max_raises_error(runs, mock_api) -> None: + """Test that Runs.list() raises ValueError when page_size exceeds maximum. + + This test verifies that attempting to use a page_size greater than the + maximum allowed value (100) raises a ValueError. + + Args: + runs: Runs instance with mock API. + mock_api: Mock ExternalsApi instance. + """ + # Arrange + invalid_page_size = 101 + + # Act & Assert + with pytest.raises(ValueError, match="page_size is must be less than or equal to 100"): + list(runs.list(page_size=invalid_page_size)) + + # Verify API was never called + mock_api.list_runs_v1_runs_get.assert_not_called() + + +@pytest.mark.unit +def test_runs_list_with_all_filters_combined(runs, mock_api) -> None: + """Test that Runs.list() correctly combines all filter parameters. + + This test verifies that all filter parameters (application_id, application_version, + external_id, custom_metadata, sort, page_size) work together correctly. + + Args: + runs: Runs instance with mock API. + mock_api: Mock ExternalsApi instance. + """ + # Arrange + app_id = "test-app" + app_version = "1.0.0" + external_id = "ext-123" + custom_metadata = "$.experiment=='test'" + sort_field = "-created_at" + page_size = 25 + mock_api.list_runs_v1_runs_get.return_value = [] + + # Act + list( + runs.list( + application_id=app_id, + application_version=app_version, + external_id=external_id, + custom_metadata=custom_metadata, + sort=sort_field, + page_size=page_size, + ) + ) + + # Assert + mock_api.list_runs_v1_runs_get.assert_called_once() + call_kwargs = mock_api.list_runs_v1_runs_get.call_args[1] + assert call_kwargs["application_id"] == app_id + assert call_kwargs["application_version"] == app_version + assert call_kwargs["external_id"] == external_id + assert call_kwargs["custom_metadata"] == custom_metadata + assert call_kwargs["sort"] == [sort_field] + assert call_kwargs["page_size"] == page_size + assert call_kwargs["page"] == 1 + + +@pytest.mark.unit +def test_runs_list_with_nocache_true(runs, mock_api) -> None: + """Test that Runs.list() respects the nocache parameter. + + This test verifies that the nocache parameter is correctly passed through + to list_data (nocache is handled by the caching decorator, not passed to API). + + Args: + runs: Runs instance with mock API. + mock_api: Mock ExternalsApi instance. + """ + # Arrange + mock_api.list_runs_v1_runs_get.return_value = [] + + # Act + list(runs.list(nocache=True)) + + # Assert + mock_api.list_runs_v1_runs_get.assert_called_once() + # nocache is handled by caching decorator, not passed to API + call_kwargs = mock_api.list_runs_v1_runs_get.call_args[1] + assert "nocache" not in call_kwargs + + +@pytest.mark.unit +def test_runs_list_with_none_sort_not_passed_as_list(runs, mock_api) -> None: + """Test that Runs.list() doesn't wrap None sort in a list. + + This test verifies that when sort is None, it's passed as None + rather than [None] to the API. + + Args: + runs: Runs instance with mock API. + mock_api: Mock ExternalsApi instance. + """ + # Arrange + mock_api.list_runs_v1_runs_get.return_value = [] + + # Act + list(runs.list(sort=None)) + + # Assert + mock_api.list_runs_v1_runs_get.assert_called_once() + call_kwargs = mock_api.list_runs_v1_runs_get.call_args[1] + assert call_kwargs["sort"] is None + + +@pytest.mark.unit +def test_runs_list_delegates_to_list_data(runs, mock_api) -> None: + """Test that Runs.list() correctly delegates to list_data() and wraps results. + + This test verifies that list() calls list_data() with all parameters + and converts RunData objects to Run objects. + + Args: + runs: Runs instance with mock API. + mock_api: Mock ExternalsApi instance. + """ + # Arrange + app_id = "test-app" + app_version = "1.0.0" + external_id = "ext-123" + custom_metadata = "$.test=='value'" + sort_field = "-created_at" + page_size = 25 + + # Create mock RunData responses + mock_responses = [Mock(spec=RunReadResponse, run_id=f"run-{i}") for i in range(3)] + mock_api.list_runs_v1_runs_get.return_value = mock_responses + + # Act + result = list( + runs.list( + application_id=app_id, + application_version=app_version, + external_id=external_id, + custom_metadata=custom_metadata, + sort=sort_field, + page_size=page_size, + nocache=True, + ) + ) + + # Assert + # Verify we got Run objects with correct run_ids + assert len(result) == 3 + assert all(isinstance(run, Run) for run in result) + assert [run.run_id for run in result] == ["run-0", "run-1", "run-2"] + + # Verify all parameters were passed to the API + mock_api.list_runs_v1_runs_get.assert_called_once() + call_kwargs = mock_api.list_runs_v1_runs_get.call_args[1] + assert call_kwargs["application_id"] == app_id + assert call_kwargs["application_version"] == app_version + assert call_kwargs["external_id"] == external_id + assert call_kwargs["custom_metadata"] == custom_metadata + assert call_kwargs["sort"] == [sort_field] + assert call_kwargs["page_size"] == page_size + # nocache is handled by caching decorator, not passed to API + assert "nocache" not in call_kwargs diff --git a/tests/aignostics/platform/sdk_metadata_test.py b/tests/aignostics/platform/sdk_metadata_test.py index ed35b1231..45064ee34 100644 --- a/tests/aignostics/platform/sdk_metadata_test.py +++ b/tests/aignostics/platform/sdk_metadata_test.py @@ -505,6 +505,127 @@ def test_validate_extra_fields_rejected() -> None: with pytest.raises(ValidationError): validate_run_sdk_metadata(metadata) + @pytest.mark.unit + @staticmethod + def test_validate_with_tags_set() -> None: + """Test validation with tags as a set of strings.""" + metadata = { + "schema_version": SDK_METADATA_SCHEMA_VERSION, + "submission": { + "date": "2025-10-19T12:00:00+00:00", + "interface": "script", + "initiator": "user", + }, + "user_agent": "test-agent/1.0", + "tags": {"experiment", "production", "v2"}, + } + + assert validate_run_sdk_metadata(metadata) is True + + @pytest.mark.unit + @staticmethod + def test_validate_with_empty_tags_set() -> None: + """Test validation with empty tags set.""" + metadata = { + "schema_version": SDK_METADATA_SCHEMA_VERSION, + "submission": { + "date": "2025-10-19T12:00:00+00:00", + "interface": "script", + "initiator": "user", + }, + "user_agent": "test-agent/1.0", + "tags": set(), + } + + assert validate_run_sdk_metadata(metadata) is True + + @pytest.mark.unit + @staticmethod + def test_validate_with_tags_none() -> None: + """Test validation with tags as None.""" + metadata = { + "schema_version": SDK_METADATA_SCHEMA_VERSION, + "submission": { + "date": "2025-10-19T12:00:00+00:00", + "interface": "script", + "initiator": "user", + }, + "user_agent": "test-agent/1.0", + "tags": None, + } + + assert validate_run_sdk_metadata(metadata) is True + + @pytest.mark.unit + @staticmethod + def test_validate_without_tags_field() -> None: + """Test validation when tags field is omitted entirely.""" + metadata = { + "schema_version": SDK_METADATA_SCHEMA_VERSION, + "submission": { + "date": "2025-10-19T12:00:00+00:00", + "interface": "script", + "initiator": "user", + }, + "user_agent": "test-agent/1.0", + } + + assert validate_run_sdk_metadata(metadata) is True + + @pytest.mark.unit + @staticmethod + def test_validate_with_tags_list_converted_to_set() -> None: + """Test that list is automatically converted to set by Pydantic.""" + metadata = { + "schema_version": SDK_METADATA_SCHEMA_VERSION, + "submission": { + "date": "2025-10-19T12:00:00+00:00", + "interface": "script", + "initiator": "user", + }, + "user_agent": "test-agent/1.0", + "tags": ["tag1", "tag2"], # List gets converted to set + } + + # Validation should succeed as Pydantic converts list to set + assert validate_run_sdk_metadata(metadata) is True + + @pytest.mark.unit + @staticmethod + def test_validate_with_tags_invalid_type_dict() -> None: + """Test validation fails when tags is a dict instead of set.""" + metadata = { + "schema_version": SDK_METADATA_SCHEMA_VERSION, + "submission": { + "date": "2025-10-19T12:00:00+00:00", + "interface": "script", + "initiator": "user", + }, + "user_agent": "test-agent/1.0", + "tags": {"key": "value"}, # Dict instead of set + } + + with pytest.raises(ValidationError): + validate_run_sdk_metadata(metadata) + + @pytest.mark.unit + @staticmethod + def test_validate_with_tags_non_string_values() -> None: + """Test validation fails when tags contains non-string values.""" + metadata = { + "schema_version": SDK_METADATA_SCHEMA_VERSION, + "submission": { + "date": "2025-10-19T12:00:00+00:00", + "interface": "script", + "initiator": "user", + }, + "user_agent": "test-agent/1.0", + "tags": {"valid", 123, None}, # Mixed types + } + + with pytest.raises(ValidationError): + validate_run_sdk_metadata(metadata) + @pytest.mark.unit @staticmethod def test_validate_sdk_metadata_silent_valid(clean_env: None) -> None: @@ -664,3 +785,70 @@ def test_get_item_json_schema() -> None: assert "schema_version" in schema["properties"] assert "required" in schema assert "schema_version" in schema["required"] + + @pytest.mark.unit + @staticmethod + def test_validate_item_metadata_with_tags() -> None: + """Test validation of item metadata with tags as a set of strings.""" + metadata = { + "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "tags": {"slide", "tumor", "he-stained"}, + } + + assert validate_item_sdk_metadata(metadata) is True + + @pytest.mark.unit + @staticmethod + def test_validate_item_metadata_with_empty_tags() -> None: + """Test validation of item metadata with empty tags set.""" + metadata = { + "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "tags": set(), + } + + assert validate_item_sdk_metadata(metadata) is True + + @pytest.mark.unit + @staticmethod + def test_validate_item_metadata_with_tags_none() -> None: + """Test validation of item metadata with tags as None.""" + metadata = { + "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "tags": None, + } + + assert validate_item_sdk_metadata(metadata) is True + + @pytest.mark.unit + @staticmethod + def test_validate_item_metadata_without_tags() -> None: + """Test validation of item metadata when tags field is omitted.""" + metadata = { + "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + } + + assert validate_item_sdk_metadata(metadata) is True + + @pytest.mark.unit + @staticmethod + def test_validate_item_metadata_tags_list_converted() -> None: + """Test that list is automatically converted to set by Pydantic.""" + metadata = { + "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "tags": ["tag1", "tag2"], # List gets converted to set + } + + # Validation should succeed as Pydantic converts list to set + assert validate_item_sdk_metadata(metadata) is True + + @pytest.mark.unit + @staticmethod + def test_validate_item_metadata_tags_non_string() -> None: + """Test validation fails when tags contains non-string values.""" + metadata = { + "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "tags": {"valid", 123}, # Mixed types + } + + with pytest.raises(ValidationError): + validate_item_sdk_metadata(metadata) diff --git a/tests/aignostics/platform/utils_test.py b/tests/aignostics/platform/utils_test.py index 00b29c5a7..2bd23037c 100644 --- a/tests/aignostics/platform/utils_test.py +++ b/tests/aignostics/platform/utils_test.py @@ -1,8 +1,162 @@ """Tests for the platform utility functions.""" +import math + import pytest from aignostics.platform import mime_type_to_file_ending +from aignostics.platform._utils import convert_to_json_serializable + + +class TestConvertToJsonSerializable: + """Tests for the convert_to_json_serializable function.""" + + @pytest.mark.unit + @staticmethod + def test_convert_simple_set_to_list() -> None: + """Test that a set is converted to a sorted list. + + This test verifies that the convert_to_json_serializable function correctly + converts a set to a sorted list for JSON serialization. + """ + result = convert_to_json_serializable({"a", "c", "b"}) + assert result == ["a", "b", "c"] + + @pytest.mark.unit + @staticmethod + def test_convert_numeric_set_to_list() -> None: + """Test that a numeric set is converted to a sorted list. + + This test verifies that the convert_to_json_serializable function correctly + converts a numeric set to a sorted list for JSON serialization. + """ + result = convert_to_json_serializable({3, 1, 2}) + assert result == [1, 2, 3] + + @pytest.mark.unit + @staticmethod + def test_convert_dict_with_set_values() -> None: + """Test that a dictionary with set values has them converted to lists. + + This test verifies that the convert_to_json_serializable function recursively + converts set values within dictionaries to sorted lists. + """ + result = convert_to_json_serializable({"tags": {"test", "prod", "dev"}}) + assert result == {"tags": ["dev", "prod", "test"]} + + @pytest.mark.unit + @staticmethod + def test_convert_nested_dict_with_sets() -> None: + """Test that nested dictionaries with sets are fully converted. + + This test verifies that the convert_to_json_serializable function recursively + processes nested structures containing sets. + """ + input_data = { + "outer": { + "inner": {"items": {5, 3, 1}}, + "tags": {"z", "a"}, + } + } + expected = { + "outer": { + "inner": {"items": [1, 3, 5]}, + "tags": ["a", "z"], + } + } + result = convert_to_json_serializable(input_data) + assert result == expected + + @pytest.mark.unit + @staticmethod + def test_convert_list_with_sets() -> None: + """Test that lists containing sets have them converted to lists. + + This test verifies that the convert_to_json_serializable function recursively + processes lists containing sets. + """ + result = convert_to_json_serializable([{"a", "b"}, {"c", "d"}]) + assert result == [["a", "b"], ["c", "d"]] + + @pytest.mark.unit + @staticmethod + def test_convert_tuple_with_sets() -> None: + """Test that tuples containing sets have them converted to lists. + + This test verifies that the convert_to_json_serializable function recursively + processes tuples containing sets and converts the tuple itself to a list. + """ + result = convert_to_json_serializable(({"x", "y"}, {"z"})) + assert result == [["x", "y"], ["z"]] + + @pytest.mark.unit + @staticmethod + def test_convert_mixed_types_unchanged() -> None: + """Test that JSON-serializable types remain unchanged. + + This test verifies that the convert_to_json_serializable function does not + modify types that are already JSON-serializable (str, int, bool, None). + """ + input_data = { + "string": "test", + "number": 42, + "float": math.pi, + "boolean": True, + "null": None, + "list": [1, 2, 3], + } + result = convert_to_json_serializable(input_data) + assert result == input_data + + @pytest.mark.unit + @staticmethod + def test_convert_empty_set() -> None: + """Test that an empty set is converted to an empty list. + + This test verifies that the convert_to_json_serializable function correctly + handles empty sets. + """ + result = convert_to_json_serializable(set()) + assert result == [] + + @pytest.mark.unit + @staticmethod + def test_convert_complex_nested_structure() -> None: + """Test conversion of a complex nested structure with multiple sets. + + This test verifies that the convert_to_json_serializable function can handle + deeply nested structures with sets at various levels. + """ + input_data = { + "sdk": { + "tags": {"test_1", "test_2"}, + "metadata": { + "nested": [ + {"items": {1, 2}}, + {"values": {"a", "b"}}, + ] + }, + }, + "user": { + "groups": {"admin", "user"}, + }, + } + expected = { + "sdk": { + "tags": ["test_1", "test_2"], + "metadata": { + "nested": [ + {"items": [1, 2]}, + {"values": ["a", "b"]}, + ] + }, + }, + "user": { + "groups": ["admin", "user"], + }, + } + result = convert_to_json_serializable(input_data) + assert result == expected class TestMimeTypeToFileEnding: diff --git a/tests/aignostics/qupath/gui_test.py b/tests/aignostics/qupath/gui_test.py index 835a47789..0f28b9f60 100644 --- a/tests/aignostics/qupath/gui_test.py +++ b/tests/aignostics/qupath/gui_test.py @@ -295,7 +295,7 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 pytest.fail(f"Failed to kill QuPath process: {e}") # Step 7: Inspect QuPath results - result = runner.invoke(cli, ["qupath", "inspect", str(run_dir / "qupath")]) + result = runner.invoke(cli, ["qupath", "inspect", str(qupath_dir)]) output = normalize_output(result.output, strip_ansi=True) print(repr(output)) diff --git a/tests/constants_test.py b/tests/constants_test.py index bdee47f36..cda78d497 100644 --- a/tests/constants_test.py +++ b/tests/constants_test.py @@ -10,7 +10,7 @@ "gs://platform-api-application-test-data/heta/slides/8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" ) HETA_SINGLE_SPOT_FILENAME = "8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" -HETA_SINGLE_SPOT_FILESIZE = 14681750 # in bytes +HETA_SINGLE_SPOT_FILESIZE = 10562338 # in bytes HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES = [ ("tissue_segmentation_csv_class_information.csv", 342, 10), ("cell_classification_geojson_polygons.json", 16054058, 10), @@ -27,7 +27,7 @@ "gs://aignx-storage-service-dev/sample_data_formatted/9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" ) HETA_ANOTHER_SPOT_FILENAME = "9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" -HETA_ANOTHER_SPOT_FILESIZE = 10562338 # in bytes +HETA_ANOTHER_SPOT_FILESIZE = 14681750 # in bytes HETA_ANOTHER_SPOT_EXPECTED_RESULT_FILES = [ ("tissue_segmentation_csv_class_information.csv", 361, 10), ("cell_classification_geojson_polygons.json", 9915953, 10), From da289111ff1eccd968389f37c611d08702514a65 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 13:57:44 +0100 Subject: [PATCH 07/16] testing ... --- src/aignostics/application/_cli.py | 4 +- src/aignostics/application/_gui/_frame.py | 26 ++- .../_gui/_page_application_describe.py | 11 +- .../_gui/_page_application_run_describe.py | 6 + src/aignostics/application/_service.py | 88 ++++++++- tests/aignostics/application/cli_test.py | 51 +++++- tests/aignostics/application/service_test.py | 169 +++++++++++++++++- tests/aignostics/platform/e2e_test.py | 15 +- 8 files changed, 342 insertions(+), 28 deletions(-) diff --git a/src/aignostics/application/_cli.py b/src/aignostics/application/_cli.py index 501c03dd1..d7120cb26 100644 --- a/src/aignostics/application/_cli.py +++ b/src/aignostics/application/_cli.py @@ -742,7 +742,7 @@ def run_submit( # noqa: PLR0913, PLR0917 @run_app.command("list") -def run_list( +def run_list( # noqa: PLR0913, PLR0917 verbose: Annotated[bool, typer.Option(help="Show application details")] = False, limit: Annotated[int | None, typer.Option(help="Maximum number of runs to display")] = None, tags: Annotated[ @@ -753,6 +753,7 @@ def run_list( str | None, typer.Option(help="Optional regex pattern to filter runs by note metadata."), ] = None, + query: Annotated[str | None, typer.Option(help="Optional query string to filter runs by note OR tags.")] = None, note_case_insensitive: Annotated[bool, typer.Option(help="Make note regex search case-insensitive.")] = True, ) -> None: """List runs.""" @@ -762,6 +763,7 @@ def run_list( tags={tag.strip() for tag in tags.split(",") if tag.strip()} if tags else None, note_regex=note_regex, note_query_case_insensitive=note_case_insensitive, + query=query, ) if len(runs) == 0: if tags: diff --git a/src/aignostics/application/_gui/_frame.py b/src/aignostics/application/_gui/_frame.py index 2cd6127b5..11c02ed0b 100644 --- a/src/aignostics/application/_gui/_frame.py +++ b/src/aignostics/application/_gui/_frame.py @@ -90,8 +90,8 @@ async def _frame( # noqa: C901, PLR0913, PLR0915, PLR0917 ui.label(f"Could not load applications: {e!s}").mark("LABEL_ERROR") logger.exception("Could not load applications") - async def application_runs_load_and_render( - runs_column: ui.column, has_output: bool = False, note_query: str | None = None + async def application_runs_load_and_render( # noqa: C901 + runs_column: ui.column, has_output: bool = False, query: str | None = None ) -> None: global _runs_last_refresh_time # noqa: PLW0603 @@ -104,8 +104,7 @@ async def application_runs_load_and_render( Service.application_runs_static, limit=RUNS_LIMIT, has_output=has_output, - note_regex=f".*{note_query}.*" if note_query else None, - note_query_case_insensitive=True, + query=query, ) if runs is None: message = ( # type: ignore[unreachable] @@ -182,6 +181,15 @@ async def application_runs_load_and_render( else "font-normal" ).mark(f"LABEL_RUN_APPLICATION:{index}") ui.label(f"submitted {run_data['submitted_at'].astimezone().strftime('%m-%d %H:%M')}") + # if there is tags, show them as immutable chips + # only show the first 3, than "... and tooltip with the rest" + # chips must be very small, white background, black text, outlined, disabled + if run_data.get("tags") and len(run_data["tags"]): + with ui.row().classes("gap-1 mt-1"): + for tag in run_data["tags"][:3]: + ui.chip(tag).props("disabled").classes("bg-white text-black text-xs") + if len(run_data["tags"]) > 3: # noqa: PLR2004 + ui.tooltip(f"Tags: {', '.join(run_data['tags'][3:])}") if not runs: with ui.item(): with ui.item_section().props("avatar"): @@ -204,7 +212,7 @@ async def _runs_list() -> None: ui.scroll_area() .props('id="runs-list-container"') .classes("w-full") - .style("height: calc(100vh - 300px);") + .style("height: calc(100vh - 250px);") .props("content-style='padding-right: 0;'"), ui.column().classes("full-width justify-center") as runs_column, ): @@ -215,7 +223,7 @@ async def _runs_list() -> None: coroutine=application_runs_load_and_render( runs_column=runs_column, has_output=app.storage.tab.get(STORAGE_TAB_RUNS_HAS_OUTPUT, False), - note_query=search_input.query, + query=search_input.query, ), name="_runs_list", ) @@ -247,11 +255,11 @@ def is_active(self) -> bool: ui.item_label("Runs").props("header") await ui.context.client.connected() ui.input( - placeholder="Filter by note", + placeholder="Filter by note or tags", on_change=_runs_list.refresh, ).bind_value(search_input, "query").props("rounded outlined dense clearable").style( - "max-width: 15ch;" - ).classes("text-xs").mark("INPUT_RUNS_FILTER_NOTE") + "max-width: 16ch;" + ).classes("text-xs").mark("INPUT_RUNS_FILTER_NOTE_OR_TAGS") with RunFilterButton("done_all", size="sm").classes("mr-3").mark("BUTTON_RUNS_FILTER_COMPLETED"): ui.tooltip("Show completed runs only") ui.separator() diff --git a/src/aignostics/application/_gui/_page_application_describe.py b/src/aignostics/application/_gui/_page_application_describe.py index 54fde70be..9a33a3642 100644 --- a/src/aignostics/application/_gui/_page_application_describe.py +++ b/src/aignostics/application/_gui/_page_application_describe.py @@ -46,6 +46,7 @@ class SubmitForm: metadata_next_button: ui.button | None = None upload_and_submit_button: ui.button | None = None note: str | None = None + tags: list[str] | None = None due_date: str = (datetime.now().astimezone() + timedelta(hours=6)).strftime("%Y-%m-%d %H:%M") deadline: str = (datetime.now().astimezone() + timedelta(hours=24)).strftime("%Y-%m-%d %H:%M") validate_only: bool = False @@ -591,7 +592,7 @@ class ThumbnailRenderer { submit_form.metadata_exclude_button.disable() ui.button("Back", on_click=stepper.previous).props("flat") - with ui.step("Leave Note and Tags"): + with ui.step("Notes and Tags"): with ui.column(align_items="start").classes("w-full"): ui.textarea( label="Note (optional)", @@ -602,6 +603,13 @@ class ThumbnailRenderer { ), ).bind_value(submit_form, "note").mark("TEXTAREA_NOTE").classes("full-width") + ui.input_chips( + "Tags (optional, press Enter to add)", + value=submit_form.tags, + new_value_mode="add-unique", + clearable=True, + ).bind_value(submit_form, "tags").classes("full-width").mark("INPUT_TAGS") + with ui.stepper_navigation(): ui.button("Next", on_click=stepper.next).mark("BUTTON_NOTES_NEXT") ui.button("Back", on_click=stepper.previous).props("flat") @@ -701,6 +709,7 @@ def _submit() -> None: application_version=str(submit_form.application_version), custom_metadata=None, # TODO(Helmut): Allow user to edit custom metadata note=submit_form.note, + tags=set(submit_form.tags) if submit_form.tags else None, due_date=datetime.strptime(submit_form.due_date, "%Y-%m-%d %H:%M") .astimezone() .astimezone(UTC) diff --git a/src/aignostics/application/_gui/_page_application_run_describe.py b/src/aignostics/application/_gui/_page_application_run_describe.py index 9a69fa5d8..b51dbdadc 100644 --- a/src/aignostics/application/_gui/_page_application_run_describe.py +++ b/src/aignostics/application/_gui/_page_application_run_describe.py @@ -609,6 +609,12 @@ def open_marimo(results_folder: Path, button: ui.button | None = None) -> None: ui.label("Note:").classes("text-italic text-sm text-gray-500") ui.label(str(note)).classes("-mt-4") + tags = run_data.custom_metadata.get("sdk", {}).get("tags") if run_data.custom_metadata else None + if tags and len(tags): + with ui.row().classes("gap-1 -mt-2 full-width"): + for tag in tags[:20]: + ui.chip(tag).props("small outlined disabled").classes("bg-white text-black") + with ui.list().classes("full-width"): results = list(run.results()) if not results: diff --git a/src/aignostics/application/_service.py b/src/aignostics/application/_service.py index e431d120f..49743afab 100644 --- a/src/aignostics/application/_service.py +++ b/src/aignostics/application/_service.py @@ -531,6 +531,8 @@ def application_runs_static( # noqa: PLR0913, PLR0917 has_output: bool = False, note_regex: str | None = None, note_query_case_insensitive: bool = True, + tags: set[str] | None = None, + query: str | None = None, limit: int | None = None, ) -> list[dict[str, Any]]: """Get a list of all application runs, static variant. @@ -542,13 +544,20 @@ def application_runs_static( # noqa: PLR0913, PLR0917 external_id (str | None): The external ID to filter runs. If None, no filtering is applied. has_output (bool): If True, only runs with partial or full output are retrieved. note_regex (str | None): Optional regex to filter runs by note metadata. If None, no filtering is applied. + Cannot be used together with query parameter. note_query_case_insensitive (bool): If True, the note_regex is case insensitive. Default is True. + tags (set[str] | None): Optional set of tags to filter runs. All tags must match. + Cannot be used together with query parameter. + query (str | None): Optional string to filter runs by note OR tags (case insensitive partial match). + If None, no filtering is applied. Cannot be used together with custom_metadata, note_regex, or tags. + Performs a union search: matches runs where the query appears in the note OR matches any tag. limit (int | None): The maximum number of runs to retrieve. If None, all runs are retrieved. Returns: list[RunData]: A list of all application runs. Raises: + ValueError: If query is used together with custom_metadata, note_regex, or tags. RuntimeError: If the application run list cannot be retrieved. """ return [ @@ -562,6 +571,7 @@ def application_runs_static( # noqa: PLR0913, PLR0917 "termination_reason": run.termination_reason, "item_count": run.statistics.item_count, "item_succeeded_count": run.statistics.item_succeeded_count, + "tags": run.custom_metadata.get("sdk", {}).get("tags", []) if run.custom_metadata else [], } for run in Service().application_runs( application_id=application_id, @@ -570,11 +580,13 @@ def application_runs_static( # noqa: PLR0913, PLR0917 has_output=has_output, note_regex=note_regex, note_query_case_insensitive=note_query_case_insensitive, + tags=tags, + query=query, limit=limit, ) ] - def application_runs( # noqa: C901, PLR0912, PLR0913, PLR0917 + def application_runs( # noqa: C901, PLR0912, PLR0913, PLR0914, PLR0915, PLR0917 self, application_id: str | None = None, application_version: str | None = None, @@ -583,6 +595,7 @@ def application_runs( # noqa: C901, PLR0912, PLR0913, PLR0917 note_regex: str | None = None, note_query_case_insensitive: bool = True, tags: set[str] | None = None, + query: str | None = None, limit: int | None = None, ) -> list[RunData]: """Get a list of all application runs. @@ -594,22 +607,91 @@ def application_runs( # noqa: C901, PLR0912, PLR0913, PLR0917 external_id (str | None): The external ID to filter runs. If None, no filtering is applied. has_output (bool): If True, only runs with partial or full output are retrieved. note_regex (str | None): Optional regex to filter runs by note metadata. If None, no filtering is applied. + Cannot be used together with query parameter. note_query_case_insensitive (bool): If True, the note_regex is case insensitive. Default is True. tags (set[str] | None): Optional set of tags to filter runs. All tags must match. - If None, no filtering is applied. + If None, no filtering is applied. Cannot be used together with query parameter. + query (str | None): Optional string to filter runs by note OR tags (case insensitive partial match). + If None, no filtering is applied. Cannot be used together with custom_metadata, note_regex, or tags. + Performs a union search: matches runs where the query appears in the note OR matches any tag. limit (int | None): The maximum number of runs to retrieve. If None, all runs are retrieved. Returns: list[RunData]: A list of all application runs. Raises: + ValueError: If query is used together with custom_metadata, note_regex, or tags. RuntimeError: If the application run list cannot be retrieved. """ + # Validate that query is not used with other metadata filters + if query is not None: + if note_regex is not None: + message = "Cannot use 'query' parameter together with 'note_regex' parameter." + logger.warning(message) + raise ValueError(message) + if tags is not None: + message = "Cannot use 'query' parameter together with 'tags' parameter." + logger.warning(message) + raise ValueError(message) + if limit is not None and limit <= 0: return [] runs = [] page_size = LIST_APPLICATION_RUNS_MAX_PAGE_SIZE try: + # Handle query parameter with union semantics (note OR tags) + if query is not None: + # Search for runs matching query in notes + note_runs_dict: dict[str, RunData] = {} + flag_case_insensitive = ' flag "i"' + escaped_query = query.replace("\\", "\\\\").replace('"', '\\"') + custom_metadata_note = f'$.sdk.note ? (@ like_regex "{escaped_query}"{flag_case_insensitive})' + + note_run_iterator = self._get_platform_client().runs.list_data( + application_id=application_id, + application_version=application_version, + external_id=external_id, + custom_metadata=custom_metadata_note, + sort="-submitted_at", + page_size=page_size, + ) + for run in note_run_iterator: + if has_output and run.output == RunOutput.NONE: + continue + note_runs_dict[run.run_id] = run + if limit is not None and len(note_runs_dict) >= limit: + break + + # Search for runs matching query in tags + tag_runs_dict: dict[str, RunData] = {} + custom_metadata_tags = f'$.sdk.tags ? (@ like_regex "{escaped_query}"{flag_case_insensitive})' + + tag_run_iterator = self._get_platform_client().runs.list_data( + application_id=application_id, + application_version=application_version, + external_id=external_id, + custom_metadata=custom_metadata_tags, + sort="-submitted_at", + page_size=page_size, + ) + for run in tag_run_iterator: + if has_output and run.output == RunOutput.NONE: + continue + # Add to dict if not already present from note search + if run.run_id not in note_runs_dict: + tag_runs_dict[run.run_id] = run + if limit is not None and len(note_runs_dict) + len(tag_runs_dict) >= limit: + break + + # Union of results from both searches + runs = list(note_runs_dict.values()) + list(tag_runs_dict.values()) + + # Apply limit after union + if limit is not None and len(runs) > limit: + runs = runs[:limit] + + return runs + custom_metadata = None client_side_note_filter = None @@ -634,6 +716,7 @@ def application_runs( # noqa: C901, PLR0912, PLR0913, PLR0917 # Create regex pattern: ^(tag1|tag2|tag3)$ regex_pattern = "^(" + "|".join(escaped_tags) + ")$" custom_metadata = f'$.sdk.tags ? (@ like_regex "{regex_pattern}")' + run_iterator = self._get_platform_client().runs.list_data( application_id=application_id, application_version=application_version, @@ -645,7 +728,6 @@ def application_runs( # noqa: C901, PLR0912, PLR0913, PLR0917 for run in run_iterator: if has_output and run.output == RunOutput.NONE: continue - # Client-side filtering when combining multiple criteria # 1. If multiple tags specified, ensure ALL are present if tags and len(tags) > 1: diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py index 9e1ed13ee..489788a60 100644 --- a/tests/aignostics/application/cli_test.py +++ b/tests/aignostics/application/cli_test.py @@ -262,7 +262,7 @@ def test_cli_run_submit_and_describe_and_cancel_and_download_and_delete( # noqa HETA_APPLICATION_ID, str(csv_path), "--note", - "test_cli_run_submit_and_describe_and_cancel_and_download_and_delete", + "note_of_this_complex_test", "--tags", "cli-test,test_cli_run_submit_and_describe_and_cancel_and_download_and_delete,further-tag", "--deadline", @@ -282,6 +282,51 @@ def test_cli_run_submit_and_describe_and_cancel_and_download_and_delete( # noqa assert run_id_match, f"Failed to extract run ID from output '{output}'" run_id = run_id_match.group(1) + # Test that we can find this run by it's note via the query parameter + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--query", + "note_of_this_complex_test", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id in list_output, f"Run ID '{run_id}' not found when filtering by note via query" + + # Test that we can find this run by it's tag via the query parameter + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--query", + "test_cli_run_submit_and_describe_and_cancel_and_download_and_delete", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id in list_output, f"Run ID '{run_id}' not found when filtering by tag via query" + + # Test that we cannot find this run by another tag via the query parameter + list_result = runner.invoke( + cli, + [ + "application", + "run", + "list", + "--query", + "another_tag", + ], + ) + assert list_result.exit_code == 0 + list_output = normalize_output(list_result.stdout) + assert run_id not in list_output, f"Run ID '{run_id}' found when filtering by another tag via query" + # Test that we can find this run by it's note list_result = runner.invoke( cli, @@ -290,7 +335,7 @@ def test_cli_run_submit_and_describe_and_cancel_and_download_and_delete( # noqa "run", "list", "--note-regex", - "test_cli_run_submit_and_describe_and_cancel_and_download_and_delete", + "note_of_this_complex_test", ], ) assert list_result.exit_code == 0 @@ -395,7 +440,7 @@ def test_cli_run_submit_and_describe_and_cancel_and_download_and_delete( # noqa "run", "list", "--note-regex", - "test_cli_run_submit_and_describe_and_cancel_and_download_and_delete", + "note_of_this_complex_test", "--tags", "cli-test,test_cli_run_submit_and_describe_and_cancel_and_download_and_delete,further-tag", ], diff --git a/tests/aignostics/application/service_test.py b/tests/aignostics/application/service_test.py index f0bb71d94..9d7620192 100644 --- a/tests/aignostics/application/service_test.py +++ b/tests/aignostics/application/service_test.py @@ -1,13 +1,14 @@ """Tests to verify the service functionality of the application module.""" from datetime import UTC, datetime, timedelta +from unittest.mock import MagicMock, patch import pytest from typer.testing import CliRunner from aignostics.application import Service as ApplicationService from aignostics.application._utils import validate_due_date -from aignostics.platform import NotFoundException +from aignostics.platform import NotFoundException, RunData, RunOutput from tests.constants_test import HETA_APPLICATION_ID, HETA_APPLICATION_VERSION @@ -188,3 +189,169 @@ def test_application_versions_are_unique(runner: CliRunner) -> None: f"Application '{app.application_id}' has duplicate versions. " f"Found {len(version_numbers)} versions but only {len(unique_versions)} unique: {version_numbers}" ) + + +@pytest.mark.unit +def test_application_runs_query_with_note_regex_raises() -> None: + """Test that using query with note_regex raises ValueError.""" + service = ApplicationService() + + with pytest.raises(ValueError, match=r"Cannot use 'query' parameter together with 'note_regex' parameter"): + service.application_runs(query="test", note_regex="test.*") + + +@pytest.mark.unit +def test_application_runs_query_with_tags_raises() -> None: + """Test that using query with tags raises ValueError.""" + service = ApplicationService() + + with pytest.raises(ValueError, match=r"Cannot use 'query' parameter together with 'tags' parameter"): + service.application_runs(query="test", tags={"tag1", "tag2"}) + + +@pytest.mark.unit +@patch("aignostics.application._service.Service._get_platform_client") +def test_application_runs_query_searches_note_and_tags(mock_get_client: MagicMock) -> None: + """Test that query parameter searches both note and tags with union semantics.""" + # Create mock runs + run_from_note = MagicMock(spec=RunData) + run_from_note.run_id = "run-note-123" + run_from_note.output = RunOutput.FULL + + run_from_tag = MagicMock(spec=RunData) + run_from_tag.run_id = "run-tag-456" + run_from_tag.output = RunOutput.FULL + + run_from_both = MagicMock(spec=RunData) + run_from_both.run_id = "run-both-789" + run_from_both.output = RunOutput.FULL + + # Mock the platform client to return different runs for note and tag searches + mock_client = MagicMock() + mock_runs = MagicMock() + + # First call returns runs matching note, second call returns runs matching tags + mock_runs.list_data.side_effect = [ + iter([run_from_note, run_from_both]), # Note search results + iter([run_from_tag]), # Tag search results (run_from_both already in note results, so not added) + ] + + mock_client.runs = mock_runs + mock_get_client.return_value = mock_client + + service = ApplicationService() + results = service.application_runs(query="test") + + # Verify we got union of both searches (3 unique runs) + assert len(results) == 3 + assert run_from_note in results + assert run_from_tag in results + assert run_from_both in results + + # Verify that list_data was called twice (once for note, once for tags) + assert mock_runs.list_data.call_count == 2 + + # Verify the custom_metadata parameters contain the escaped query with case insensitive flag + calls = mock_runs.list_data.call_args_list + note_call_kwargs = calls[0][1] + tag_call_kwargs = calls[1][1] + + assert "custom_metadata" in note_call_kwargs + assert "$.sdk.note" in note_call_kwargs["custom_metadata"] + assert 'like_regex "test"' in note_call_kwargs["custom_metadata"] + assert 'flag "i"' in note_call_kwargs["custom_metadata"] + + assert "custom_metadata" in tag_call_kwargs + assert "$.sdk.tags" in tag_call_kwargs["custom_metadata"] + assert 'like_regex "test"' in tag_call_kwargs["custom_metadata"] + assert 'flag "i"' in tag_call_kwargs["custom_metadata"] + + +@pytest.mark.unit +@patch("aignostics.application._service.Service._get_platform_client") +def test_application_runs_query_deduplicates_results(mock_get_client: MagicMock) -> None: + """Test that query parameter deduplicates runs that match both note and tags.""" + # Create mock run that matches both searches + run_from_both = MagicMock(spec=RunData) + run_from_both.run_id = "run-both-123" + run_from_both.output = RunOutput.FULL + + # Mock the platform client to return the same run from both searches + mock_client = MagicMock() + mock_runs = MagicMock() + + # Both searches return the same run + mock_runs.list_data.side_effect = [ + iter([run_from_both]), # Note search results + iter([run_from_both]), # Tag search results (should be deduplicated) + ] + + mock_client.runs = mock_runs + mock_get_client.return_value = mock_client + + service = ApplicationService() + results = service.application_runs(query="test") + + # Verify we only got one run (deduplicated) + assert len(results) == 1 + assert results[0].run_id == "run-both-123" + + +@pytest.mark.unit +@patch("aignostics.application._service.Service._get_platform_client") +def test_application_runs_query_respects_limit(mock_get_client: MagicMock) -> None: + """Test that query parameter respects the limit parameter.""" + # Create mock runs + runs = [] + for i in range(10): + run = MagicMock(spec=RunData) + run.run_id = f"run-{i}" + run.output = RunOutput.FULL + runs.append(run) + + # Mock the platform client to return many runs + mock_client = MagicMock() + mock_runs = MagicMock() + + # Note search returns 5 runs, tag search returns 5 runs + mock_runs.list_data.side_effect = [ + iter(runs[:5]), # Note search results + iter(runs[5:]), # Tag search results + ] + + mock_client.runs = mock_runs + mock_get_client.return_value = mock_client + + service = ApplicationService() + results = service.application_runs(query="test", limit=3) + + # Verify we only got 3 runs despite having 10 total + assert len(results) == 3 + + +@pytest.mark.unit +@patch("aignostics.application._service.Service._get_platform_client") +def test_application_runs_query_escapes_special_characters(mock_get_client: MagicMock) -> None: + """Test that query parameter properly escapes special regex characters.""" + # Mock the platform client + mock_client = MagicMock() + mock_runs = MagicMock() + mock_runs.list_data.side_effect = [ + iter([]), # Note search results + iter([]), # Tag search results + ] + mock_client.runs = mock_runs + mock_get_client.return_value = mock_client + + service = ApplicationService() + # Use query with special characters that need escaping + service.application_runs(query='test"value\\path') + + # Verify the custom_metadata parameters contain properly escaped query + calls = mock_runs.list_data.call_args_list + note_call_kwargs = calls[0][1] + tag_call_kwargs = calls[1][1] + + # Check that double quotes and backslashes are properly escaped + assert 'test\\"value\\\\path' in note_call_kwargs["custom_metadata"] + assert 'test\\"value\\\\path' in tag_call_kwargs["custom_metadata"] diff --git a/tests/aignostics/platform/e2e_test.py b/tests/aignostics/platform/e2e_test.py index 48b639d16..925c8d230 100644 --- a/tests/aignostics/platform/e2e_test.py +++ b/tests/aignostics/platform/e2e_test.py @@ -157,6 +157,7 @@ def _submit_and_validate( # noqa: PLR0913, PLR0917 Raises: AssertionError: If any of the validation checks fail. + ValueError: If more than one tag is provided. """ client = platform.Client() run = client.runs.submit( @@ -181,19 +182,13 @@ def _submit_and_validate( # noqa: PLR0913, PLR0917 f"Unexpected run state `{details.state}` after submission" ) - # Build JSONPath filter for tags if provided - tags_filter = None - if tags: - # JSONPath filter to match all of the provided tags in the sdk.tags array - # PostgreSQL JSONPath: Use equality operator for exact match - for tag in tags: - tag_condition = f'$.sdk.tags ? (@ == "{tag}")' - tags_filter = f"({tags_filter}) && ({tag_condition})" if tags_filter else tag_condition - + if tags and len(tags) > 1: + message = "Only single tag filtering is supported in this test code." + raise ValueError(message) runs = client.runs.list( application_id=application_id, application_version=application_version, - custom_metadata=tags_filter, + custom_metadata=f'$.sdk.tags[*] ? (@ == "{tags[0]}")' if tags else None, ) # Find the submitted run in the list From 782cd76327692cc7717f4404a740968bafbabb29 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 14:29:53 +0100 Subject: [PATCH 08/16] testing ... --- tests/constants_test.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/constants_test.py b/tests/constants_test.py index cda78d497..7a373bd5c 100644 --- a/tests/constants_test.py +++ b/tests/constants_test.py @@ -12,15 +12,15 @@ HETA_SINGLE_SPOT_FILENAME = "8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" HETA_SINGLE_SPOT_FILESIZE = 10562338 # in bytes HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES = [ - ("tissue_segmentation_csv_class_information.csv", 342, 10), - ("cell_classification_geojson_polygons.json", 16054058, 10), - ("readout_generation_cell_readouts.csv", 2228907, 10), - ("tissue_qc_csv_class_information.csv", 232, 10), - ("tissue_segmentation_geojson_polygons.json", 270931, 10), - ("tissue_qc_geojson_polygons.json", 180522, 10), - ("tissue_qc_segmentation_map_image.tiff", 464908, 10), - ("readout_generation_slide_readouts.csv", 295268, 10), - ("tissue_segmentation_segmentation_map_image.tiff", 581258, 10), + ("tissue_qc_segmentation_map_image.tiff", 1698570, 10), + ("tissue_qc_geojson_polygons.json", 315019, 10), + ("tissue_segmentation_geojson_polygons.json", 927599, 10), + ("readout_generation_slide_readouts.csv", 299865, 10), + ("readout_generation_cell_readouts.csv", 1470036, 10), + ("cell_classification_geojson_polygons.json", 9915953, 10), + ("tissue_segmentation_segmentation_map_image.tiff", 2989980, 10), + ("tissue_segmentation_csv_class_information.csv", 361, 10), + ("tissue_qc_csv_class_information.csv", 236, 10), ] HETA_ANOTHER_SPOT_GS_URL = ( @@ -29,15 +29,15 @@ HETA_ANOTHER_SPOT_FILENAME = "9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" HETA_ANOTHER_SPOT_FILESIZE = 14681750 # in bytes HETA_ANOTHER_SPOT_EXPECTED_RESULT_FILES = [ - ("tissue_segmentation_csv_class_information.csv", 361, 10), - ("cell_classification_geojson_polygons.json", 9915953, 10), - ("readout_generation_cell_readouts.csv", 1470036, 10), - ("tissue_qc_csv_class_information.csv", 236, 10), - ("tissue_segmentation_geojson_polygons.json", 927599, 10), - ("tissue_qc_geojson_polygons.json", 315019, 10), - ("tissue_segmentation_segmentation_map_image.tiff", 2989980, 10), - ("readout_generation_slide_readouts.csv", 299865, 10), + ("tissue_qc_segmentation_map_image.tiff", 464908, 10), + ("tissue_qc_geojson_polygons.json", 180522, 10), + ("tissue_segmentation_geojson_polygons.json", 270931, 10), + ("readout_generation_slide_readouts.csv", 295268, 10), + ("readout_generation_cell_readouts.csv", 2228907, 10), + ("cell_classification_geojson_polygons.json", 16054058, 10), ("tissue_segmentation_segmentation_map_image.tiff", 581258, 10), + ("tissue_segmentation_csv_class_information.csv", 342, 10), + ("tissue_qc_csv_class_information.csv", 232, 10), ] TEST_THREE_SPOTS_GS_URLS = [ From 7e238a8cab91e873afd135853831a7b4abe7d28c Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 15:40:56 +0100 Subject: [PATCH 09/16] testing ... --- src/aignostics/application/_gui/_frame.py | 21 ++++-- .../_gui/_page_application_describe.py | 2 +- .../_gui/_page_application_run_describe.py | 7 +- .../application/_gui/_page_builder.py | 4 +- .../application/_gui/_page_index.py | 11 ++- tests/aignostics/application/cli_test.py | 24 +++---- tests/aignostics/application/gui_test.py | 67 +++++++++++++----- tests/aignostics/platform/e2e_test.py | 70 ++++++++++++------- tests/aignostics/qupath/gui_test.py | 31 ++++---- tests/constants_test.py | 47 ++++++++----- 10 files changed, 184 insertions(+), 100 deletions(-) diff --git a/src/aignostics/application/_gui/_frame.py b/src/aignostics/application/_gui/_frame.py index 11c02ed0b..f3a6f3153 100644 --- a/src/aignostics/application/_gui/_frame.py +++ b/src/aignostics/application/_gui/_frame.py @@ -40,6 +40,12 @@ async def _frame( # noqa: C901, PLR0913, PLR0915, PLR0917 ) -> None: if args is None: args = {} + + if args.get("query"): + search_input.query = args["query"] + else: + search_input.query = "" + with frame( # noqa: PLR1702 navigation_title=navigation_title, navigation_icon=navigation_icon, @@ -145,7 +151,7 @@ async def application_runs_load_and_render( # noqa: C901 .props("clickable") .classes("w-full") .style("padding-left: 0; padding-right: 0;") - .mark(f"SIDEBAR_RUN_ITEM:{index}") + .mark(f"SIDEBAR_RUN_ITEM:{index}:{run_data['run_id']}") ): with ui.item_section().props("avatar"): icon, color = run_status_to_icon_and_color( @@ -181,13 +187,18 @@ async def application_runs_load_and_render( # noqa: C901 else "font-normal" ).mark(f"LABEL_RUN_APPLICATION:{index}") ui.label(f"submitted {run_data['submitted_at'].astimezone().strftime('%m-%d %H:%M')}") - # if there is tags, show them as immutable chips - # only show the first 3, than "... and tooltip with the rest" - # chips must be very small, white background, black text, outlined, disabled if run_data.get("tags") and len(run_data["tags"]): with ui.row().classes("gap-1 mt-1"): for tag in run_data["tags"][:3]: - ui.chip(tag).props("disabled").classes("bg-white text-black text-xs") + + def _on_tag_click(t: str = tag) -> None: + search_input.query = t + _runs_list.refresh() + + ui.chip( + tag, + on_click=_on_tag_click, + ).props("clickable").classes("bg-white text-black text-xs") if len(run_data["tags"]) > 3: # noqa: PLR2004 ui.tooltip(f"Tags: {', '.join(run_data['tags'][3:])}") if not runs: diff --git a/src/aignostics/application/_gui/_page_application_describe.py b/src/aignostics/application/_gui/_page_application_describe.py index 9a33a3642..654e2c75c 100644 --- a/src/aignostics/application/_gui/_page_application_describe.py +++ b/src/aignostics/application/_gui/_page_application_describe.py @@ -611,7 +611,7 @@ class ThumbnailRenderer { ).bind_value(submit_form, "tags").classes("full-width").mark("INPUT_TAGS") with ui.stepper_navigation(): - ui.button("Next", on_click=stepper.next).mark("BUTTON_NOTES_NEXT") + ui.button("Next", on_click=stepper.next).mark("BUTTON_NOTES_AND_TAGS_NEXT") ui.button("Back", on_click=stepper.previous).props("flat") with ui.step("Schedule"): diff --git a/src/aignostics/application/_gui/_page_application_run_describe.py b/src/aignostics/application/_gui/_page_application_run_describe.py index b51dbdadc..154ecac41 100644 --- a/src/aignostics/application/_gui/_page_application_run_describe.py +++ b/src/aignostics/application/_gui/_page_application_run_describe.py @@ -543,7 +543,7 @@ def open_marimo(results_folder: Path, button: ui.button | None = None) -> None: * Error: {run_data.error_message or "N/A"} ({run_data.error_code or "N/A"}) """, language="markdown", - ).classes("full-width") + ).classes("full-width").mark("CODE_RUN_METADATA") if run_data.custom_metadata: properties = { "content": {"json": run_data.custom_metadata}, @@ -613,7 +613,10 @@ def open_marimo(results_folder: Path, button: ui.button | None = None) -> None: if tags and len(tags): with ui.row().classes("gap-1 -mt-2 full-width"): for tag in tags[:20]: - ui.chip(tag).props("small outlined disabled").classes("bg-white text-black") + ui.chip( + tag, + on_click=lambda t=tag: ui.navigate.to(f"/?query={quote(str(t))}"), + ).props("small outlined clickable").classes("bg-white text-black") with ui.list().classes("full-width"): results = list(run.results()) diff --git a/src/aignostics/application/_gui/_page_builder.py b/src/aignostics/application/_gui/_page_builder.py index c3e5516e4..6169985e3 100644 --- a/src/aignostics/application/_gui/_page_builder.py +++ b/src/aignostics/application/_gui/_page_builder.py @@ -13,11 +13,11 @@ def register_pages() -> None: app.add_static_files("/application_assets", Path(__file__).parent / "assets") @ui.page("/") - async def page_index(client: Client) -> None: + async def page_index(client: Client, query: str | None = None) -> None: """Index page of application module, serving as the homepage of Aignostics Launchpad.""" from ._page_index import _page_index # noqa: PLC0415 - await _page_index(client) + await _page_index(client, query=query) @ui.page("/application/{application_id}") async def page_application_describe(application_id: str) -> None: diff --git a/src/aignostics/application/_gui/_page_index.py b/src/aignostics/application/_gui/_page_index.py index 59b167c77..9b25feacf 100644 --- a/src/aignostics/application/_gui/_page_index.py +++ b/src/aignostics/application/_gui/_page_index.py @@ -20,15 +20,20 @@ pyi_splash = None -async def _page_index(client: Client) -> None: - """Homepage of Applications.""" +async def _page_index(client: Client, query: str | None = None) -> None: + """Homepage of Applications. + + Args: + client: The NiceGUI client. + query: Optional query parameter for filtering runs. + """ client.content.classes(remove="nicegui-content") client.content.classes(add="pl-5 pt-5") if pyi_splash and pyi_splash.is_alive(): pyi_splash.update_text("Connecting with API ...") - await _frame("Analyze your Whole Slide Images with AI", left_sidebar=True) + await _frame("Analyze your Whole Slide Images with AI", left_sidebar=True, args={"query": query}) if pyi_splash and pyi_splash.is_alive(): pyi_splash.close() diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py index 489788a60..5128e98d3 100644 --- a/tests/aignostics/application/cli_test.py +++ b/tests/aignostics/application/cli_test.py @@ -13,12 +13,12 @@ from aignostics.utils import sanitize_path from tests.conftest import normalize_output, print_directory_structure from tests.constants_test import ( - HETA_ANOTHER_SPOT_EXPECTED_RESULT_FILES, - HETA_ANOTHER_SPOT_FILENAME, - HETA_ANOTHER_SPOT_FILESIZE, - HETA_ANOTHER_SPOT_GS_URL, HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, + SPOT_1_EXPECTED_RESULT_FILES, + SPOT_1_FILENAME, + SPOT_1_FILESIZE, + SPOT_1_GS_URL, TEST_APPLICATION_ID, TEST_APPLICATION_VERSION, ) @@ -639,7 +639,7 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: "dataset", "aignostics", "download", - HETA_ANOTHER_SPOT_GS_URL, + SPOT_1_GS_URL, str(tmp_path), ], ) @@ -649,11 +649,11 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: # Validate what was downloaded assert "Successfully downloaded" in normalize_output(result.stdout) - assert HETA_ANOTHER_SPOT_FILENAME in normalize_output(result.stdout) - expected_file = tmp_path / HETA_ANOTHER_SPOT_FILENAME + assert SPOT_1_FILENAME in normalize_output(result.stdout) + expected_file = tmp_path / SPOT_1_FILENAME assert expected_file.exists(), f"Expected file {expected_file} not found" - assert expected_file.stat().st_size == HETA_ANOTHER_SPOT_FILESIZE, ( - f"Expected file size {HETA_ANOTHER_SPOT_FILESIZE}, but got {expected_file.stat().st_size}" + assert expected_file.stat().st_size == SPOT_1_FILESIZE, ( + f"Expected file size {SPOT_1_FILESIZE}, but got {expected_file.stat().st_size}" ) # Validate the download command exited successfully @@ -687,21 +687,21 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: assert not input_dir.is_dir(), f"Expected input directory {input_dir} not found" # Validate results generated and downloaded - results_dir = tmp_path / HETA_ANOTHER_SPOT_FILENAME.replace(".tiff", "") + results_dir = tmp_path / SPOT_1_FILENAME.replace(".tiff", "") assert results_dir.is_dir(), f"Expected directory {results_dir} not found" files_in_dir = list(results_dir.glob("*")) assert len(files_in_dir) == 9, ( f"Expected 9 files in {results_dir}, but found {len(files_in_dir)}: {[f.name for f in files_in_dir]}" ) print(f"Found files in {results_dir}:") - for filename, expected_size, tolerance_percent in HETA_ANOTHER_SPOT_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in SPOT_1_EXPECTED_RESULT_FILES: file_path = results_dir / filename if file_path.exists(): actual_size = file_path.stat().st_size print(f" {filename}: {actual_size} bytes (expected: {expected_size} ±{tolerance_percent}%)") else: print(f" {filename}: NOT FOUND") - for filename, expected_size, tolerance_percent in HETA_ANOTHER_SPOT_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in SPOT_1_EXPECTED_RESULT_FILES: file_path = results_dir / filename assert file_path.exists(), f"Expected file {filename} not found" actual_size = file_path.stat().st_size diff --git a/tests/aignostics/application/gui_test.py b/tests/aignostics/application/gui_test.py index 36190f5d3..8427efd48 100644 --- a/tests/aignostics/application/gui_test.py +++ b/tests/aignostics/application/gui_test.py @@ -19,10 +19,10 @@ from tests.constants_test import ( HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, - HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES, - HETA_SINGLE_SPOT_FILENAME, - HETA_SINGLE_SPOT_FILESIZE, - HETA_SINGLE_SPOT_GS_URL, + SPOT_0_EXPECTED_RESULT_FILES, + SPOT_0_FILENAME, + SPOT_0_FILESIZE, + SPOT_0_GS_URL, ) if TYPE_CHECKING: @@ -167,13 +167,13 @@ async def test_gui_cli_submit_to_run_result_delete(user: User, runner: CliRunner @pytest.mark.e2e @pytest.mark.long_running -@pytest.mark.flaky(retries=1, delay=5) +# @pytest.mark.flaky(retries=1, delay=5) @pytest.mark.timeout(timeout=60 * 10) @pytest.mark.sequential -async def test_gui_download_dataset_via_application_to_run_cancel( # noqa: PLR0915 +async def test_gui_download_dataset_via_application_to_run_cancel_to_find_back( # noqa: PLR0915 user: User, runner: CliRunner, silent_logging: None ) -> None: - """Test that the user can download a dataset via the application page and cancel the run.""" + """Test that the user can download a dataset via the application page and cancel the run, then find it back.""" with tempfile.TemporaryDirectory() as tmpdir: tmp_path = Path(tmpdir) @@ -246,19 +246,28 @@ async def test_gui_download_dataset_via_application_to_run_cancel( # noqa: PLR0 user.find(marker="BUTTON_METADATA_NEXT").click() await assert_notified(user, "Metadata captured.") - # Navigate through Notes step + # Navigate through Notes and Tags step await user.should_see("Note (optional)", retries=100) user.find("TEXTAREA_NOTE").type("test_gui_download_dataset_via_application_to_run_cancel:note").trigger( "keydown.enter" ) - user.find(marker="BUTTON_NOTES_NEXT").click() + await user.should_see("Tags (optional, press Enter to add)") + tags_input: ui.input_chips = user.find(marker="INPUT_TAGS").elements.pop() + tags_input.value = ["test_gui_tag1", "test_gui_tag2"] + + user.find(marker="BUTTON_NOTES_AND_TAGS_NEXT").click() # Navigate through Scheduling step await user.should_see("Soft Due Date", retries=100) await user.should_see("The platform will try to complete the run before this time", retries=100) + + await user.should_see("Hard Deadline") + await user.should_see("The platform might cancel the run if not completed by this time.", retries=100) + time_deadline: ui.time = user.find(marker="TIME_DEADLINE").elements.pop() + time_deadline.value = (datetime.now().astimezone() + timedelta(minutes=10)).strftime("%Y-%m-%d %H:%M") + user.find(marker="BUTTON_SCHEDULING_NEXT").click() - # TODO(Helmut): Set short deadline via GUI await assert_notified(user, "Prepared upload UI.") # Now on Submission step @@ -284,6 +293,13 @@ async def test_gui_download_dataset_via_application_to_run_cancel( # noqa: PLR0 except AssertionError: await user.should_see("PROCESSING", retries=100) + code_run_metadata: ui.code = user.find(marker="CODE_RUN_METADATA").elements.pop() + metadata_text = code_run_metadata.props["content"] + # extract run id, with metadata text containing Run ID: '{run_data.run_id}' + run_id_match = re.search(r"Run ID: ([0-9a-f-]+)", metadata_text) + assert run_id_match is not None, f"Could not extract run ID from metadata: {metadata_text}" + run_id = run_id_match.group(1) + # Check user can cancel run await user.should_see(marker="BUTTON_APPLICATION_RUN_CANCEL", retries=100) user.find(marker="BUTTON_APPLICATION_RUN_CANCEL").click() @@ -293,8 +309,21 @@ async def test_gui_download_dataset_via_application_to_run_cancel( # noqa: PLR0 # Check user sees refreshed run page and run is cancelled await user.should_see("CANCELED_BY_USER", retries=200) - # Check the note was saved correctly + # Check the tags were saved correctly await user.should_see("test_gui_download_dataset_via_application_to_run_cancel:note", retries=100) + await user.should_see("test_gui_tag1", retries=100) + await user.should_see("test_gui_tag2", retries=100) + + # Click on a tag to go to the homagepage with filtered runs + user.find("test_gui_tag1").click() + await sleep(10) + + # Check. user is on the homepage and the run filter is set to the tag clicked + user.should_see("Welcome to the Aignostics Launchpad") + user.should_see("test_gui_tag1", marker="INPUT_RUNS_FILTER_NOTE_OR_TAGS") + + # Check the first run is the one we created + user.should_see(marker=f"SIDEBAR_RUN_ITEM:0:{run_id}") @pytest.mark.e2e @@ -312,7 +341,7 @@ async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, s runs = Service().application_runs( application_id=HETA_APPLICATION_ID, application_version=HETA_APPLICATION_VERSION, - external_id=HETA_SINGLE_SPOT_GS_URL, + external_id=SPOT_0_GS_URL, has_output=True, limit=1, ) @@ -372,14 +401,14 @@ async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, s input_dir = run_dir / "input" assert input_dir.is_dir(), f"Expected input directory {input_dir} not found" - results_dir = run_dir / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") + results_dir = run_dir / SPOT_0_FILENAME.replace(".tiff", "") assert results_dir.is_dir(), f"Expected run results directory {results_dir} not found" # Check for input file having been downloaded - input_file = input_dir / HETA_SINGLE_SPOT_FILENAME - assert input_file.is_file(), f"Expected input file {input_file} not found" - assert input_file.stat().st_size == HETA_SINGLE_SPOT_FILESIZE, ( - f"Expected input file size {HETA_SINGLE_SPOT_FILESIZE}, but got {input_file.stat().st_size}" + input_file = input_dir / SPOT_0_FILENAME + assert input_file.exists(), f"Expected input file {input_file} not found" + assert input_file.stat().st_size == SPOT_0_FILESIZE, ( + f"Expected input file size {SPOT_0_FILESIZE}, but got {input_file.stat().st_size}" ) # Check for files in the results directory @@ -390,14 +419,14 @@ async def test_gui_run_download(user: User, runner: CliRunner, tmp_path: Path, s ) print(f"Found files in {results_dir}:") - for filename, expected_size, tolerance_percent in HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in SPOT_0_EXPECTED_RESULT_FILES: file_path = results_dir / filename if file_path.exists(): actual_size = file_path.stat().st_size print(f" {filename}: {actual_size} bytes (expected: {expected_size} ±{tolerance_percent}%)") else: print(f" {filename}: NOT FOUND") - for filename, expected_size, tolerance_percent in HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in SPOT_0_EXPECTED_RESULT_FILES: file_path = results_dir / filename assert file_path.exists(), f"Expected file {filename} not found" actual_size = file_path.stat().st_size diff --git a/tests/aignostics/platform/e2e_test.py b/tests/aignostics/platform/e2e_test.py index 925c8d230..443269f0b 100644 --- a/tests/aignostics/platform/e2e_test.py +++ b/tests/aignostics/platform/e2e_test.py @@ -25,10 +25,28 @@ from tests.constants_test import ( HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, - HETA_SINGLE_SPOT_GS_URL, + SPOT_0_CRC32C, + SPOT_0_GS_URL, + SPOT_0_HEIGHT, + SPOT_0_RESOLUTION_MPP, + SPOT_0_WIDTH, + SPOT_1_CRC32C, + SPOT_1_GS_URL, + SPOT_1_HEIGHT, + SPOT_1_RESOLUTION_MPP, + SPOT_1_WIDTH, + SPOT_2_CRC32C, + SPOT_2_GS_URL, + SPOT_2_HEIGHT, + SPOT_2_RESOLUTION_MPP, + SPOT_2_WIDTH, + SPOT_3_CRC32C, + SPOT_3_GS_URL, + SPOT_3_HEIGHT, + SPOT_3_RESOLUTION_MPP, + SPOT_3_WIDTH, TEST_APPLICATION_ID, TEST_APPLICATION_VERSION, - TEST_THREE_SPOTS_GS_URLS, ) TEST_APPLICATION_SUBMIT_AND_WAIT_DEADLINE_SECONDS = 60 * 45 # 45 minutes @@ -48,19 +66,19 @@ def _get_single_spot_payload_for_heta(expires_seconds: int) -> list[platform.Inp """Generates a payload using a single spot.""" return [ platform.InputItem( - external_id=HETA_SINGLE_SPOT_GS_URL, + external_id=SPOT_0_GS_URL, input_artifacts=[ platform.InputArtifact( name="whole_slide_image", download_url=platform.generate_signed_url( - url=HETA_SINGLE_SPOT_GS_URL, + url=SPOT_0_GS_URL, expires_seconds=expires_seconds, ), metadata={ - "checksum_base64_crc32c": "5onqtA==", - "resolution_mpp": 0.26268186053789266, - "width_px": 7447, - "height_px": 7196, + "checksum_base64_crc32c": SPOT_0_CRC32C, + "resolution_mpp": SPOT_0_RESOLUTION_MPP, + "width_px": SPOT_0_WIDTH, + "height_px": SPOT_0_HEIGHT, "media_type": "image/tiff", "staining_method": "H&E", "specimen": { @@ -78,57 +96,57 @@ def _get_three_spots_payload_for_test(expires_seconds: int) -> list[platform.Inp """Generates a payload using three spots.""" return [ platform.InputItem( - external_id=TEST_THREE_SPOTS_GS_URLS[0], + external_id=SPOT_1_GS_URL, input_artifacts=[ platform.InputArtifact( name="whole_slide_image", download_url=platform.generate_signed_url( - url=TEST_THREE_SPOTS_GS_URLS[0], + url=SPOT_1_GS_URL, expires_seconds=expires_seconds, ), metadata={ - "checksum_base64_crc32c": "9l3NNQ==", - "width_px": 3728, - "height_px": 3640, - "resolution_mpp": 0.46499982, + "checksum_base64_crc32c": SPOT_1_CRC32C, + "width_px": SPOT_1_WIDTH, + "height_px": SPOT_1_HEIGHT, + "resolution_mpp": SPOT_1_RESOLUTION_MPP, "media_type": "image/tiff", }, ) ], ), platform.InputItem( - external_id=TEST_THREE_SPOTS_GS_URLS[1], + external_id=SPOT_2_GS_URL, input_artifacts=[ platform.InputArtifact( name="whole_slide_image", download_url=platform.generate_signed_url( - url=TEST_THREE_SPOTS_GS_URLS[1], + url=SPOT_2_GS_URL, expires_seconds=expires_seconds, ), metadata={ - "checksum_base64_crc32c": "w+ud3g==", - "width_px": 3616, - "height_px": 3400, - "resolution_mpp": 0.46499982, + "checksum_base64_crc32c": SPOT_2_CRC32C, + "width_px": SPOT_2_WIDTH, + "height_px": SPOT_2_HEIGHT, + "resolution_mpp": SPOT_2_RESOLUTION_MPP, "media_type": "image/tiff", }, ) ], ), platform.InputItem( - external_id=TEST_THREE_SPOTS_GS_URLS[2], + external_id=SPOT_3_GS_URL, input_artifacts=[ platform.InputArtifact( name="whole_slide_image", download_url=platform.generate_signed_url( - url=TEST_THREE_SPOTS_GS_URLS[2], + url=SPOT_3_GS_URL, expires_seconds=expires_seconds, ), metadata={ - "checksum_base64_crc32c": "Zmx0wA==", - "width_px": 4016, - "height_px": 3952, - "resolution_mpp": 0.46499982, + "checksum_base64_crc32c": SPOT_3_CRC32C, + "width_px": SPOT_3_WIDTH, + "height_px": SPOT_3_HEIGHT, + "resolution_mpp": SPOT_3_RESOLUTION_MPP, "media_type": "image/tiff", }, ) diff --git a/tests/aignostics/qupath/gui_test.py b/tests/aignostics/qupath/gui_test.py index 0f28b9f60..ec72d2e8c 100644 --- a/tests/aignostics/qupath/gui_test.py +++ b/tests/aignostics/qupath/gui_test.py @@ -22,10 +22,10 @@ from tests.constants_test import ( HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, - HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES, - HETA_SINGLE_SPOT_FILENAME, - HETA_SINGLE_SPOT_FILESIZE, - HETA_SINGLE_SPOT_GS_URL, + SPOT_0_EXPECTED_RESULT_FILES, + SPOT_0_FILENAME, + SPOT_0_FILESIZE, + SPOT_0_GS_URL, ) if TYPE_CHECKING: @@ -156,7 +156,7 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 runs = Service().application_runs( application_id=HETA_APPLICATION_ID, application_version=HETA_APPLICATION_VERSION, - external_id=HETA_SINGLE_SPOT_GS_URL, + external_id=SPOT_0_GS_URL, has_output=True, limit=1, ) @@ -242,17 +242,17 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 input_dir = run_dir / "input" assert input_dir.is_dir(), f"Expected input directory {input_dir} not found" - results_dir = run_dir / HETA_SINGLE_SPOT_FILENAME.replace(".tiff", "") + results_dir = run_dir / SPOT_0_FILENAME.replace(".tiff", "") assert results_dir.is_dir(), f"Expected run results directory {results_dir} not found" qupath_dir = run_dir / "qupath" assert qupath_dir.is_dir(), f"Expected QuPath directory {qupath_dir} not found" # Check for input file having been downloaded - input_file = input_dir / HETA_SINGLE_SPOT_FILENAME - assert input_file.is_file(), f"Expected input file {input_file} not found" - assert input_file.stat().st_size == HETA_SINGLE_SPOT_FILESIZE, ( - f"Expected input file size {HETA_SINGLE_SPOT_FILESIZE}, but got {input_file.stat().st_size}" + input_file = input_dir / SPOT_0_FILENAME + assert input_file.exists(), f"Expected input file {input_file} not found" + assert input_file.stat().st_size == SPOT_0_FILESIZE, ( + f"Expected input file size {SPOT_0_FILESIZE}, but got {input_file.stat().st_size}" ) # Check for files in the results directory @@ -263,14 +263,14 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 ) print(f"Found files in {results_dir}:") - for filename, expected_size, tolerance_percent in HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in SPOT_0_EXPECTED_RESULT_FILES: file_path = results_dir / filename if file_path.exists(): actual_size = file_path.stat().st_size print(f" {filename}: {actual_size} bytes (expected: {expected_size} ±{tolerance_percent}%)") else: print(f" {filename}: NOT FOUND") - for filename, expected_size, tolerance_percent in HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES: + for filename, expected_size, tolerance_percent in SPOT_0_EXPECTED_RESULT_FILES: file_path = results_dir / filename assert file_path.exists(), f"Expected file {filename} not found" actual_size = file_path.stat().st_size @@ -302,12 +302,17 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 try: project_info = json.loads(output) annotations_total = 0 + original_image_found = False for image in project_info["images"]: + if image.get("name") == SPOT_0_FILENAME: + original_image_found = True hierarchy = image.get("hierarchy", {}) total = hierarchy.get("total", 0) if total > 0: annotations_total += total - assert annotations_total >= 10000, "Expected at least 10000 annotations in the QuPath results" + assert original_image_found, f"Original image '{SPOT_0_FILENAME}' not found in QuPath project" + # TODO(helmut): Fix + assert annotations_total >= 0, "Expected at least 10000 annotations in the QuPath results" except json.JSONDecodeError as e: pytest.fail(f"Failed to parse QuPath inspect output as JSON: {e}\nOutput: {output!r}\n") diff --git a/tests/constants_test.py b/tests/constants_test.py index 7a373bd5c..a8218f181 100644 --- a/tests/constants_test.py +++ b/tests/constants_test.py @@ -6,12 +6,10 @@ import os -HETA_SINGLE_SPOT_GS_URL = ( - "gs://platform-api-application-test-data/heta/slides/8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" -) -HETA_SINGLE_SPOT_FILENAME = "8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" -HETA_SINGLE_SPOT_FILESIZE = 10562338 # in bytes -HETA_SINGLE_SPOT_EXPECTED_RESULT_FILES = [ +SPOT_0_GS_URL = "gs://platform-api-application-test-data/heta/slides/8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" +SPOT_0_FILENAME = "8fafc17d-a5cc-4e9d-a982-030b1486ca88.tiff" +SPOT_0_FILESIZE = 10562338 +SPOT_0_EXPECTED_RESULT_FILES = [ ("tissue_qc_segmentation_map_image.tiff", 1698570, 10), ("tissue_qc_geojson_polygons.json", 315019, 10), ("tissue_segmentation_geojson_polygons.json", 927599, 10), @@ -22,13 +20,15 @@ ("tissue_segmentation_csv_class_information.csv", 361, 10), ("tissue_qc_csv_class_information.csv", 236, 10), ] +SPOT_0_CRC32C = "5onqtA==" +SPOT_0_RESOLUTION_MPP = 0.26268186053789266 +SPOT_0_WIDTH = 7447 +SPOT_0_HEIGHT = 7196 -HETA_ANOTHER_SPOT_GS_URL = ( - "gs://aignx-storage-service-dev/sample_data_formatted/9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" -) -HETA_ANOTHER_SPOT_FILENAME = "9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" -HETA_ANOTHER_SPOT_FILESIZE = 14681750 # in bytes -HETA_ANOTHER_SPOT_EXPECTED_RESULT_FILES = [ +SPOT_1_GS_URL = "gs://aignx-storage-service-dev/sample_data_formatted/9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" +SPOT_1_FILENAME = "9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff" +SPOT_1_FILESIZE = 14681750 +SPOT_1_EXPECTED_RESULT_FILES = [ ("tissue_qc_segmentation_map_image.tiff", 464908, 10), ("tissue_qc_geojson_polygons.json", 180522, 10), ("tissue_segmentation_geojson_polygons.json", 270931, 10), @@ -39,12 +39,25 @@ ("tissue_segmentation_csv_class_information.csv", 342, 10), ("tissue_qc_csv_class_information.csv", 232, 10), ] +SPOT_1_CRC32C = "9l3NNQ==" +SPOT_1_WIDTH = 3728 +SPOT_1_HEIGHT = 3640 +SPOT_1_RESOLUTION_MPP = 0.46499982 -TEST_THREE_SPOTS_GS_URLS = [ - "gs://aignx-storage-service-dev/sample_data_formatted/9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff", - "gs://aignx-storage-service-dev/sample_data_formatted/8c7b079e-8b8a-4036-bfde-5818352b503a.tiff", - "gs://aignx-storage-service-dev/sample_data_formatted/1f4f366f-a2c5-4407-9f5e-23400b22d50e.tiff", -] +SPOT_2_GS_URL = "gs://aignx-storage-service-dev/sample_data_formatted/8c7b079e-8b8a-4036-bfde-5818352b503a.tiff" +SPOT_2_FILENAME = "8c7b079e-8b8a-4036-bfde-5818352b503a.tiff" +SPOT_2_FILESIZE = 20153772 +SPOT_2_CRC32C = "w+ud3g==" +SPOT_2_WIDTH = 3616 +SPOT_2_HEIGHT = 3400 +SPOT_2_RESOLUTION_MPP = 0.46499982 + +SPOT_3_GS_URL = "gs://aignx-storage-service-dev/sample_data_formatted/1f4f366f-a2c5-4407-9f5e-23400b22d50e.tiff" +SPOT_3_FILENAME = "1f4f366f-a2c5-4407-9f5e-23400b22d50e.tiff" +SPOT_3_CRC32C = "Zmx0wA==" +SPOT_3_WIDTH = 4016 +SPOT_3_HEIGHT = 3952 +SPOT_3_RESOLUTION_MPP = 0.46499982 match os.getenv("AIGNOSTICS_PLATFORM_ENVIRONMENT", "production"): case "production": From f2523eab8f44b913eed964bed9d9c10ce862627f Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 15:46:33 +0100 Subject: [PATCH 10/16] testing ... --- src/aignostics/application/_service.py | 1 + tests/aignostics/qupath/gui_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aignostics/application/_service.py b/src/aignostics/application/_service.py index 49743afab..83a7f596d 100644 --- a/src/aignostics/application/_service.py +++ b/src/aignostics/application/_service.py @@ -1334,6 +1334,7 @@ def update_qupath_annotate_input_with_results_progress( image_path = Path(item.external_id) if not image_path.is_file(): + logger.warning("Input slide '%s' not found, skipping QuPath annotation.", image_path) continue for artifact in item.output_artifacts: if ( diff --git a/tests/aignostics/qupath/gui_test.py b/tests/aignostics/qupath/gui_test.py index ec72d2e8c..b0308c683 100644 --- a/tests/aignostics/qupath/gui_test.py +++ b/tests/aignostics/qupath/gui_test.py @@ -149,7 +149,7 @@ async def test_gui_qupath_install_and_launch( @pytest.mark.timeout(timeout=60 * 15) @pytest.mark.sequential async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR0914, PLR0915 - user: User, runner: CliRunner, tmp_path: Path, silent_logging: None, qupath_teardown: None + user: User, runner: CliRunner, tmp_path: Path, qupath_teardown: None ) -> None: """Test installing QuPath, downloading run results, creating QuPath project from it, and inspecting results.""" # Find run From 37169796487af412c938d971f3fec61fa8cb230f Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 16:11:05 +0100 Subject: [PATCH 11/16] testing ... --- src/aignostics/application/_service.py | 8 ++++---- tests/aignostics/qupath/gui_test.py | 18 ++++++++++++++++-- tests/constants_test.py | 1 + 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/aignostics/application/_service.py b/src/aignostics/application/_service.py index 83a7f596d..9ba7ddc54 100644 --- a/src/aignostics/application/_service.py +++ b/src/aignostics/application/_service.py @@ -1209,7 +1209,8 @@ def application_run_download( # noqa: C901, PLR0912, PLR0913, PLR0914, PLR0915, logger.warning(message) raise ValueError(message) from e - for item_index, item in enumerate(application_run.results()): + results = list(application_run.results()) + for item_index, item in enumerate(results): if item.external_id.startswith(("gs://", "http://", "https://")): # Download URL to local input directory and update external_id try: @@ -1240,7 +1241,7 @@ def update_qupath_add_input_progress(qupath_add_input_progress: QuPathAddProgres logger.debug("Adding input slides to QuPath project ...") image_paths = [] - for item in application_run.results(): + for item in results: local_path = Path(item.external_id) if not local_path.is_file(): logger.warning("Input slide '%s' not found, skipping QuPath addition.", local_path) @@ -1326,9 +1327,8 @@ def update_qupath_annotate_input_with_results_progress( update_progress(progress, download_progress_callable, download_progress_queue) total_annotations = 0 - results = list(application_run.results()) progress.item_count = len(results) - for item_index, item in enumerate(application_run.results()): + for item_index, item in enumerate(results): progress.item_index = item_index update_progress(progress, download_progress_callable, download_progress_queue) diff --git a/tests/aignostics/qupath/gui_test.py b/tests/aignostics/qupath/gui_test.py index b0308c683..a48b3a931 100644 --- a/tests/aignostics/qupath/gui_test.py +++ b/tests/aignostics/qupath/gui_test.py @@ -22,10 +22,13 @@ from tests.constants_test import ( HETA_APPLICATION_ID, HETA_APPLICATION_VERSION, + SPOT_0_EXPECTED_CELLS_CLASSIFIED, SPOT_0_EXPECTED_RESULT_FILES, SPOT_0_FILENAME, SPOT_0_FILESIZE, SPOT_0_GS_URL, + SPOT_0_HEIGHT, + SPOT_0_WIDTH, ) if TYPE_CHECKING: @@ -303,16 +306,27 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 project_info = json.loads(output) annotations_total = 0 original_image_found = False + width = None + height = None for image in project_info["images"]: if image.get("name") == SPOT_0_FILENAME: original_image_found = True + width = image.get("width") + height = image.get("height") hierarchy = image.get("hierarchy", {}) total = hierarchy.get("total", 0) if total > 0: annotations_total += total assert original_image_found, f"Original image '{SPOT_0_FILENAME}' not found in QuPath project" - # TODO(helmut): Fix - assert annotations_total >= 0, "Expected at least 10000 annotations in the QuPath results" + assert width == SPOT_0_WIDTH, f"Expected image width {SPOT_0_WIDTH}, but got {width}" + assert height == SPOT_0_HEIGHT, f"Expected image height {SPOT_0_HEIGHT}, but got {height}" + assert abs(annotations_total - SPOT_0_EXPECTED_CELLS_CLASSIFIED[0]) <= ( + SPOT_0_EXPECTED_CELLS_CLASSIFIED[0] * SPOT_0_EXPECTED_CELLS_CLASSIFIED[1] // 100 + ), ( + f"Expected approximately {SPOT_0_EXPECTED_CELLS_CLASSIFIED[0]} " + f"({SPOT_0_EXPECTED_CELLS_CLASSIFIED[1]}% tolerance) annotations in the QuPath results, " + f"but found {annotations_total}" + ) except json.JSONDecodeError as e: pytest.fail(f"Failed to parse QuPath inspect output as JSON: {e}\nOutput: {output!r}\n") diff --git a/tests/constants_test.py b/tests/constants_test.py index a8218f181..6d277ad46 100644 --- a/tests/constants_test.py +++ b/tests/constants_test.py @@ -20,6 +20,7 @@ ("tissue_segmentation_csv_class_information.csv", 361, 10), ("tissue_qc_csv_class_information.csv", 236, 10), ] +SPOT_0_EXPECTED_CELLS_CLASSIFIED = (35160, 10) SPOT_0_CRC32C = "5onqtA==" SPOT_0_RESOLUTION_MPP = 0.26268186053789266 SPOT_0_WIDTH = 7447 From b8ad0c1db2e8b9834bb1932835fe73f337fee555 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 16:20:39 +0100 Subject: [PATCH 12/16] chore(qupath): deeper check in e2e flow --- tests/aignostics/qupath/gui_test.py | 35 +++++++++++++++++------------ 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/aignostics/qupath/gui_test.py b/tests/aignostics/qupath/gui_test.py index a48b3a931..757f4f7c2 100644 --- a/tests/aignostics/qupath/gui_test.py +++ b/tests/aignostics/qupath/gui_test.py @@ -302,25 +302,32 @@ async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR091 output = normalize_output(result.output, strip_ansi=True) print(repr(output)) + # Check for (1) spot added to QuPath project, (2) heatmaps added, (3) spot annotated try: project_info = json.loads(output) annotations_total = 0 - original_image_found = False - width = None - height = None + spot_found = False + spot_width = None + spot_height = None + qc_segmentation_map_found = False + tissue_segmentation_map_found = False for image in project_info["images"]: if image.get("name") == SPOT_0_FILENAME: - original_image_found = True - width = image.get("width") - height = image.get("height") - hierarchy = image.get("hierarchy", {}) - total = hierarchy.get("total", 0) - if total > 0: - annotations_total += total - assert original_image_found, f"Original image '{SPOT_0_FILENAME}' not found in QuPath project" - assert width == SPOT_0_WIDTH, f"Expected image width {SPOT_0_WIDTH}, but got {width}" - assert height == SPOT_0_HEIGHT, f"Expected image height {SPOT_0_HEIGHT}, but got {height}" - assert abs(annotations_total - SPOT_0_EXPECTED_CELLS_CLASSIFIED[0]) <= ( + spot_found = True + spot_width = image.get("width") + spot_height = image.get("height") + hierarchy = image.get("hierarchy", {}) + spot_annotations = hierarchy.get("total", 0) + if image.get("name") == "tissue_qc_segmentation_map_image.tiff": + qc_segmentation_map_found = True + if image.get("name") == "tissue_segmentation_segmentation_map_image.tiff": + tissue_segmentation_map_found = True + assert spot_found, f"Spot '{SPOT_0_FILENAME}' not found in QuPath project" + assert spot_width == SPOT_0_WIDTH, f"Expected width of spot {SPOT_0_WIDTH}, but got {spot_width}" + assert spot_height == SPOT_0_HEIGHT, f"Expected height of spot {SPOT_0_HEIGHT}, but got {spot_height}" + assert qc_segmentation_map_found, "QC segmentation map image not found in QuPath project" + assert tissue_segmentation_map_found, "Tissue segmentation map image not found in QuPath project" + assert abs(spot_annotations - SPOT_0_EXPECTED_CELLS_CLASSIFIED[0]) <= ( SPOT_0_EXPECTED_CELLS_CLASSIFIED[0] * SPOT_0_EXPECTED_CELLS_CLASSIFIED[1] // 100 ), ( f"Expected approximately {SPOT_0_EXPECTED_CELLS_CLASSIFIED[0]} " From 2c1e44f496360dacb982b720dac07f107814dded Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 18:07:12 +0100 Subject: [PATCH 13/16] feat(application): editable custom run metadata editable for admnin in GUI feat(application): custom run and item metadata can be dumped as json via the CLI feat(application): custom run metadata can be updated via the CLI feat(platform): custom run and item metadata can be updated --- VERSION | 2 +- src/aignostics/application/_cli.py | 174 +++++++++++++ .../_gui/_page_application_run_describe.py | 40 ++- src/aignostics/application/_service.py | 137 +++++++++- src/aignostics/platform/_sdk_metadata.py | 42 ++- src/aignostics/platform/resources/runs.py | 79 +++++- tests/aignostics/application/cli_test.py | 239 ++++++++++++++++++ tests/aignostics/application/gui_test.py | 3 +- tests/aignostics/application/service_test.py | 74 ++++++ .../aignostics/platform/sdk_metadata_test.py | 40 +++ tests/aignostics/qupath/gui_test.py | 2 +- 11 files changed, 811 insertions(+), 21 deletions(-) diff --git a/VERSION b/VERSION index ce9664f75..2ec81553f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.195 +0.2.196 diff --git a/src/aignostics/application/_cli.py b/src/aignostics/application/_cli.py index d7120cb26..e56af780c 100644 --- a/src/aignostics/application/_cli.py +++ b/src/aignostics/application/_cli.py @@ -802,6 +802,84 @@ def run_describe(run_id: Annotated[str, typer.Argument(help="Id of the run to de sys.exit(1) +@run_app.command("dump-metadata") +def run_dump_metadata( + run_id: Annotated[str, typer.Argument(help="Id of the run to dump custom metadata for")], + pretty: Annotated[bool, typer.Option(help="Pretty print JSON output with indentation")] = False, +) -> None: + """Dump custom metadata of a run as JSON to stdout.""" + logger.debug("Dumping custom metadata for run with ID '%s'", run_id) + + try: + run = Service().application_run(run_id).details() + custom_metadata = run.custom_metadata if hasattr(run, "custom_metadata") else {} + + # Output JSON to stdout + if pretty: + print(json.dumps(custom_metadata, indent=2)) + else: + print(json.dumps(custom_metadata)) + + logger.info("Dumped custom metadata for run with ID '%s'", run_id) + except NotFoundException: + logger.warning("Run with ID '%s' not found.", run_id) + console.print(f"[warning]Warning:[/warning] Run with ID '{run_id}' not found.") + sys.exit(2) + except Exception as e: + logger.exception("Failed to dump custom metadata for run with ID '%s'", run_id) + console.print(f"[error]Error:[/error] Failed to dump custom metadata for run with ID '{run_id}': {e}") + sys.exit(1) + + +@run_app.command("dump-item-metadata") +def run_dump_item_metadata( + run_id: Annotated[str, typer.Argument(help="Id of the run containing the item")], + external_id: Annotated[str, typer.Argument(help="External ID of the item to dump custom metadata for")], + pretty: Annotated[bool, typer.Option(help="Pretty print JSON output with indentation")] = False, +) -> None: + """Dump custom metadata of an item as JSON to stdout.""" + logger.debug("Dumping custom metadata for item '%s' in run with ID '%s'", external_id, run_id) + + try: + run = Service().application_run(run_id) + + # Find the item with the matching external_id in the results + item = None + for result_item in run.results(): + if result_item.external_id == external_id: + item = result_item + break + + if item is None: + logger.warning("Item with external ID '%s' not found in run '%s'.", external_id, run_id) + print( + f"Warning: Item with external ID '{external_id}' not found in run '{run_id}'.", + file=sys.stderr, + ) + sys.exit(2) + + custom_metadata = item.custom_metadata if hasattr(item, "custom_metadata") else {} + + # Output JSON to stdout + if pretty: + print(json.dumps(custom_metadata, indent=2)) + else: + print(json.dumps(custom_metadata)) + + logger.info("Dumped custom metadata for item '%s' in run with ID '%s'", external_id, run_id) + except NotFoundException: + logger.warning("Run with ID '%s' not found.", run_id) + print(f"Warning: Run with ID '{run_id}' not found.", file=sys.stderr) + sys.exit(2) + except Exception as e: + logger.exception("Failed to dump custom metadata for item '%s' in run with ID '%s'", external_id, run_id) + print( + f"Error: Failed to dump custom metadata for item '{external_id}' in run with ID '{run_id}': {e}", + file=sys.stderr, + ) + sys.exit(1) + + @run_app.command("cancel") def run_cancel( run_id: Annotated[str, typer.Argument(..., help="Id of the run to cancel")], @@ -827,6 +905,102 @@ def run_cancel( sys.exit(1) +@run_app.command("update-metadata") +def run_update_metadata( + run_id: Annotated[str, typer.Argument(..., help="Id of the run to update")], + metadata_json: Annotated[ + str, typer.Argument(..., help='Custom metadata as JSON string (e.g., \'{"key": "value"}\')') + ], +) -> None: + """Update custom metadata for a run.""" + import json # noqa: PLC0415 + + logger.debug("Updating custom metadata for run with ID '%s'", run_id) + + try: + # Parse JSON metadata + try: + custom_metadata = json.loads(metadata_json) + if not isinstance(custom_metadata, dict): + console.print("[error]Error:[/error] Metadata must be a JSON object (dictionary).") + sys.exit(1) + except json.JSONDecodeError as e: + console.print(f"[error]Error:[/error] Invalid JSON: {e}") + sys.exit(1) + + Service().application_run_update_custom_metadata(run_id, custom_metadata) + logger.info("Updated custom metadata for run with ID '%s'.", run_id) + console.print(f"Successfully updated custom metadata for run with ID '{run_id}'.") + except NotFoundException: + logger.warning("Run with ID '%s' not found.", run_id) + console.print(f"[warning]Warning:[/warning] Run with ID '{run_id}' not found.") + sys.exit(2) + except ValueError as e: + logger.warning("Run ID '%s' invalid or metadata invalid: %s", run_id, e) + console.print(f"[warning]Warning:[/warning] Run ID '{run_id}' invalid or metadata invalid: {e}") + sys.exit(2) + except Exception as e: + logger.exception("Failed to update custom metadata for run with ID '%s'", run_id) + console.print(f"[bold red]Error:[/bold red] Failed to update custom metadata for run with ID '{run_id}': {e}") + sys.exit(1) + + +@run_app.command("update-item-metadata") +def run_update_item_metadata( + run_id: Annotated[str, typer.Argument(..., help="Id of the run containing the item")], + external_id: Annotated[str, typer.Argument(..., help="External ID of the item to update")], + metadata_json: Annotated[ + str, typer.Argument(..., help='Custom metadata as JSON string (e.g., \'{"key": "value"}\')') + ], +) -> None: + """Update custom metadata for an item in a run.""" + import json # noqa: PLC0415 + + logger.debug("Updating custom metadata for item '%s' in run with ID '%s'", external_id, run_id) + + try: + # Parse JSON metadata + try: + custom_metadata = json.loads(metadata_json) + if not isinstance(custom_metadata, dict): + console.print("[error]Error:[/error] Metadata must be a JSON object (dictionary).") + sys.exit(1) + except json.JSONDecodeError as e: + console.print(f"[error]Error:[/error] Invalid JSON: {e}") + sys.exit(1) + + Service().application_run_update_item_custom_metadata(run_id, external_id, custom_metadata) + logger.info("Updated custom metadata for item '%s' in run with ID '%s'.", external_id, run_id) + console.print(f"Successfully updated custom metadata for item '{external_id}' in run with ID '{run_id}'.") + except NotFoundException: + logger.warning("Run with ID '%s' or item '%s' not found.", run_id, external_id) + console.print(f"[warning]Warning:[/warning] Run with ID '{run_id}' or item '{external_id}' not found.") + sys.exit(2) + except ValueError as e: + logger.warning( + "Run ID '%s' or item external ID '%s' invalid or metadata invalid: %s", + run_id, + external_id, + e, + ) + console.print( + f"[warning]Warning:[/warning] Run ID '{run_id}' or item external ID '{external_id}' " + f"invalid or metadata invalid: {e}" + ) + sys.exit(2) + except Exception as e: + logger.exception( + "Failed to update custom metadata for item '%s' in run with ID '%s'", + external_id, + run_id, + ) + console.print( + f"[bold red]Error:[/bold red] Failed to update custom metadata for item '{external_id}' " + f"in run with ID '{run_id}': {e}" + ) + sys.exit(1) + + @result_app.command("download") def result_download( # noqa: C901, PLR0913, PLR0915, PLR0917 run_id: Annotated[str, typer.Argument(..., help="Id of the run to download results for")], diff --git a/src/aignostics/application/_gui/_page_application_run_describe.py b/src/aignostics/application/_gui/_page_application_run_describe.py index 154ecac41..6bec7c5ec 100644 --- a/src/aignostics/application/_gui/_page_application_run_describe.py +++ b/src/aignostics/application/_gui/_page_application_run_describe.py @@ -3,18 +3,24 @@ from importlib.util import find_spec from multiprocessing import Manager from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from urllib.parse import quote import humanize from aiopath import AsyncPath +from nicegui import ( + app, + ui, # noq +) from nicegui import run as nicegui_run -from nicegui import ui # noq from aignostics.platform import ItemOutput, ItemState, RunState from aignostics.third_party.showinfm.showinfm import show_in_file_manager from aignostics.utils import GUILocalFilePicker, get_logger, get_user_data_directory +if TYPE_CHECKING: + from aignostics.platform import UserInfo + from .._models import DownloadProgressState # noqa: TID252 from .._service import Service # noqa: TID252 from .._utils import get_mime_type_for_artifact # noqa: TID252 @@ -527,7 +533,7 @@ def open_marimo(results_folder: Path, button: ui.button | None = None) -> None: ui.code( f""" - * Run ID: {run_data.run_id} + * Run ID: {run_data.run_id}' * Application: {run_data.application_id} ({run_data.version_number}) * Status: {status_str} * Output: {run_data.output.name} @@ -544,16 +550,40 @@ def open_marimo(results_folder: Path, button: ui.button | None = None) -> None: """, language="markdown", ).classes("full-width").mark("CODE_RUN_METADATA") + user_info: UserInfo | None = app.storage.tab.get("user_info", None) if run_data.custom_metadata: + is_editable = user_info and user_info.role in {"admin", "super_admin"} properties = { "content": {"json": run_data.custom_metadata}, "mode": "tree", - "readOnly": True, + "readOnly": not is_editable, "mainMenuBar": True, "navigationBar": False, "statusBar": False, } - ui.json_editor(properties).classes("full-width").mark("JSON_EDITOR_HEALTH") + + async def handle_metadata_change(e: Any) -> None: # noqa: ANN401 + """Handle changes to the custom metadata and update the run.""" + if not is_editable: + return + try: + # Extract the new metadata from the event's content attribute + new_metadata = e.content.get("json") if hasattr(e, "content") else None + if new_metadata: + ui.notify("Updating custom metadata...", type="info") + await nicegui_run.io_bound( + Service.application_run_update_custom_metadata_static, + run_id=run_id, + custom_metadata=new_metadata, + ) + ui.notify("Custom metadata updated successfully!", type="positive") + ui.navigate.reload() + except Exception as ex: # noqa: BLE001 + ui.notify(f"Failed to update custom metadata: {ex!s}", type="negative") + + ui.json_editor(properties, on_change=handle_metadata_change).classes("full-width").mark( + "JSON_EDITOR_CUSTOM_METADATA" + ) ui.space() with ui.row().classes("justify-end"): if run_data.state.value == RunState.TERMINATED and run_data.statistics.item_succeeded_count > 0: diff --git a/src/aignostics/application/_service.py b/src/aignostics/application/_service.py index 9ba7ddc54..842519b2c 100644 --- a/src/aignostics/application/_service.py +++ b/src/aignostics/application/_service.py @@ -66,7 +66,7 @@ APPLICATION_RUN_UPLOAD_CHUNK_SIZE = 1024 * 1024 # 1MB -class Service(BaseService): +class Service(BaseService): # noqa: PLR0904 """Service of the application module.""" _settings: Settings @@ -1019,6 +1019,141 @@ def application_run_submit( # noqa: PLR0913, PLR0917 logger.exception(message) raise RuntimeError(message) from e + def application_run_update_custom_metadata( + self, + run_id: str, + custom_metadata: dict[str, Any], + ) -> None: + """Update custom metadata for an existing application run. + + Args: + run_id (str): The ID of the run to update + custom_metadata (dict[str, Any]): The new custom metadata to attach to the run. + + Raises: + NotFoundException: If the application run with the given ID is not found. + ValueError: If the run ID is invalid. + RuntimeError: If updating the run metadata fails unexpectedly. + """ + try: + logger.debug("Updating custom metadata for run with ID '%s'", run_id) + self._get_platform_client().run(run_id).update_custom_metadata(custom_metadata) + logger.debug("Updated custom metadata for run with ID '%s'", run_id) + except ValueError as e: + message = f"Failed to update custom metadata for run with ID '{run_id}': ValueError {e}" + logger.warning(message) + raise ValueError(message) from e + except NotFoundException as e: + message = f"Application run with ID '{run_id}' not found: {e}" + logger.warning(message) + raise NotFoundException(message) from e + except ApiException as e: + if e.status == HTTPStatus.UNPROCESSABLE_ENTITY: + message = f"Run ID '{run_id}' invalid: {e!s}." + logger.warning(message) + raise ValueError(message) from e + message = f"Failed to update custom metadata for run with ID '{run_id}': {e}" + logger.exception(message) + raise RuntimeError(message) from e + except Exception as e: + message = f"Failed to update custom metadata for run with ID '{run_id}': {e}" + logger.exception(message) + raise RuntimeError(message) from e + + @staticmethod + def application_run_update_custom_metadata_static( + run_id: str, + custom_metadata: dict[str, Any], + ) -> None: + """Static wrapper for updating custom metadata for an application run. + + Args: + run_id (str): The ID of the run to update + custom_metadata (dict[str, Any]): The new custom metadata to attach to the run. + + Raises: + NotFoundException: If the application run with the given ID is not found. + ValueError: If the run ID is invalid. + RuntimeError: If updating the run metadata fails unexpectedly. + """ + Service().application_run_update_custom_metadata(run_id, custom_metadata) + + def application_run_update_item_custom_metadata( + self, + run_id: str, + external_id: str, + custom_metadata: dict[str, Any], + ) -> None: + """Update custom metadata for an existing item in an application run. + + Args: + run_id (str): The ID of the run containing the item + external_id (str): The external ID of the item to update + custom_metadata (dict[str, Any]): The new custom metadata to attach to the item. + + Raises: + NotFoundException: If the application run or item with the given IDs is not found. + ValueError: If the run ID or item external ID is invalid. + RuntimeError: If updating the item metadata fails unexpectedly. + """ + try: + logger.debug( + "Updating custom metadata for item '%s' in run with ID '%s'", + external_id, + run_id, + ) + self._get_platform_client().run(run_id).update_item_custom_metadata( + external_id, + custom_metadata, + ) + logger.debug( + "Updated custom metadata for item '%s' in run with ID '%s'", + external_id, + run_id, + ) + except ValueError as e: + message = ( + f"Failed to update custom metadata for item '{external_id}' in run with ID '{run_id}': ValueError {e}" + ) + logger.warning(message) + raise ValueError(message) from e + except NotFoundException as e: + message = f"Application run with ID '{run_id}' or item '{external_id}' not found: {e}" + logger.warning(message) + raise NotFoundException(message) from e + except ApiException as e: + if e.status == HTTPStatus.UNPROCESSABLE_ENTITY: + message = f"Run ID '{run_id}' or item external ID '{external_id}' invalid: {e!s}." + logger.warning(message) + raise ValueError(message) from e + message = f"Failed to update custom metadata for item '{external_id}' in run with ID '{run_id}': {e}" + logger.exception(message) + raise RuntimeError(message) from e + except Exception as e: + message = f"Failed to update custom metadata for item '{external_id}' in run with ID '{run_id}': {e}" + logger.exception(message) + raise RuntimeError(message) from e + + @staticmethod + def application_run_update_item_custom_metadata_static( + run_id: str, + external_id: str, + custom_metadata: dict[str, Any], + ) -> None: + """Static wrapper for updating custom metadata for an item in an application run. + + Args: + run_id (str): The ID of the run containing the item + external_id (str): The external ID of the item to update + custom_metadata (dict[str, Any]): The new custom metadata to attach to the item. + + Raises: + NotFoundException: If the application run or item with the given IDs is not found. + ValueError: If the run ID or item external ID is invalid. + RuntimeError: If updating the item metadata fails unexpectedly. + """ + Service().application_run_update_item_custom_metadata(run_id, external_id, custom_metadata) + def application_run_cancel(self, run_id: str) -> None: """Cancel a run by its ID. diff --git a/src/aignostics/platform/_sdk_metadata.py b/src/aignostics/platform/_sdk_metadata.py index 7d84aba6b..8e3a0be4a 100644 --- a/src/aignostics/platform/_sdk_metadata.py +++ b/src/aignostics/platform/_sdk_metadata.py @@ -15,8 +15,8 @@ logger = get_logger(__name__) -SDK_METADATA_SCHEMA_VERSION = "0.0.3" -ITEM_SDK_METADATA_SCHEMA_VERSION = "0.0.2" +SDK_METADATA_SCHEMA_VERSION = "0.0.4" +ITEM_SDK_METADATA_SCHEMA_VERSION = "0.0.3" class SubmissionMetadata(BaseModel): @@ -100,7 +100,7 @@ class RunSdkMetadata(BaseModel): This model defines the structure and validation rules for SDK metadata that is attached to application runs. It includes information about: - - SDK version and submission details + - SDK version and timestamps - User information (when available) - CI/CD environment context (GitHub Actions, pytest) - Workflow control flags @@ -112,6 +112,8 @@ class RunSdkMetadata(BaseModel): ..., description="Schema version for this metadata format", pattern=r"^\d+\.\d+\.\d+-?.*$" ) + created_at: str = Field(..., description="ISO 8601 timestamp when the metadata was first created") + updated_at: str = Field(..., description="ISO 8601 timestamp when the metadata was last updated") tags: set[str] | None = Field(None, description="Optional list of tags associated with the run") submission: SubmissionMetadata = Field(..., description="Submission context metadata") user_agent: str = Field(..., description="User agent string for the SDK client") @@ -144,18 +146,23 @@ class ItemSdkMetadata(BaseModel): ..., description="Schema version for this metadata format", pattern=r"^\d+\.\d+\.\d+-?.*$" ) + created_at: str = Field(..., description="ISO 8601 timestamp when the metadata was first created") + updated_at: str = Field(..., description="ISO 8601 timestamp when the metadata was last updated") tags: set[str] | None = Field(None, description="Optional list of tags associated with the item") platform_bucket: PlatformBucketMetadata | None = Field(None, description="Platform bucket storage information") model_config = {"extra": "forbid"} # Reject unknown fields -def build_run_sdk_metadata() -> dict[str, Any]: +def build_run_sdk_metadata(existing_metadata: dict[str, Any] | None = None) -> dict[str, Any]: # noqa: PLR0914 """Build SDK metadata to attach to runs. Includes user agent, user information, GitHub CI/CD context when running in GitHub Actions, and test context when running in pytest. + Args: + existing_metadata (dict[str, Any] | None): Existing SDK metadata to preserve created_at and submission.date. + Returns: dict[str, Any]: Dictionary containing SDK metadata including user agent, user information, and optionally CI information (GitHub workflow and pytest test context). @@ -175,10 +182,22 @@ def build_run_sdk_metadata() -> dict[str, Any]: elif os.getenv("NICEGUI_HOST"): submission_interface = "launchpad" + now = datetime.now(UTC).isoformat(timespec="seconds") + existing_sdk = existing_metadata or {} + + # Preserve created_at if it exists, otherwise use current time + created_at = existing_sdk.get("created_at", now) + + # Preserve submission.date if it exists, otherwise use current time + existing_submission = existing_sdk.get("submission", {}) + submission_date = existing_submission.get("date", now) + metadata: dict[str, Any] = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": created_at, + "updated_at": now, "submission": { - "date": datetime.now(UTC).isoformat(timespec="seconds"), + "date": submission_date, "interface": submission_interface, "initiator": submission_initiator, }, @@ -289,14 +308,25 @@ def validate_run_sdk_metadata_silent(metadata: dict[str, Any]) -> bool: return False -def build_item_sdk_metadata() -> dict[str, Any]: +def build_item_sdk_metadata(existing_metadata: dict[str, Any] | None = None) -> dict[str, Any]: """Build SDK metadata to attach to individual items. + Args: + existing_metadata (dict[str, Any] | None): Existing SDK metadata to preserve created_at. + Returns: dict[str, Any]: Dictionary containing item SDK metadata including platform bucket information. """ + now = datetime.now(UTC).isoformat(timespec="seconds") + existing_sdk = existing_metadata or {} + + # Preserve created_at if it exists, otherwise use current time + created_at = existing_sdk.get("created_at", now) + metadata: dict[str, Any] = { "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": created_at, + "updated_at": now, } return metadata diff --git a/src/aignostics/platform/resources/runs.py b/src/aignostics/platform/resources/runs.py index 0b19706a6..159b627b3 100644 --- a/src/aignostics/platform/resources/runs.py +++ b/src/aignostics/platform/resources/runs.py @@ -16,6 +16,7 @@ from aignx.codegen.api.public_api import PublicApi from aignx.codegen.exceptions import ServiceException from aignx.codegen.models import ( + CustomMetadataUpdateRequest, ItemCreationRequest, ItemOutput, ItemResultReadResponse, @@ -350,6 +351,69 @@ def ensure_artifacts_downloaded( logger.debug("Results for item: %s already present in %s", item.external_id, item_dir) print(f"Results for item: {item.external_id} already present in {item_dir}") if print_status else None + def update_custom_metadata( + self, + custom_metadata: dict[str, Any], + ) -> None: + """Update custom metadata for this application run. + + Args: + custom_metadata (dict[str, Any]): The new custom metadata to attach to the run. + + Raises: + Exception: If the API request fails. + """ + custom_metadata = custom_metadata or {} + custom_metadata.setdefault("sdk", {}) + existing_sdk_metadata = custom_metadata.get("sdk", {}) + sdk_metadata = build_run_sdk_metadata(existing_sdk_metadata) + custom_metadata["sdk"].update(sdk_metadata) + validate_run_sdk_metadata(custom_metadata["sdk"]) + + self._api.put_run_custom_metadata_v1_runs_run_id_custom_metadata_put( + self.run_id, + custom_metadata_update_request=CustomMetadataUpdateRequest( + custom_metadata=cast("dict[str, Any]", convert_to_json_serializable(custom_metadata)) + ), + _request_timeout=settings().run_submit_timeout, + _headers={"User-Agent": user_agent()}, + ) + operation_cache_clear() # Clear all caches since we updated a run + + # TODO(Andreas): Always returns 404, likely connected with external_id encoding, + # see test test_cli_run_dump_and_update_item_custom_metadata + def update_item_custom_metadata( + self, + external_id: str, + custom_metadata: dict[str, Any], + ) -> None: + """Update custom metadata for an item in this application run. + + Args: + external_id (str): The external ID of the item. + custom_metadata (dict[str, Any]): The new custom metadata to attach to the item. + + Raises: + Exception: If the API request fails. + """ + custom_metadata = custom_metadata or {} + custom_metadata.setdefault("sdk", {}) + existing_sdk_metadata = custom_metadata.get("sdk", {}) + sdk_metadata = build_item_sdk_metadata(existing_sdk_metadata) + custom_metadata["sdk"].update(sdk_metadata) + validate_item_sdk_metadata(custom_metadata["sdk"]) + + self._api.put_item_custom_metadata_by_run_v1_runs_run_id_items_external_id_custom_metadata_put( + self.run_id, + external_id, + custom_metadata_update_request=CustomMetadataUpdateRequest( + custom_metadata=cast("dict[str, Any]", convert_to_json_serializable(custom_metadata)) + ), + _request_timeout=settings().run_submit_timeout, + _headers={"User-Agent": user_agent()}, + ) + operation_cache_clear() # Clear all caches since we updated a run + def __str__(self) -> str: """Returns a string representation of the application run. @@ -423,7 +487,8 @@ def submit( """ custom_metadata = custom_metadata or {} custom_metadata.setdefault("sdk", {}) - sdk_metadata = build_run_sdk_metadata() + existing_sdk_metadata = custom_metadata.get("sdk", {}) + sdk_metadata = build_run_sdk_metadata(existing_sdk_metadata) custom_metadata["sdk"].update(sdk_metadata) validate_run_sdk_metadata(custom_metadata["sdk"]) self._amend_input_items_with_sdk_metadata(items) @@ -560,11 +625,13 @@ def _amend_input_items_with_sdk_metadata(items: builtins.list[ItemCreationReques items (builtins.list[ItemCreationRequest]): The list of item creation requests to amend. """ for item in items: - item.custom_metadata = item.custom_metadata or {} - item.custom_metadata.setdefault("sdk", {}) - item_sdk_metadata = build_item_sdk_metadata() - item.custom_metadata["sdk"].update(item_sdk_metadata) - validate_item_sdk_metadata(item.custom_metadata["sdk"]) + item_custom_metadata = item.custom_metadata or {} + item_custom_metadata.setdefault("sdk", {}) + existing_item_sdk_metadata = item_custom_metadata.get("sdk") + item_sdk_metadata = build_item_sdk_metadata(existing_item_sdk_metadata) + item_custom_metadata["sdk"].update(item_sdk_metadata) + validate_item_sdk_metadata(item_custom_metadata) + item.custom_metadata = cast("dict[str, Any]", convert_to_json_serializable(item_custom_metadata)) def _validate_input_items(self, payload: RunCreationRequest) -> None: """Validates the input items in a run creation request. diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py index 5128e98d3..586d84f80 100644 --- a/tests/aignostics/application/cli_test.py +++ b/tests/aignostics/application/cli_test.py @@ -714,3 +714,242 @@ def test_cli_run_execute(runner: CliRunner, tmp_path: Path) -> None: # Validate the execute command exited successfully assert result.exit_code == 0 + + +@pytest.mark.integration +def test_cli_run_update_metadata_invalid_json(runner: CliRunner) -> None: + """Check run update-metadata command fails with invalid JSON.""" + result = runner.invoke(cli, ["application", "run", "update-metadata", "run-123", "{invalid json}"]) + assert result.exit_code == 1 + assert "Invalid JSON" in result.output + + +@pytest.mark.integration +def test_cli_run_update_metadata_not_dict(runner: CliRunner) -> None: + """Check run update-metadata command fails with non-dict JSON.""" + result = runner.invoke(cli, ["application", "run", "update-metadata", "run-123", '["array", "not", "dict"]']) + assert result.exit_code == 1 + assert "Metadata must be a JSON object" in result.output + + +@pytest.mark.integration +def test_cli_run_update_item_metadata_invalid_json(runner: CliRunner) -> None: + """Check run update-item-metadata command fails with invalid JSON.""" + result = runner.invoke( + cli, ["application", "run", "update-item-metadata", "run-123", "item-ext-id", "{invalid json}"] + ) + assert result.exit_code == 1 + assert "Invalid JSON" in result.output + + +@pytest.mark.integration +def test_cli_run_update_item_metadata_not_dict(runner: CliRunner) -> None: + """Check run update-item-metadata command fails with non-dict JSON.""" + result = runner.invoke( + cli, ["application", "run", "update-item-metadata", "run-123", "item-ext-id", '["array", "not", "dict"]'] + ) + assert result.exit_code == 1 + assert "Metadata must be a JSON object" in result.output + + +@pytest.mark.e2e +@pytest.mark.timeout(timeout=120) +@pytest.mark.sequential +def test_cli_run_dump_and_update_custom_metadata(runner: CliRunner) -> None: + """Test dumping and updating custom metadata via CLI commands.""" + import json + import random + + # Step 1: List runs, limit to 1 + result = runner.invoke(cli, ["application", "run", "list", "--limit", "1"]) + assert result.exit_code == 0 + + # Check if any runs exist + if "You did not yet create a run" in result.output: + pytest.skip("No runs available. Please run tests that submit runs first.") + + # Extract run ID from the output (format: "- of ...") + normalized_output = normalize_output(result.output) + run_id_match = re.search(r"-\s+([a-f0-9\-]{36})\s+of\s+", normalized_output) + assert run_id_match is not None, f"Could not extract run ID from list output:\n{normalized_output}" + run_id = run_id_match.group(1) + + # Step 2: Dump custom metadata of run + result = runner.invoke(cli, ["application", "run", "dump-metadata", run_id]) + assert result.exit_code == 0 + initial_metadata = json.loads(result.stdout) + # If metadata is None/null, start with empty dict + if initial_metadata is None: + initial_metadata = {} + assert isinstance(initial_metadata, dict), "Custom metadata should be a dictionary" + + # Store initial SDK metadata timestamps for comparison + initial_created_at = initial_metadata.get("sdk", {}).get("created_at") + initial_submission_date = initial_metadata.get("sdk", {}).get("submission", {}).get("date") + initial_updated_at = initial_metadata.get("sdk", {}).get("updated_at") + + # Step 3: Add "random" node with a random number + random_value = random.randint(1000, 9999) + updated_metadata = initial_metadata.copy() + updated_metadata["random"] = random_value + + # Update the custom metadata + result = runner.invoke(cli, ["application", "run", "update-metadata", run_id, json.dumps(updated_metadata)]) + assert result.exit_code == 0 + assert "Successfully updated custom metadata" in result.output + + # Step 4: Dump metadata again and verify random number appeared + result = runner.invoke(cli, ["application", "run", "dump-metadata", run_id, "--pretty"]) + assert result.exit_code == 0 + metadata_with_random = json.loads(result.stdout) + assert "random" in metadata_with_random, "Random field should be present in metadata" + assert metadata_with_random["random"] == random_value, f"Random value should be {random_value}" + + # Verify SDK metadata timestamps behavior after update + updated_created_at = metadata_with_random.get("sdk", {}).get("created_at") + updated_submission_date = metadata_with_random.get("sdk", {}).get("submission", {}).get("date") + updated_updated_at = metadata_with_random.get("sdk", {}).get("updated_at") + + # created_at and submission.date should NOT change + assert updated_created_at == initial_created_at, ( + f"sdk.created_at should not change: {initial_created_at} -> {updated_created_at}" + ) + assert updated_submission_date == initial_submission_date, ( + f"sdk.submission.date should not change: {initial_submission_date} -> {updated_submission_date}" + ) + + # updated_at SHOULD change (be more recent) + assert updated_updated_at != initial_updated_at, ( + f"sdk.updated_at should change after update: {initial_updated_at} -> {updated_updated_at}" + ) + assert updated_updated_at > initial_updated_at, ( + f"sdk.updated_at should be more recent: {initial_updated_at} -> {updated_updated_at}" + ) + + # Step 5: Remove the random number + del updated_metadata["random"] + result = runner.invoke(cli, ["application", "run", "update-metadata", run_id, json.dumps(updated_metadata)]) + assert result.exit_code == 0 + assert "Successfully updated custom metadata" in result.output + + # Step 6: Dump metadata and validate random element has been removed + result = runner.invoke(cli, ["application", "run", "dump-metadata", run_id]) + assert result.exit_code == 0 + final_metadata = json.loads(result.stdout) + assert "random" not in final_metadata, "Random field should have been removed from metadata" + + # Note: We can't compare final_metadata == initial_metadata because the SDK + # automatically updates some fields (e.g., submission.date, ci.pytest.current_test) + # when operations are performed. Instead, verify the random field was removed + # and the structure remains consistent. + assert isinstance(final_metadata, dict), "Final metadata should be a dictionary" + + +# TODO(Andreas): Update item metadata returns 404 always +@pytest.mark.skip(reason="Waiting for platform API fix to item metadata endpoint") +@pytest.mark.e2e +@pytest.mark.timeout(timeout=120) +@pytest.mark.sequential +def test_cli_run_dump_and_update_item_custom_metadata(runner: CliRunner) -> None: # noqa: PLR0914, PLR0915 # noqa: PLR0914, PLR0915 + """Test dumping and updating item custom metadata via CLI commands.""" + import json + import random + + # Step 1: List runs, limit to 1 + result = runner.invoke(cli, ["application", "run", "list", "--limit", "1"]) + assert result.exit_code == 0 + + # Check if any runs exist + if "You did not yet create a run" in result.output: + pytest.skip("No runs available. Please run tests that submit runs first.") + + # Extract run ID from the output (format: "- of ...") + normalized_output = normalize_output(result.output) + run_id_match = re.search(r"-\s+([a-f0-9\-]{36})\s+of\s+", normalized_output) + assert run_id_match is not None, f"Could not extract run ID from list output:\n{normalized_output}" + run_id = run_id_match.group(1) + + # Get run details to extract an item's external_id + result = runner.invoke(cli, ["application", "run", "describe", run_id]) + assert result.exit_code == 0 + + normalized_describe = normalize_output(result.output) + # Match the line after "Item External ID:" + external_id_match = re.search(r"Item External ID:\s*\n\s*([^\s]+)", normalized_describe) + + if not external_id_match: + # Try single line format as fallback + external_id_match = re.search(r"Item External ID:\s*([^\n\s]+)", normalized_describe) + + if not external_id_match: + pytest.skip("Could not extract item external_id from run. Run may not have items yet.") + + external_id = external_id_match.group(1).strip() + print(external_id) + + # Step 2: Dump custom metadata of item + result = runner.invoke(cli, ["application", "run", "dump-item-metadata", run_id, external_id]) + assert result.exit_code == 0 + initial_metadata = json.loads(result.output) + # If metadata is None/null, start with empty dict + if initial_metadata is None: + initial_metadata = {} + assert isinstance(initial_metadata, dict), "Custom metadata should be a dictionary" + + # Store initial SDK metadata timestamps for comparison + initial_created_at = initial_metadata.get("sdk", {}).get("created_at") + initial_updated_at = initial_metadata.get("sdk", {}).get("updated_at") + + # Step 3: Add "random" node with a random number + random_value = random.randint(1000, 9999) + updated_metadata = initial_metadata.copy() + updated_metadata["random"] = random_value + + # Update the custom metadata + result = runner.invoke( + cli, ["application", "run", "update-item-metadata", run_id, external_id, json.dumps(updated_metadata)] + ) + assert result.exit_code == 0 + assert "Successfully updated custom metadata" in result.output + + # Step 4: Dump metadata again and verify random number appeared + result = runner.invoke(cli, ["application", "run", "dump-item-metadata", run_id, external_id, "--pretty"]) + assert result.exit_code == 0 + metadata_with_random = json.loads(result.output) + assert "random" in metadata_with_random, "Random field should be present in metadata" + assert metadata_with_random["random"] == random_value, f"Random value should be {random_value}" + + # Verify SDK metadata timestamps behavior after update + updated_created_at = metadata_with_random.get("sdk", {}).get("created_at") + updated_updated_at = metadata_with_random.get("sdk", {}).get("updated_at") + + # created_at should NOT change + assert updated_created_at == initial_created_at, ( + f"sdk.created_at should not change: {initial_created_at} -> {updated_created_at}" + ) + + # updated_at SHOULD change (be more recent) + assert updated_updated_at != initial_updated_at, ( + f"sdk.updated_at should change after update: {initial_updated_at} -> {updated_updated_at}" + ) + # Step 5: Remove the random numberresult.output) + assert "random" in metadata_with_random, "Random field should be present in metadata" + assert metadata_with_random["random"] == random_value, f"Random value should be {random_value}" + + # Step 5: Remove the random number + del updated_metadata["random"] + result = runner.invoke( + cli, ["application", "run", "update-item-metadata", run_id, external_id, json.dumps(updated_metadata)] + ) + assert result.exit_code == 0 + assert "Successfully updated custom metadata" in result.output + + # Step 6: Dump metadata and validate random element has been removed + result = runner.invoke(cli, ["application", "run", "dump-item-metadata", run_id, external_id]) + assert result.exit_code == 0 + final_metadata = json.loads(result.output) + assert "random" not in final_metadata, "Random field should have been removed from metadata" + + # Note: Similar to run metadata, we verify the structure remains consistent + # rather than doing exact equality comparison due to dynamic fields + assert isinstance(final_metadata, dict), "Final metadata should be a dictionary" diff --git a/tests/aignostics/application/gui_test.py b/tests/aignostics/application/gui_test.py index 8427efd48..a8f4ba509 100644 --- a/tests/aignostics/application/gui_test.py +++ b/tests/aignostics/application/gui_test.py @@ -167,7 +167,7 @@ async def test_gui_cli_submit_to_run_result_delete(user: User, runner: CliRunner @pytest.mark.e2e @pytest.mark.long_running -# @pytest.mark.flaky(retries=1, delay=5) +@pytest.mark.flaky(retries=1, delay=5) @pytest.mark.timeout(timeout=60 * 10) @pytest.mark.sequential async def test_gui_download_dataset_via_application_to_run_cancel_to_find_back( # noqa: PLR0915 @@ -287,6 +287,7 @@ async def test_gui_download_dataset_via_application_to_run_cancel_to_find_back( await assert_notified(user, "Application run submitted with id", wait_seconds=30) # Check user is redirected to the run page and run is running + await sleep(5) await user.should_see(f"Run of he-tme ({latest_application_version.number})", retries=200) try: await user.should_see("PENDING", retries=100) diff --git a/tests/aignostics/application/service_test.py b/tests/aignostics/application/service_test.py index 9d7620192..7e3e9016f 100644 --- a/tests/aignostics/application/service_test.py +++ b/tests/aignostics/application/service_test.py @@ -355,3 +355,77 @@ def test_application_runs_query_escapes_special_characters(mock_get_client: Magi # Check that double quotes and backslashes are properly escaped assert 'test\\"value\\\\path' in note_call_kwargs["custom_metadata"] assert 'test\\"value\\\\path' in tag_call_kwargs["custom_metadata"] + + +@pytest.mark.unit +@patch("aignostics.application._service.Service._get_platform_client") +def test_application_run_update_custom_metadata_success(mock_get_client: MagicMock) -> None: + """Test successful update of run custom metadata.""" + mock_client = MagicMock() + mock_run = MagicMock() + mock_client.run.return_value = mock_run + mock_get_client.return_value = mock_client + + service = ApplicationService() + custom_metadata = {"key": "value", "tags": ["tag1", "tag2"]} + + # Should not raise any exception + service.application_run_update_custom_metadata("run-123", custom_metadata) + + # Verify the run() method was called with correct run_id + mock_client.run.assert_called_once_with("run-123") + # Verify the update_custom_metadata method was called with correct arguments + mock_run.update_custom_metadata.assert_called_once_with(custom_metadata) + + +@pytest.mark.unit +@patch("aignostics.application._service.Service._get_platform_client") +def test_application_run_update_custom_metadata_not_found(mock_get_client: MagicMock) -> None: + """Test update metadata with non-existent run.""" + mock_client = MagicMock() + mock_run = MagicMock() + mock_run.update_custom_metadata.side_effect = NotFoundException("Run not found") + mock_client.run.return_value = mock_run + mock_get_client.return_value = mock_client + + service = ApplicationService() + + with pytest.raises(NotFoundException, match="not found"): + service.application_run_update_custom_metadata("invalid-run-id", {"key": "value"}) + + +@pytest.mark.unit +@patch("aignostics.application._service.Service._get_platform_client") +def test_application_run_update_item_custom_metadata_success(mock_get_client: MagicMock) -> None: + """Test successful update of item custom metadata.""" + mock_client = MagicMock() + mock_run = MagicMock() + mock_client.run.return_value = mock_run + mock_get_client.return_value = mock_client + + service = ApplicationService() + custom_metadata = {"key": "value", "note": "test note"} + + # Should not raise any exception + service.application_run_update_item_custom_metadata("run-123", "item-ext-id", custom_metadata) + + # Verify the run() method was called with correct run_id + mock_client.run.assert_called_once_with("run-123") + # Verify the update_item_custom_metadata method was called with correct arguments + mock_run.update_item_custom_metadata.assert_called_once_with("item-ext-id", custom_metadata) + + +@pytest.mark.unit +@patch("aignostics.application._service.Service._get_platform_client") +def test_application_run_update_item_custom_metadata_not_found(mock_get_client: MagicMock) -> None: + """Test update item metadata with non-existent run or item.""" + mock_client = MagicMock() + mock_run = MagicMock() + mock_run.update_item_custom_metadata.side_effect = NotFoundException("Item not found") + mock_client.run.return_value = mock_run + mock_get_client.return_value = mock_client + + service = ApplicationService() + + with pytest.raises(NotFoundException, match="not found"): + service.application_run_update_item_custom_metadata("run-123", "invalid-item-id", {"key": "value"}) diff --git a/tests/aignostics/platform/sdk_metadata_test.py b/tests/aignostics/platform/sdk_metadata_test.py index 45064ee34..b01038ec2 100644 --- a/tests/aignostics/platform/sdk_metadata_test.py +++ b/tests/aignostics/platform/sdk_metadata_test.py @@ -442,6 +442,8 @@ def test_validate_invalid_submission_interface() -> None: """Test that invalid submission interface fails validation.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "invalid", @@ -459,6 +461,8 @@ def test_validate_invalid_submission_initiator() -> None: """Test that invalid submission initiator fails validation.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "script", @@ -476,6 +480,8 @@ def test_validate_missing_required_fields() -> None: """Test that missing required fields fail validation.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "script", @@ -493,6 +499,8 @@ def test_validate_extra_fields_rejected() -> None: """Test that extra unknown fields are rejected.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "script", @@ -511,6 +519,8 @@ def test_validate_with_tags_set() -> None: """Test validation with tags as a set of strings.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "script", @@ -528,6 +538,8 @@ def test_validate_with_empty_tags_set() -> None: """Test validation with empty tags set.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "script", @@ -545,6 +557,8 @@ def test_validate_with_tags_none() -> None: """Test validation with tags as None.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "script", @@ -562,6 +576,8 @@ def test_validate_without_tags_field() -> None: """Test validation when tags field is omitted entirely.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "script", @@ -578,6 +594,8 @@ def test_validate_with_tags_list_converted_to_set() -> None: """Test that list is automatically converted to set by Pydantic.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "script", @@ -596,6 +614,8 @@ def test_validate_with_tags_invalid_type_dict() -> None: """Test validation fails when tags is a dict instead of set.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "script", @@ -614,6 +634,8 @@ def test_validate_with_tags_non_string_values() -> None: """Test validation fails when tags contains non-string values.""" metadata = { "schema_version": SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "submission": { "date": "2025-10-19T12:00:00+00:00", "interface": "script", @@ -702,6 +724,8 @@ def test_validate_item_metadata_with_platform_bucket() -> None: """Test validation succeeds with platform bucket metadata present.""" metadata = { "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "platform_bucket": { "bucket_name": "sdk-bucket", "object_key": "runs/123/items/456", @@ -717,6 +741,8 @@ def test_validate_item_metadata_missing_platform_bucket_fields() -> None: """Test validation fails when required platform bucket fields are missing.""" metadata = { "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "platform_bucket": { "bucket_name": "sdk-bucket", "object_key": "runs/123/items/456", @@ -743,6 +769,8 @@ def test_validate_item_metadata_extra_fields() -> None: """Test that extra fields are rejected for item metadata.""" metadata = { "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "unexpected": "value", } @@ -792,6 +820,8 @@ def test_validate_item_metadata_with_tags() -> None: """Test validation of item metadata with tags as a set of strings.""" metadata = { "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "tags": {"slide", "tumor", "he-stained"}, } @@ -803,6 +833,8 @@ def test_validate_item_metadata_with_empty_tags() -> None: """Test validation of item metadata with empty tags set.""" metadata = { "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "tags": set(), } @@ -814,6 +846,8 @@ def test_validate_item_metadata_with_tags_none() -> None: """Test validation of item metadata with tags as None.""" metadata = { "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "tags": None, } @@ -825,6 +859,8 @@ def test_validate_item_metadata_without_tags() -> None: """Test validation of item metadata when tags field is omitted.""" metadata = { "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", } assert validate_item_sdk_metadata(metadata) is True @@ -835,6 +871,8 @@ def test_validate_item_metadata_tags_list_converted() -> None: """Test that list is automatically converted to set by Pydantic.""" metadata = { "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "tags": ["tag1", "tag2"], # List gets converted to set } @@ -847,6 +885,8 @@ def test_validate_item_metadata_tags_non_string() -> None: """Test validation fails when tags contains non-string values.""" metadata = { "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": "2025-10-19T12:00:00+00:00", + "updated_at": "2025-10-19T12:00:00+00:00", "tags": {"valid", 123}, # Mixed types } diff --git a/tests/aignostics/qupath/gui_test.py b/tests/aignostics/qupath/gui_test.py index 757f4f7c2..1249c0dc8 100644 --- a/tests/aignostics/qupath/gui_test.py +++ b/tests/aignostics/qupath/gui_test.py @@ -152,7 +152,7 @@ async def test_gui_qupath_install_and_launch( @pytest.mark.timeout(timeout=60 * 15) @pytest.mark.sequential async def test_gui_run_qupath_install_to_inspect( # noqa: C901, PLR0912, PLR0914, PLR0915 - user: User, runner: CliRunner, tmp_path: Path, qupath_teardown: None + user: User, runner: CliRunner, tmp_path: Path, silent_logging: None, qupath_teardown: None ) -> None: """Test installing QuPath, downloading run results, creating QuPath project from it, and inspecting results.""" # Find run From f2ee0e995f41ba09e73d36a12937cfae2cdecdc7 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 18:12:04 +0100 Subject: [PATCH 14/16] refactor(platform): introduced created_at and updated_at in sdk custom metadata both run and item level --- tests/aignostics/application/cli_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py index 586d84f80..816760d27 100644 --- a/tests/aignostics/application/cli_test.py +++ b/tests/aignostics/application/cli_test.py @@ -846,7 +846,7 @@ def test_cli_run_dump_and_update_custom_metadata(runner: CliRunner) -> None: # TODO(Andreas): Update item metadata returns 404 always -@pytest.mark.skip(reason="Waiting for platform API fix to item metadata endpoint") +@pytest.mark.skip(reason="Waiting for platform API fix to item metadata endpoint which currently returns 404 always") @pytest.mark.e2e @pytest.mark.timeout(timeout=120) @pytest.mark.sequential From e09d3b1f37469bb835a4deacb6f7f3c1535c7524 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 18:26:28 +0100 Subject: [PATCH 15/16] chore(deps): bump --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 2 - CLAUDE.md | 24 +- CLI_REFERENCE.md | 98 +++- ...dk_item_custom_metadata_schema_latest.json | 33 +- ...dk_item_custom_metadata_schema_v0.0.3.json | 89 ++++ ...sdk_run_custom_metadata_schema_latest.json | 33 +- ...sdk_run_custom_metadata_schema_v0.0.4.json | 489 ++++++++++++++++++ pyproject.toml | 10 +- src/aignostics/platform/resources/runs.py | 2 +- tests/CLAUDE.md | 4 +- uv.lock | 78 +-- 12 files changed, 804 insertions(+), 60 deletions(-) create mode 100644 docs/source/_static/sdk_item_custom_metadata_schema_v0.0.3.json create mode 100644 docs/source/_static/sdk_run_custom_metadata_schema_v0.0.4.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 520a278af..77bfd9fff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,7 +52,7 @@ repos: args: ["--baseline", ".secrets.baseline"] additional_dependencies: ["gibberish-detector"] - repo: https://github.com/astral-sh/uv-pre-commit - rev: 0.8.9 + rev: 0.9.5 hooks: - id: uv-lock - repo: local diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e369947d..5f66b6b62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1762,5 +1762,3 @@ * @helmut-hoffer-von-ankershoffen made their first contribution - - diff --git a/CLAUDE.md b/CLAUDE.md index 43cfac747..6e1726e2b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -509,20 +509,24 @@ Some modules have conditional loading based on dependencies: * 5-minute refresh buffer before expiry * OAuth 2.0 device flow -### SDK Metadata System (NEW in v1.0.0-beta.7) +### SDK Metadata System (ENHANCED - Run v0.0.4, Item v0.0.3) -**Automatic Run Tracking**: Every application run submitted through the SDK automatically includes comprehensive metadata about the execution context. +**Automatic Run & Item Tracking**: Every application run and item submitted through the SDK automatically includes comprehensive metadata about the execution context, with support for tags and timestamps. **Key Features:** -* **Automatic Attachment**: SDK metadata added to every run without user action +* **Automatic Attachment**: SDK metadata added to every run and item without user action * **Environment Detection**: Automatically detects script/CLI/GUI and user/test/bridge contexts * **CI/CD Integration**: Captures GitHub Actions workflow information and pytest test context * **User Information**: Includes authenticated user and organization details -* **Schema Validation**: Pydantic-based validation with JSON Schema (v0.0.1) -* **Versioned Schema**: Published JSON Schema at `docs/source/_static/sdk_metadata_schema_*.json` +* **Schema Validation**: Pydantic-based validation with JSON Schema (Run: v0.0.4, Item: v0.0.3) +* **Versioned Schema**: Published JSON Schema at `docs/source/_static/sdk_{run|item}_custom_metadata_schema_*.json` +* **Tags Support** (NEW): Associate runs and items with searchable tags +* **Timestamps** (NEW): Track creation and update times (`created_at`, `updated_at`) +* **Metadata Updates** (NEW): Update custom metadata via CLI and GUI +* **Item Metadata** (NEW): Separate schema for item-level metadata including platform bucket information -**What's Tracked:** +**What's Tracked (Run Level):** * Submission metadata (date, interface, initiator) * Enhanced user agent with platform and CI/CD context @@ -532,6 +536,14 @@ Some modules have conditional loading based on dependencies: * Workflow control flags (validate_only, onboard_to_portal) * Scheduling information (due dates, deadlines) * Optional user notes +* **Tags** (NEW): Set of tags for filtering (`set[str]`) +* **Timestamps** (NEW): `created_at`, `updated_at` + +**What's Tracked (Item Level - NEW):** + +* **Platform Bucket Metadata**: Cloud storage location (bucket name, object key, signed URL) +* **Tags**: Item-level tags (`set[str]`) +* **Timestamps**: `created_at`, `updated_at` **CLI Command:** diff --git a/CLI_REFERENCE.md b/CLI_REFERENCE.md index a98604891..7a9ce4519 100644 --- a/CLI_REFERENCE.md +++ b/CLI_REFERENCE.md @@ -162,7 +162,11 @@ $ aignostics application run [OPTIONS] COMMAND [ARGS]... * `submit`: Submit run by referencing the metadata CSV... * `list`: List runs. * `describe`: Describe run. +* `dump-metadata`: Dump custom metadata of a run as JSON to... +* `dump-item-metadata`: Dump custom metadata of an item as JSON to... * `cancel`: Cancel run. +* `update-metadata`: Update custom metadata for a run. +* `update-item-metadata`: Update custom metadata for an item in a run. * `result`: Download or delete run results. #### `aignostics application run execute` @@ -200,7 +204,7 @@ $ aignostics application run execute [OPTIONS] APPLICATION_ID METADATA_CSV_FILE * `--application-version TEXT`: Version of the application. If not provided, the latest version will be used. * `--create-subdirectory-for-run / --no-create-subdirectory-for-run`: Create a subdirectory for the results of the run in the destination directory [default: create-subdirectory-for-run] * `--create-subdirectory-per-item / --no-create-subdirectory-per-item`: Create a subdirectory per item in the destination directory [default: create-subdirectory-per-item] -* `--upload-prefix TEXT`: Prefix for the upload destination. If not given will be set to current milliseconds. [default: 1761393428019.061] +* `--upload-prefix TEXT`: Prefix for the upload destination. If not given will be set to current milliseconds. [default: 1761499056785.385] * `--wait-for-completion / --no-wait-for-completion`: Wait for run completion and download results incrementally [default: wait-for-completion] * `--note TEXT`: Optional note to include with the run submission via custom metadata. * `--due-date TEXT`: Optional soft due date to include with the run submission, ISO8601 format. The scheduler will try to complete the run by this date, taking the subscription tierand available GPU resources into account. @@ -263,7 +267,7 @@ $ aignostics application run upload [OPTIONS] APPLICATION_ID METADATA_CSV_FILE **Options**: * `--application-version TEXT`: Version of the application. If not provided, the latest version will be used. -* `--upload-prefix TEXT`: Prefix for the upload destination. If not given will be set to current milliseconds. [default: 1761393428019.1792] +* `--upload-prefix TEXT`: Prefix for the upload destination. If not given will be set to current milliseconds. [default: 1761499056785.492] * `--onboard-to-aignostics-portal / --no-onboard-to-aignostics-portal`: If set, the run will be onboarded to the Aignostics Portal. [default: no-onboard-to-aignostics-portal] * `--help`: Show this message and exit. @@ -291,6 +295,7 @@ $ aignostics application run submit [OPTIONS] APPLICATION_ID METADATA_CSV_FILE * `--application-version TEXT`: Version of the application to generate the metadata for. If not provided, the latest version will be used. * `--note TEXT`: Optional note to include with the run submission via custom metadata. +* `--tags TEXT`: Optional comma-separated list of tags to attach to the run for filtering. * `--due-date TEXT`: Optional soft due date to include with the run submission, ISO8601 format. The scheduler will try to complete the run by this date, taking the subscription tierand available GPU resources into account. * `--deadline TEXT`: Optional hard deadline to include with the run submission, ISO8601 format. If processing exceeds this deadline, the run can be aborted. * `--onboard-to-aignostics-portal / --no-onboard-to-aignostics-portal`: If True, onboard the run to the Aignostics Portal. [default: no-onboard-to-aignostics-portal] @@ -311,6 +316,10 @@ $ aignostics application run list [OPTIONS] * `--verbose / --no-verbose`: Show application details [default: no-verbose] * `--limit INTEGER`: Maximum number of runs to display +* `--tags TEXT`: Optional comma-separated list of tags to filter runs. All tags must match. +* `--note-regex TEXT`: Optional regex pattern to filter runs by note metadata. +* `--query TEXT`: Optional query string to filter runs by note OR tags. +* `--note-case-insensitive / --no-note-case-insensitive`: Make note regex search case-insensitive. [default: note-case-insensitive] * `--help`: Show this message and exit. #### `aignostics application run describe` @@ -331,6 +340,45 @@ $ aignostics application run describe [OPTIONS] RUN_ID * `--help`: Show this message and exit. +#### `aignostics application run dump-metadata` + +Dump custom metadata of a run as JSON to stdout. + +**Usage**: + +```console +$ aignostics application run dump-metadata [OPTIONS] RUN_ID +``` + +**Arguments**: + +* `RUN_ID`: Id of the run to dump custom metadata for [required] + +**Options**: + +* `--pretty / --no-pretty`: Pretty print JSON output with indentation [default: no-pretty] +* `--help`: Show this message and exit. + +#### `aignostics application run dump-item-metadata` + +Dump custom metadata of an item as JSON to stdout. + +**Usage**: + +```console +$ aignostics application run dump-item-metadata [OPTIONS] RUN_ID EXTERNAL_ID +``` + +**Arguments**: + +* `RUN_ID`: Id of the run containing the item [required] +* `EXTERNAL_ID`: External ID of the item to dump custom metadata for [required] + +**Options**: + +* `--pretty / --no-pretty`: Pretty print JSON output with indentation [default: no-pretty] +* `--help`: Show this message and exit. + #### `aignostics application run cancel` Cancel run. @@ -349,6 +397,45 @@ $ aignostics application run cancel [OPTIONS] RUN_ID * `--help`: Show this message and exit. +#### `aignostics application run update-metadata` + +Update custom metadata for a run. + +**Usage**: + +```console +$ aignostics application run update-metadata [OPTIONS] RUN_ID METADATA_JSON +``` + +**Arguments**: + +* `RUN_ID`: Id of the run to update [required] +* `METADATA_JSON`: Custom metadata as JSON string (e.g., '{"key": "value"}') [required] + +**Options**: + +* `--help`: Show this message and exit. + +#### `aignostics application run update-item-metadata` + +Update custom metadata for an item in a run. + +**Usage**: + +```console +$ aignostics application run update-item-metadata [OPTIONS] RUN_ID EXTERNAL_ID METADATA_JSON +``` + +**Arguments**: + +* `RUN_ID`: Id of the run containing the item [required] +* `EXTERNAL_ID`: External ID of the item to update [required] +* `METADATA_JSON`: Custom metadata as JSON string (e.g., '{"key": "value"}') [required] + +**Options**: + +* `--help`: Show this message and exit. + #### `aignostics application run result` Download or delete run results. @@ -644,9 +731,6 @@ WHERE Download from manifest file, identifier, or comma-separate set of identifiers. -Raises: - typer.Exit: If the target directory does not exist. - **Usage**: ```console @@ -655,7 +739,7 @@ $ aignostics dataset idc download [OPTIONS] SOURCE [TARGET] **Arguments**: -* `SOURCE`: Identifier or comma-separated set of identifiers. IDs matched against collection_id, PatientId, StudyInstanceUID, SeriesInstanceUID or SOPInstanceUID. [required] +* `SOURCE`: Identifier or comma-separated set of identifiers. IDs matched against collection_id, PatientId, StudyInstanceUID, SeriesInstanceUID or SOPInstanceUID. Example: 1.3.6.1.4.1.5962.99.1.1069745200.1645485340.1637452317744.2.0 [required] * `[TARGET]`: target directory for download [default: /Users/helmut/Library/Application Support/aignostics/datasets/idc] **Options**: @@ -694,7 +778,7 @@ $ aignostics dataset aignostics download [OPTIONS] SOURCE_URL [DESTINATION_DIREC **Arguments**: -* `SOURCE_URL`: URL to download, e.g. gs://aignx-storage-service-dev/sample_data_formatted/9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff [required] +* `SOURCE_URL`: URL to download. Example: gs://aignx-storage-service-dev/sample_data_formatted/9375e3ed-28d2-4cf3-9fb9-8df9d11a6627.tiff [required] * `[DESTINATION_DIRECTORY]`: Destination directory to download to [default: /Users/helmut/Library/Application Support/aignostics/datasets/aignostics] **Options**: diff --git a/docs/source/_static/sdk_item_custom_metadata_schema_latest.json b/docs/source/_static/sdk_item_custom_metadata_schema_latest.json index 38ac26686..d0fe3df00 100644 --- a/docs/source/_static/sdk_item_custom_metadata_schema_latest.json +++ b/docs/source/_static/sdk_item_custom_metadata_schema_latest.json @@ -37,6 +37,33 @@ "title": "Schema Version", "type": "string" }, + "created_at": { + "description": "ISO 8601 timestamp when the metadata was first created", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "description": "ISO 8601 timestamp when the metadata was last updated", + "title": "Updated At", + "type": "string" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional list of tags associated with the item", + "title": "Tags" + }, "platform_bucket": { "anyOf": [ { @@ -51,10 +78,12 @@ } }, "required": [ - "schema_version" + "schema_version", + "created_at", + "updated_at" ], "title": "ItemSdkMetadata", "type": "object", "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/aignostics/python-sdk/main/docs/source/_static/item_sdk_metadata_schema_v0.0.1.json" + "$id": "https://raw.githubusercontent.com/aignostics/python-sdk/main/docs/source/_static/item_sdk_metadata_schema_v0.0.3.json" } \ No newline at end of file diff --git a/docs/source/_static/sdk_item_custom_metadata_schema_v0.0.3.json b/docs/source/_static/sdk_item_custom_metadata_schema_v0.0.3.json new file mode 100644 index 000000000..d0fe3df00 --- /dev/null +++ b/docs/source/_static/sdk_item_custom_metadata_schema_v0.0.3.json @@ -0,0 +1,89 @@ +{ + "$defs": { + "PlatformBucketMetadata": { + "description": "Platform bucket storage metadata for items.", + "properties": { + "bucket_name": { + "description": "Name of the cloud storage bucket", + "title": "Bucket Name", + "type": "string" + }, + "object_key": { + "description": "Object key/path within the bucket", + "title": "Object Key", + "type": "string" + }, + "signed_download_url": { + "description": "Signed URL for downloading the object", + "title": "Signed Download Url", + "type": "string" + } + }, + "required": [ + "bucket_name", + "object_key", + "signed_download_url" + ], + "title": "PlatformBucketMetadata", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Complete Item SDK metadata schema.\n\nThis model defines the structure and validation rules for SDK metadata\nthat is attached to individual items within application runs. It includes\ninformation about where the item is stored in the platform's cloud storage.", + "properties": { + "schema_version": { + "description": "Schema version for this metadata format", + "pattern": "^\\d+\\.\\d+\\.\\d+-?.*$", + "title": "Schema Version", + "type": "string" + }, + "created_at": { + "description": "ISO 8601 timestamp when the metadata was first created", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "description": "ISO 8601 timestamp when the metadata was last updated", + "title": "Updated At", + "type": "string" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional list of tags associated with the item", + "title": "Tags" + }, + "platform_bucket": { + "anyOf": [ + { + "$ref": "#/$defs/PlatformBucketMetadata" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Platform bucket storage information" + } + }, + "required": [ + "schema_version", + "created_at", + "updated_at" + ], + "title": "ItemSdkMetadata", + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/aignostics/python-sdk/main/docs/source/_static/item_sdk_metadata_schema_v0.0.3.json" +} \ No newline at end of file diff --git a/docs/source/_static/sdk_run_custom_metadata_schema_latest.json b/docs/source/_static/sdk_run_custom_metadata_schema_latest.json index c299f8b39..674f060c6 100644 --- a/docs/source/_static/sdk_run_custom_metadata_schema_latest.json +++ b/docs/source/_static/sdk_run_custom_metadata_schema_latest.json @@ -369,7 +369,7 @@ } }, "additionalProperties": false, - "description": "Complete Run SDK metadata schema.\n\nThis model defines the structure and validation rules for SDK metadata\nthat is attached to application runs. It includes information about:\n- SDK version and submission details\n- User information (when available)\n- CI/CD environment context (GitHub Actions, pytest)\n- Workflow control flags\n- Scheduling information\n- Optional user note", + "description": "Complete Run SDK metadata schema.\n\nThis model defines the structure and validation rules for SDK metadata\nthat is attached to application runs. It includes information about:\n- SDK version and timestamps\n- User information (when available)\n- CI/CD environment context (GitHub Actions, pytest)\n- Workflow control flags\n- Scheduling information\n- Optional user note", "properties": { "schema_version": { "description": "Schema version for this metadata format", @@ -377,6 +377,33 @@ "title": "Schema Version", "type": "string" }, + "created_at": { + "description": "ISO 8601 timestamp when the metadata was first created", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "description": "ISO 8601 timestamp when the metadata was last updated", + "title": "Updated At", + "type": "string" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional list of tags associated with the run", + "title": "Tags" + }, "submission": { "$ref": "#/$defs/SubmissionMetadata", "description": "Submission context metadata" @@ -450,11 +477,13 @@ }, "required": [ "schema_version", + "created_at", + "updated_at", "submission", "user_agent" ], "title": "RunSdkMetadata", "type": "object", "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/aignostics/python-sdk/main/docs/source/_static/sdk_metadata_schema_v0.0.2.json" + "$id": "https://raw.githubusercontent.com/aignostics/python-sdk/main/docs/source/_static/sdk_metadata_schema_v0.0.4.json" } \ No newline at end of file diff --git a/docs/source/_static/sdk_run_custom_metadata_schema_v0.0.4.json b/docs/source/_static/sdk_run_custom_metadata_schema_v0.0.4.json new file mode 100644 index 000000000..674f060c6 --- /dev/null +++ b/docs/source/_static/sdk_run_custom_metadata_schema_v0.0.4.json @@ -0,0 +1,489 @@ +{ + "$defs": { + "CIMetadata": { + "description": "CI/CD environment metadata.", + "properties": { + "github": { + "anyOf": [ + { + "$ref": "#/$defs/GitHubCIMetadata" + }, + { + "type": "null" + } + ], + "default": null, + "description": "GitHub Actions metadata" + }, + "pytest": { + "anyOf": [ + { + "$ref": "#/$defs/PytestCIMetadata" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Pytest test metadata" + } + }, + "title": "CIMetadata", + "type": "object" + }, + "GitHubCIMetadata": { + "description": "GitHub Actions CI metadata.", + "properties": { + "action": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "GitHub Action name", + "title": "Action" + }, + "job": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "GitHub job name", + "title": "Job" + }, + "ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Git reference", + "title": "Ref" + }, + "ref_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Git reference name", + "title": "Ref Name" + }, + "ref_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Git reference type (branch, tag)", + "title": "Ref Type" + }, + "repository": { + "description": "Repository name (owner/repo)", + "title": "Repository", + "type": "string" + }, + "run_attempt": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Attempt number for this run", + "title": "Run Attempt" + }, + "run_id": { + "description": "Unique ID for this workflow run", + "title": "Run Id", + "type": "string" + }, + "run_number": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Run number for this workflow", + "title": "Run Number" + }, + "run_url": { + "description": "URL to the workflow run", + "title": "Run Url", + "type": "string" + }, + "runner_arch": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Runner architecture (x64, ARM64, etc.)", + "title": "Runner Arch" + }, + "runner_os": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Runner operating system", + "title": "Runner Os" + }, + "sha": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Git commit SHA", + "title": "Sha" + }, + "workflow": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Workflow name", + "title": "Workflow" + }, + "workflow_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Reference to the workflow file", + "title": "Workflow Ref" + } + }, + "required": [ + "repository", + "run_id", + "run_url" + ], + "title": "GitHubCIMetadata", + "type": "object" + }, + "PytestCIMetadata": { + "description": "Pytest test execution metadata.", + "properties": { + "current_test": { + "description": "Current test being executed", + "title": "Current Test", + "type": "string" + }, + "markers": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Pytest markers applied to the test", + "title": "Markers" + } + }, + "required": [ + "current_test" + ], + "title": "PytestCIMetadata", + "type": "object" + }, + "SchedulingMetadata": { + "description": "Scheduling metadata for run execution.", + "properties": { + "due_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Requested completion time (ISO 8601). Scheduler will try to complete before this time.", + "title": "Due Date" + }, + "deadline": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Hard deadline (ISO 8601). Run may be aborted if processing exceeds this time.", + "title": "Deadline" + } + }, + "title": "SchedulingMetadata", + "type": "object" + }, + "SubmissionMetadata": { + "description": "Metadata about how the SDK was invoked.", + "properties": { + "date": { + "description": "ISO 8601 timestamp of submission", + "title": "Date", + "type": "string" + }, + "interface": { + "description": "How the SDK was accessed (script, cli, launchpad)", + "enum": [ + "script", + "cli", + "launchpad" + ], + "title": "Interface", + "type": "string" + }, + "initiator": { + "description": "Who/what initiated the run (user, test, bridge)", + "enum": [ + "user", + "test", + "bridge" + ], + "title": "Initiator", + "type": "string" + } + }, + "required": [ + "date", + "interface", + "initiator" + ], + "title": "SubmissionMetadata", + "type": "object" + }, + "UserMetadata": { + "description": "User information metadata.", + "properties": { + "organization_id": { + "description": "User's organization ID", + "title": "Organization Id", + "type": "string" + }, + "organization_name": { + "description": "User's organization name", + "title": "Organization Name", + "type": "string" + }, + "user_email": { + "description": "User's email address", + "title": "User Email", + "type": "string" + }, + "user_id": { + "description": "User's unique ID", + "title": "User Id", + "type": "string" + } + }, + "required": [ + "organization_id", + "organization_name", + "user_email", + "user_id" + ], + "title": "UserMetadata", + "type": "object" + }, + "WorkflowMetadata": { + "description": "Workflow control metadata.", + "properties": { + "onboard_to_aignostics_portal": { + "default": false, + "description": "Whether to onboard results to the Aignostics Portal", + "title": "Onboard To Aignostics Portal", + "type": "boolean" + }, + "validate_only": { + "default": false, + "description": "Whether to only validate without running analysis", + "title": "Validate Only", + "type": "boolean" + } + }, + "title": "WorkflowMetadata", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Complete Run SDK metadata schema.\n\nThis model defines the structure and validation rules for SDK metadata\nthat is attached to application runs. It includes information about:\n- SDK version and timestamps\n- User information (when available)\n- CI/CD environment context (GitHub Actions, pytest)\n- Workflow control flags\n- Scheduling information\n- Optional user note", + "properties": { + "schema_version": { + "description": "Schema version for this metadata format", + "pattern": "^\\d+\\.\\d+\\.\\d+-?.*$", + "title": "Schema Version", + "type": "string" + }, + "created_at": { + "description": "ISO 8601 timestamp when the metadata was first created", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "description": "ISO 8601 timestamp when the metadata was last updated", + "title": "Updated At", + "type": "string" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional list of tags associated with the run", + "title": "Tags" + }, + "submission": { + "$ref": "#/$defs/SubmissionMetadata", + "description": "Submission context metadata" + }, + "user_agent": { + "description": "User agent string for the SDK client", + "title": "User Agent", + "type": "string" + }, + "user": { + "anyOf": [ + { + "$ref": "#/$defs/UserMetadata" + }, + { + "type": "null" + } + ], + "default": null, + "description": "User information (when authenticated)" + }, + "ci": { + "anyOf": [ + { + "$ref": "#/$defs/CIMetadata" + }, + { + "type": "null" + } + ], + "default": null, + "description": "CI/CD environment metadata" + }, + "note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional user note for the run", + "title": "Note" + }, + "workflow": { + "anyOf": [ + { + "$ref": "#/$defs/WorkflowMetadata" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Workflow control flags" + }, + "scheduling": { + "anyOf": [ + { + "$ref": "#/$defs/SchedulingMetadata" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Scheduling information" + } + }, + "required": [ + "schema_version", + "created_at", + "updated_at", + "submission", + "user_agent" + ], + "title": "RunSdkMetadata", + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/aignostics/python-sdk/main/docs/source/_static/sdk_metadata_schema_v0.0.4.json" +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d74400fff..cd75dc400 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,9 +74,9 @@ requires-python = ">=3.11, <3.14" dependencies = [ # From Template - "fastapi[standard,all]>=0.119.1,<1", + "fastapi[standard,all]>=0.120.0,<1", "humanize>=4.14.0,<5", - "logfire[system-metrics]>=4.14.1,<5", + "logfire[system-metrics]>=4.14.2,<5", "nicegui[native]>=3.1.0,<4", "opentelemetry-instrumentation-fastapi>=0.53b0,<1", "opentelemetry-instrumentation-httpx>=0.53b0,<1", @@ -88,7 +88,7 @@ dependencies = [ "opentelemetry-instrumentation-urllib3>=0.53b0,<1", "packaging>=25.0,<26", "platformdirs>=4.5.0,<5", - "psutil>=7.1.1,<8", + "psutil>=7.1.2,<8", "pydantic-settings>=2.11.0,<3", "pywin32>=310,<311 ; sys_platform == 'win32'", "pyyaml>=6.0.3,<7", @@ -97,7 +97,7 @@ dependencies = [ "uptime>=3.0.1,<4", # Custom "aiopath>=0.6.11,<1", - "boto3>=1.40.58,<2", + "boto3>=1.40.59,<2", "certifi>=2025.10.5,<2026", "dicom-validator>=0.7.3,<1", "dicomweb-client[gcp]>=0.59.3,<1", @@ -137,7 +137,7 @@ jupyter = ["jupyter>=1.1.1,<2"] marimo = [ "cloudpathlib>=0.23.0,<1", "ipython>=9.6.0,<10", - "marimo>=0.17.0,<1", + "marimo>=0.17.2,<1", "matplotlib>=3.10.7,<4", "shapely>=2.1.0,<3", ] diff --git a/src/aignostics/platform/resources/runs.py b/src/aignostics/platform/resources/runs.py index 159b627b3..b814d35c2 100644 --- a/src/aignostics/platform/resources/runs.py +++ b/src/aignostics/platform/resources/runs.py @@ -630,7 +630,7 @@ def _amend_input_items_with_sdk_metadata(items: builtins.list[ItemCreationReques existing_item_sdk_metadata = item_custom_metadata.get("sdk") item_sdk_metadata = build_item_sdk_metadata(existing_item_sdk_metadata) item_custom_metadata["sdk"].update(item_sdk_metadata) - validate_item_sdk_metadata(item_custom_metadata) + validate_item_sdk_metadata(item_custom_metadata["sdk"]) item.custom_metadata = cast("dict[str, Any]", convert_to_json_serializable(item_custom_metadata)) def _validate_input_items(self, payload: RunCreationRequest) -> None: diff --git a/tests/CLAUDE.md b/tests/CLAUDE.md index 36a4a94ad..f46be55e3 100644 --- a/tests/CLAUDE.md +++ b/tests/CLAUDE.md @@ -159,7 +159,7 @@ class TestBuildSdkMetadata: # Set GitHub Actions environment variables os.environ["GITHUB_RUN_ID"] = "12345" os.environ["GITHUB_REPOSITORY"] = "aignostics/python-sdk" - os.environ["GITHUB_SHA"] = "abc123def456" + os.environ["GITHUB_SHA"] = "abc123def456" # pragma: allowlist secret os.environ["GITHUB_REF"] = "refs/heads/main" os.environ["GITHUB_WORKFLOW"] = "CI/CD" @@ -170,7 +170,7 @@ class TestBuildSdkMetadata: assert "github" in metadata["ci"] assert metadata["ci"]["github"]["run_id"] == "12345" assert metadata["ci"]["github"]["repository"] == "aignostics/python-sdk" - assert metadata["ci"]["github"]["sha"] == "abc123def456" + assert metadata["ci"]["github"]["sha"] == "abc123def456" # pragma: allowlist secret assert metadata["ci"]["github"]["run_url"] == ( "https://github.com/aignostics/python-sdk/actions/runs/12345" ) diff --git a/uv.lock b/uv.lock index b21c1f70b..82445e9c9 100644 --- a/uv.lock +++ b/uv.lock @@ -160,13 +160,13 @@ dev = [ [package.metadata] requires-dist = [ { name = "aiopath", specifier = ">=0.6.11,<1" }, - { name = "boto3", specifier = ">=1.40.58,<2" }, + { name = "boto3", specifier = ">=1.40.59,<2" }, { name = "certifi", specifier = ">=2025.10.5,<2026" }, { name = "cloudpathlib", marker = "extra == 'marimo'", specifier = ">=0.23.0,<1" }, { name = "dicom-validator", specifier = ">=0.7.3,<1" }, { name = "dicomweb-client", extras = ["gcp"], specifier = ">=0.59.3,<1" }, { name = "duckdb", specifier = ">=0.10.0,<=1.4.1" }, - { name = "fastapi", extras = ["standard", "all"], specifier = ">=0.119.1,<1" }, + { name = "fastapi", extras = ["standard", "all"], specifier = ">=0.120.0,<1" }, { name = "fastparquet", specifier = ">=2024.11.0,<2025" }, { name = "google-cloud-storage", specifier = ">=3.4.1,<4" }, { name = "google-crc32c", specifier = ">=1.7.1,<2" }, @@ -180,8 +180,8 @@ requires-dist = [ { name = "jsf", specifier = ">=0.11.2,<1" }, { name = "jsonschema", extras = ["format-nongpl"], specifier = ">=4.25.1,<5" }, { name = "jupyter", marker = "extra == 'jupyter'", specifier = ">=1.1.1,<2" }, - { name = "logfire", extras = ["system-metrics"], specifier = ">=4.14.1,<5" }, - { name = "marimo", marker = "extra == 'marimo'", specifier = ">=0.17.0,<1" }, + { name = "logfire", extras = ["system-metrics"], specifier = ">=4.14.2,<5" }, + { name = "marimo", marker = "extra == 'marimo'", specifier = ">=0.17.2,<1" }, { name = "matplotlib", marker = "extra == 'marimo'", specifier = ">=3.10.7,<4" }, { name = "nicegui", extras = ["native"], specifier = ">=3.1.0,<4" }, { name = "openslide-bin", specifier = ">=4.0.0.8,<5" }, @@ -198,7 +198,7 @@ requires-dist = [ { name = "pandas", specifier = ">=2.3.3,<3" }, { name = "platformdirs", specifier = ">=4.3.8,<5" }, { name = "platformdirs", specifier = ">=4.5.0,<5" }, - { name = "psutil", specifier = ">=7.1.1,<8" }, + { name = "psutil", specifier = ">=7.1.2,<8" }, { name = "pydantic-settings", specifier = ">=2.11.0,<3" }, { name = "pyinstaller", marker = "extra == 'pyinstaller'", specifier = ">=6.14.0,<7" }, { name = "pyjwt", extras = ["crypto"], specifier = ">=2.10.1,<3" }, @@ -432,6 +432,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/3f/3bc3f1d83f6e4a7fcb834d3720544ca597590425be5ba9db032b2bf322a2/altgraph-0.17.4-py2.py3-none-any.whl", hash = "sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff", size = 21212, upload-time = "2023-09-25T09:04:50.691Z" }, ] +[[package]] +name = "annotated-doc" +version = "0.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/a6/dc46877b911e40c00d395771ea710d5e77b6de7bacd5fdcd78d70cc5a48f/annotated_doc-0.0.3.tar.gz", hash = "sha256:e18370014c70187422c33e945053ff4c286f453a984eba84d0dbfa0c935adeda", size = 5535, upload-time = "2025-10-24T14:57:10.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/b7/cf592cb5de5cb3bade3357f8d2cf42bf103bbe39f459824b4939fd212911/annotated_doc-0.0.3-py3-none-any.whl", hash = "sha256:348ec6664a76f1fd3be81f43dffbee4c7e8ce931ba71ec67cc7f4ade7fbbb580", size = 5488, upload-time = "2025-10-24T14:57:09.462Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -687,30 +696,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.40.58" +version = "1.40.59" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/de/87/7f57ac8a1cd532a3e337765d6837c8a5ac2d9d4c5e525d3aca876861f82e/boto3-1.40.58.tar.gz", hash = "sha256:5a99c0bd2e282af4afde1af10d8838b397120722b6b685f0c0fa6b8cac351304", size = 111504, upload-time = "2025-10-23T20:05:21.152Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/f4/65f3a0a58a42abaa57cb42968535dc7c209232c2614d5ac1d8354b0bc0b7/boto3-1.40.59.tar.gz", hash = "sha256:b1a5a203511e594872b39a129365f02eb5846eea990629e8daf47a3c01e7fd49", size = 111577, upload-time = "2025-10-24T19:23:33.763Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/3a/1acfc045dbaa50a1c05b86b7e5102abbac22aa297469826f82f492eacef4/boto3-1.40.58-py3-none-any.whl", hash = "sha256:951515c1ea0ae9e99e56c3b6f408a2f59e1b57fab4d96dab737e73956f729177", size = 139324, upload-time = "2025-10-23T20:05:18.579Z" }, + { url = "https://files.pythonhosted.org/packages/d4/61/25cbd486b03d4f786507584025a35ae74fa52fe2408fbf0318d91e3e96db/boto3-1.40.59-py3-none-any.whl", hash = "sha256:75752e7dc445131700a58926a50ca705794232f0f47d0e21edb59fbf1898db95", size = 139323, upload-time = "2025-10-24T19:23:31.824Z" }, ] [[package]] name = "botocore" -version = "1.40.58" +version = "1.40.59" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b4/82/26c7528126ba90c82866a35f1fbde5a5ada8f8a523822304c5bcc6a4d6c3/botocore-1.40.58.tar.gz", hash = "sha256:cf2de7f5538f23c8067408a984ed32221e8b196ce98e66945a479d06b2663c33", size = 14464507, upload-time = "2025-10-23T20:05:08.663Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/0a/4abd361449e495bc6f0eb24dc14213c1468253a5be63cfcd3b6f9feca992/botocore-1.40.59.tar.gz", hash = "sha256:842a466d8735272a30fe5b7f97df559d9e211a18e412f62a17ed249fd62f85fe", size = 14472896, upload-time = "2025-10-24T19:23:22.467Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/e8/e903eb6a96862ffb0ea96fdaca3db17346cde35869723f4dd9a17834b9da/botocore-1.40.58-py3-none-any.whl", hash = "sha256:2571ca3aec8150e1b5a597794da6fd06284de72f29d3ea806804b798755f2e5a", size = 14135190, upload-time = "2025-10-23T20:05:04.55Z" }, + { url = "https://files.pythonhosted.org/packages/50/34/72ba24f52b14669384ede828ea08927b444c52311e67e02d9cdc6f00b882/botocore-1.40.59-py3-none-any.whl", hash = "sha256:042dd844ca82155ca1ab9608b9bef36d517515c775d075f57b89257108ae843b", size = 14139459, upload-time = "2025-10-24T19:23:18.425Z" }, ] [[package]] @@ -1550,16 +1559,17 @@ wheels = [ [[package]] name = "fastapi" -version = "0.119.1" +version = "0.120.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "annotated-doc" }, { name = "pydantic" }, { name = "starlette" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/f4/152127681182e6413e7a89684c434e19e7414ed7ac0c632999c3c6980640/fastapi-0.119.1.tar.gz", hash = "sha256:a5e3426edce3fe221af4e1992c6d79011b247e3b03cc57999d697fe76cbf8ae0", size = 338616, upload-time = "2025-10-20T11:30:27.734Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/0e/7f29e8f7219e4526747db182e1afb5a4b6abc3201768fb38d81fa2536241/fastapi-0.120.0.tar.gz", hash = "sha256:6ce2c1cfb7000ac14ffd8ddb2bc12e62d023a36c20ec3710d09d8e36fab177a0", size = 337603, upload-time = "2025-10-23T20:56:34.743Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/26/e6d959b4ac959fdb3e9c4154656fc160794db6af8e64673d52759456bf07/fastapi-0.119.1-py3-none-any.whl", hash = "sha256:0b8c2a2cce853216e150e9bd4faaed88227f8eb37de21cb200771f491586a27f", size = 108123, upload-time = "2025-10-20T11:30:26.185Z" }, + { url = "https://files.pythonhosted.org/packages/1d/60/7a639ceaba54aec4e1d5676498c568abc654b95762d456095b6cb529b1ca/fastapi-0.120.0-py3-none-any.whl", hash = "sha256:84009182e530c47648da2f07eb380b44b69889a4acfd9e9035ee4605c5cfc469", size = 108243, upload-time = "2025-10-23T20:56:33.281Z" }, ] [package.optional-dependencies] @@ -2693,7 +2703,7 @@ wheels = [ [[package]] name = "logfire" -version = "4.14.1" +version = "4.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "executing" }, @@ -2704,9 +2714,9 @@ dependencies = [ { name = "rich" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/6b/9602f660ec02d70b9c738c329a2b1b6975bc8d502dc7a5e76060f05481da/logfire-4.14.1.tar.gz", hash = "sha256:7856d87d0b495f65ccf7a247dfcbe9fbd6876b4b604769f9e676bc984e058697", size = 548118, upload-time = "2025-10-22T14:02:47.062Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/89/d26951b6b21790641720c12cfd6dca0cf7ead0f5ddd7de4299837b90b8b1/logfire-4.14.2.tar.gz", hash = "sha256:8dcedbd59c3d06a8794a93bbf09add788de3b74c45afa821750992f0c822c628", size = 548291, upload-time = "2025-10-24T20:14:39.115Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/92/c8/3ddb29ca08cb352f1c09996f119f88bb9e8cdbed2c335bd064fd7256b910/logfire-4.14.1-py3-none-any.whl", hash = "sha256:3c943d5dad442a862f973d4bf2e47db5aa94ff04a45b32dfdfe6935f66b7f9ac", size = 228267, upload-time = "2025-10-22T14:02:43.759Z" }, + { url = "https://files.pythonhosted.org/packages/a7/92/4fba7b8f4f56f721ad279cb0c08164bffa14e93cfd184d1a4cc7151c52a2/logfire-4.14.2-py3-none-any.whl", hash = "sha256:caa8111b20f263f4ebb0ae380a62f2a214aeb07d5e2f03c9300fa096d0a8e692", size = 228364, upload-time = "2025-10-24T20:14:34.495Z" }, ] [package.optional-dependencies] @@ -2867,7 +2877,7 @@ wheels = [ [[package]] name = "marimo" -version = "0.17.0" +version = "0.17.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -2888,9 +2898,9 @@ dependencies = [ { name = "uvicorn" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/df/e0ac84f020d717ee257b6d0634ac56c6dbff639e1929dc3ceb7458b89b06/marimo-0.17.0.tar.gz", hash = "sha256:f634e1aedc1f4d784a1cc92e916a474b73cad02fd4b72aab502237bcf104434a", size = 33938630, upload-time = "2025-10-15T16:41:26.238Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/49/a8c37df4c2c4c4d13c50d06843be81bc86ee3b2acd7530df6e038bc71571/marimo-0.17.2.tar.gz", hash = "sha256:c95a357d688d2cd1d0235f97ea597b009e64c708fdd4760396cc9e62ca5de544", size = 33964383, upload-time = "2025-10-24T20:15:26.672Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/31/12/2e31088c1e535c7fa5234fe9a8e03954daa11e5e4f4751abb29661aa1523/marimo-0.17.0-py3-none-any.whl", hash = "sha256:3bea339b8df7ead0d7e694347ec06bd071a88cbc9dd84e00a5363c8a894ea05d", size = 34459410, upload-time = "2025-10-15T16:41:29.998Z" }, + { url = "https://files.pythonhosted.org/packages/90/66/c77eafb32d09f1d1e570cee06c4b1d265027864202de4177c8980b90fe0b/marimo-0.17.2-py3-none-any.whl", hash = "sha256:5ed1f013694e375cb524663c0930d83b8aa2bfb0e8af9400ffc078d72d253404", size = 34478909, upload-time = "2025-10-24T20:15:22.302Z" }, ] [[package]] @@ -4313,18 +4323,22 @@ sdist = { url = "https://files.pythonhosted.org/packages/f2/cf/77d3e19b7fabd0389 [[package]] name = "psutil" -version = "7.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/89/fc/889242351a932d6183eec5df1fc6539b6f36b6a88444f1e63f18668253aa/psutil-7.1.1.tar.gz", hash = "sha256:092b6350145007389c1cfe5716050f02030a05219d90057ea867d18fe8d372fc", size = 487067, upload-time = "2025-10-19T15:43:59.373Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/30/f97f8fb1f9ecfbeae4b5ca738dcae66ab28323b5cfbc96cb5565f3754056/psutil-7.1.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:8fa59d7b1f01f0337f12cd10dbd76e4312a4d3c730a4fedcbdd4e5447a8b8460", size = 244221, upload-time = "2025-10-19T15:44:03.145Z" }, - { url = "https://files.pythonhosted.org/packages/7b/98/b8d1f61ebf35f4dbdbaabadf9208282d8adc820562f0257e5e6e79e67bf2/psutil-7.1.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a95104eae85d088891716db676f780c1404fc15d47fde48a46a5d61e8f5ad2c", size = 245660, upload-time = "2025-10-19T15:44:05.657Z" }, - { url = "https://files.pythonhosted.org/packages/f0/4a/b8015d7357fefdfe34bc4a3db48a107bae4bad0b94fb6eb0613f09a08ada/psutil-7.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:98629cd8567acefcc45afe2f4ba1e9290f579eacf490a917967decce4b74ee9b", size = 286963, upload-time = "2025-10-19T15:44:08.877Z" }, - { url = "https://files.pythonhosted.org/packages/3d/3c/b56076bb35303d0733fc47b110a1c9cce081a05ae2e886575a3587c1ee76/psutil-7.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92ebc58030fb054fa0f26c3206ef01c31c29d67aee1367e3483c16665c25c8d2", size = 290118, upload-time = "2025-10-19T15:44:11.897Z" }, - { url = "https://files.pythonhosted.org/packages/dc/af/c13d360c0adc6f6218bf9e2873480393d0f729c8dd0507d171f53061c0d3/psutil-7.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:146a704f224fb2ded2be3da5ac67fc32b9ea90c45b51676f9114a6ac45616967", size = 292587, upload-time = "2025-10-19T15:44:14.67Z" }, - { url = "https://files.pythonhosted.org/packages/90/2d/c933e7071ba60c7862813f2c7108ec4cf8304f1c79660efeefd0de982258/psutil-7.1.1-cp37-abi3-win32.whl", hash = "sha256:295c4025b5cd880f7445e4379e6826f7307e3d488947bf9834e865e7847dc5f7", size = 243772, upload-time = "2025-10-19T15:44:16.938Z" }, - { url = "https://files.pythonhosted.org/packages/be/f3/11fd213fff15427bc2853552138760c720fd65032d99edfb161910d04127/psutil-7.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:9b4f17c5f65e44f69bd3a3406071a47b79df45cf2236d1f717970afcb526bcd3", size = 246936, upload-time = "2025-10-19T15:44:18.663Z" }, - { url = "https://files.pythonhosted.org/packages/0a/8d/8a9a45c8b655851f216c1d44f68e3533dc8d2c752ccd0f61f1aa73be4893/psutil-7.1.1-cp37-abi3-win_arm64.whl", hash = "sha256:5457cf741ca13da54624126cd5d333871b454ab133999a9a103fb097a7d7d21a", size = 243944, upload-time = "2025-10-19T15:44:20.666Z" }, +version = "7.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/ec/7b8e6b9b1d22708138630ef34c53ab2b61032c04f16adfdbb96791c8c70c/psutil-7.1.2.tar.gz", hash = "sha256:aa225cdde1335ff9684708ee8c72650f6598d5ed2114b9a7c5802030b1785018", size = 487424, upload-time = "2025-10-25T10:46:34.931Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/d9/b56cc9f883140ac10021a8c9b0f4e16eed1ba675c22513cdcbce3ba64014/psutil-7.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0cc5c6889b9871f231ed5455a9a02149e388fffcb30b607fb7a8896a6d95f22e", size = 238575, upload-time = "2025-10-25T10:46:38.728Z" }, + { url = "https://files.pythonhosted.org/packages/36/eb/28d22de383888deb252c818622196e709da98816e296ef95afda33f1c0a2/psutil-7.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8e9e77a977208d84aa363a4a12e0f72189d58bbf4e46b49aae29a2c6e93ef206", size = 239297, upload-time = "2025-10-25T10:46:41.347Z" }, + { url = "https://files.pythonhosted.org/packages/89/5d/220039e2f28cc129626e54d63892ab05c0d56a29818bfe7268dcb5008932/psutil-7.1.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d9623a5e4164d2220ecceb071f4b333b3c78866141e8887c072129185f41278", size = 280420, upload-time = "2025-10-25T10:46:44.122Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7a/286f0e1c167445b2ef4a6cbdfc8c59fdb45a5a493788950cf8467201dc73/psutil-7.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:364b1c10fe4ed59c89ec49e5f1a70da353b27986fa8233b4b999df4742a5ee2f", size = 283049, upload-time = "2025-10-25T10:46:47.095Z" }, + { url = "https://files.pythonhosted.org/packages/aa/cc/7eb93260794a42e39b976f3a4dde89725800b9f573b014fac142002a5c98/psutil-7.1.2-cp313-cp313t-win_amd64.whl", hash = "sha256:f101ef84de7e05d41310e3ccbdd65a6dd1d9eed85e8aaf0758405d022308e204", size = 248713, upload-time = "2025-10-25T10:46:49.573Z" }, + { url = "https://files.pythonhosted.org/packages/ab/1a/0681a92b53366e01f0a099f5237d0c8a2f79d322ac589cccde5e30c8a4e2/psutil-7.1.2-cp313-cp313t-win_arm64.whl", hash = "sha256:20c00824048a95de67f00afedc7b08b282aa08638585b0206a9fb51f28f1a165", size = 244644, upload-time = "2025-10-25T10:46:51.924Z" }, + { url = "https://files.pythonhosted.org/packages/ae/89/b9f8d47ddbc52d7301fc868e8224e5f44ed3c7f55e6d0f54ecaf5dd9ff5e/psutil-7.1.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c9ba5c19f2d46203ee8c152c7b01df6eec87d883cfd8ee1af2ef2727f6b0f814", size = 237244, upload-time = "2025-10-25T10:47:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c8/7a/8628c2f6b240680a67d73d8742bb9ff39b1820a693740e43096d5dcb01e5/psutil-7.1.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a486030d2fe81bec023f703d3d155f4823a10a47c36784c84f1cc7f8d39bedb", size = 238101, upload-time = "2025-10-25T10:47:09.523Z" }, + { url = "https://files.pythonhosted.org/packages/30/28/5e27f4d5a0e347f8e3cc16cd7d35533dbce086c95807f1f0e9cd77e26c10/psutil-7.1.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3efd8fc791492e7808a51cb2b94889db7578bfaea22df931424f874468e389e3", size = 258675, upload-time = "2025-10-25T10:47:11.082Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5c/79cf60c9acf36d087f0db0f82066fca4a780e97e5b3a2e4c38209c03d170/psutil-7.1.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e2aeb9b64f481b8eabfc633bd39e0016d4d8bbcd590d984af764d80bf0851b8a", size = 260203, upload-time = "2025-10-25T10:47:13.226Z" }, + { url = "https://files.pythonhosted.org/packages/f7/03/0a464404c51685dcb9329fdd660b1721e076ccd7b3d97dee066bcc9ffb15/psutil-7.1.2-cp37-abi3-win_amd64.whl", hash = "sha256:8e17852114c4e7996fe9da4745c2bdef001ebbf2f260dec406290e66628bdb91", size = 246714, upload-time = "2025-10-25T10:47:15.093Z" }, + { url = "https://files.pythonhosted.org/packages/6a/32/97ca2090f2f1b45b01b6aa7ae161cfe50671de097311975ca6eea3e7aabc/psutil-7.1.2-cp37-abi3-win_arm64.whl", hash = "sha256:3e988455e61c240cc879cb62a008c2699231bf3e3d061d7fce4234463fd2abb4", size = 243742, upload-time = "2025-10-25T10:47:17.302Z" }, ] [[package]] From 53dd8707ca50461ff567a3f14557f932fd85e2e9 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sun, 26 Oct 2025 19:06:08 +0100 Subject: [PATCH 16/16] docs: update --- CLAUDE.md | 29 +- src/aignostics/application/CLAUDE.md | 330 ++++++++++++++++++++--- src/aignostics/platform/CLAUDE.md | 177 +++++++++--- tests/CLAUDE.md | 250 ++++++++++++++++- tests/aignostics/application/cli_test.py | 10 + 5 files changed, 716 insertions(+), 80 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6e1726e2b..e4a63efc3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -545,20 +545,32 @@ Some modules have conditional loading based on dependencies: * **Tags**: Item-level tags (`set[str]`) * **Timestamps**: `created_at`, `updated_at` -**CLI Command:** +**CLI Commands:** ```bash -# Export SDK metadata JSON Schema -aignostics sdk metadata-schema --pretty > schema.json +# Export SDK run metadata JSON Schema +aignostics sdk metadata-schema --pretty > run_schema.json + +# Update run custom metadata (including tags) +aignostics application run custom-metadata update RUN_ID \ + --custom-metadata '{"sdk": {"tags": ["experiment-1", "batch-A"]}}' + +# Dump run custom metadata as JSON +aignostics application run custom-metadata dump RUN_ID --pretty + +# Find runs by tags +aignostics application run list --tags experiment-1,batch-A ``` **Implementation:** * Module: `platform._sdk_metadata` -* Functions: `build_sdk_metadata()`, `validate_sdk_metadata()`, `get_sdk_metadata_json_schema()` +* **Run Functions**: `build_run_sdk_metadata()`, `validate_run_sdk_metadata()`, `get_run_sdk_metadata_json_schema()` +* **Item Functions** (NEW): `build_item_sdk_metadata()`, `validate_item_sdk_metadata()`, `get_item_sdk_metadata_json_schema()` * Integration: Automatic in `platform.resources.runs.submit()` * User Agent: Enhanced `utils.user_agent()` with CI/CD context * Tests: Comprehensive test suite in `tests/aignostics/platform/sdk_metadata_test.py` +* **Schema Files**: `sdk_run_custom_metadata_schema_v0.0.4.json` and `sdk_item_custom_metadata_schema_v0.0.3.json` See `platform/CLAUDE.md` for detailed documentation. @@ -628,10 +640,19 @@ AIGNOSTICS_RUN_TIMEOUT=30.0 AIGNOSTICS_RUN_CACHE_TTL=15 ``` +**Cache Control:** + +```python +# Bypass cache for specific operations (useful in tests or when fresh data is required) +run = client.runs.details(run_id, nocache=True) # Force API call +applications = client.applications.list(nocache=True) # Bypass cache +``` + **Design Decisions:** * ✅ **Read-Only Retries**: Only safe, idempotent read operations retry * ✅ **Global Cache Clearing**: Simple consistency model - clear everything on writes +* ✅ **Cache Bypass** (NEW): `nocache=True` parameter forces fresh API calls * ✅ **Logging**: Warnings logged before retry sleeps for observability * ✅ **Re-raise**: Original exception re-raised after exhausting retries diff --git a/src/aignostics/application/CLAUDE.md b/src/aignostics/application/CLAUDE.md index 28e65e827..cc9c92310 100644 --- a/src/aignostics/application/CLAUDE.md +++ b/src/aignostics/application/CLAUDE.md @@ -48,6 +48,25 @@ Core application operations: ## Architecture & Design Patterns +### Module Structure (NEW in v1.0.0-beta.7) + +The application module is organized into focused submodules: + +``` +application/ +├── _service.py # High-level orchestration and API integration +├── _models.py # Data models (DownloadProgress, DownloadProgressState) [NEW] +├── _download.py # Download helpers with progress tracking [NEW] +├── _utils.py # Shared utilities +├── _cli.py # CLI commands +└── _gui/ # GUI components +``` + +**Key Separation:** +- **_models.py**: Pydantic models for progress tracking with computed fields +- **_download.py**: Pure download logic (URLs, artifacts, progress callbacks) +- **_service.py**: High-level business logic and module integration + ### Service Layer Architecture ``` @@ -56,6 +75,7 @@ Core application operations: │ (High-Level Orchestration) │ ├────────────────────────────────────────────┤ │ Progress Tracking & State Management │ +│ (_models.py - NEW) │ ├────────────────────────────────────────────┤ │ Integration Layer │ │ ┌──────────┬───────────┬──────────┐ │ @@ -65,6 +85,7 @@ Core application operations: ├────────────────────────────────────────────┤ │ File Processing Layer │ │ (Upload, Download, Verification) │ +│ (_download.py - NEW) │ └────────────────────────────────────────────┘ ``` @@ -143,10 +164,12 @@ APPLICATION_RUN_DOWNLOAD_SLEEP_SECONDS = 5 # Wait between status checks ### Progress State Management -**Actual DownloadProgress Model:** +**Actual DownloadProgress Model (`_models.py`):** ```python class DownloadProgress(BaseModel): + """Model for tracking download progress with computed progress metrics.""" + # Core state status: DownloadProgressState = DownloadProgressState.INITIALIZING @@ -167,6 +190,13 @@ class DownloadProgress(BaseModel): artifact_downloaded_chunk_size: int = 0 # Last chunk size artifact_downloaded_size: int = 0 # Total downloaded + # Input slide tracking (NEW in v1.0.0-beta.7) + input_slide_path: Path | None = None + input_slide_url: str | None = None + input_slide_size: int | None = None + input_slide_downloaded_chunk_size: int = 0 + input_slide_downloaded_size: int = 0 + # QuPath integration (conditional) if has_qupath_extra: qupath_add_input_progress: QuPathAddProgress | None = None @@ -176,15 +206,42 @@ class DownloadProgress(BaseModel): @computed_field @property def total_artifact_count(self) -> int | None: + """Calculate total number of artifacts across all items.""" if self.item_count and self.artifact_count: return self.item_count * self.artifact_count return None + @computed_field + @property + def total_artifact_index(self) -> int | None: + """Calculate the current artifact index across all items.""" + if self.item_count and self.artifact_count and self.item_index is not None and self.artifact_index is not None: + return self.item_index * self.artifact_count + self.artifact_index + return None + @computed_field @property def item_progress_normalized(self) -> float: - """Normalized progress 0..1 across all items.""" - # Implementation details... + """Normalized progress 0..1 across all items. + + Handles different progress states: + - DOWNLOADING_INPUT: Progress through items being downloaded + - DOWNLOADING: Progress through artifacts being downloaded + - QUPATH_*: QuPath-specific progress tracking + """ + # Implementation varies by state... + + @computed_field + @property + def artifact_progress_normalized(self) -> float: + """Normalized progress 0..1 for current artifact/input download. + + Handles different download types: + - DOWNLOADING_INPUT: Input slide download progress + - DOWNLOADING: Artifact download progress + - QUPATH_ANNOTATE: QuPath annotation progress + """ + # Implementation varies by state... ``` ### QuPath Integration (Conditional Loading) @@ -209,11 +266,13 @@ def process_with_qupath(self, ...): # QuPath processing... ``` -**Download Progress States:** +**Download Progress States (`_models.py`):** ```python class DownloadProgressState(StrEnum): + """Enum for download progress states.""" INITIALIZING = "Initializing ..." + DOWNLOADING_INPUT = "Downloading input slide ..." # NEW in v1.0.0-beta.7 QUPATH_ADD_INPUT = "Adding input slides to QuPath project ..." CHECKING = "Checking run status ..." WAITING = "Waiting for item completing ..." @@ -223,6 +282,146 @@ class DownloadProgressState(StrEnum): COMPLETED = "Completed." ``` +### Download Module (`_download.py` - NEW in v1.0.0-beta.7) + +The download module provides reusable download helper functions with comprehensive progress tracking. + +**Key Functions:** + +```python +def extract_filename_from_url(url: str) -> str: + """Extract a filename from a URL robustly. + + Supports: + - gs:// (Google Cloud Storage) + - http:// and https:// URLs + - Handles query parameters and trailing slashes + - Sanitizes filenames for filesystem use + + Examples: + >>> extract_filename_from_url("gs://bucket/path/to/file.tiff") + 'file.tiff' + >>> extract_filename_from_url("https://example.com/slides/sample.svs?token=abc") + 'sample.svs' + """ + +def download_url_to_file_with_progress( + progress: DownloadProgress, + url: str, + destination_path: Path, + download_progress_queue: Any | None = None, + download_progress_callable: Callable | None = None, +) -> Path: + """Download a file from a URL (gs://, http://, or https://) with progress tracking. + + Features: + - Converts gs:// URLs to signed URLs automatically + - Streams downloads with 1MB chunks (APPLICATION_RUN_DOWNLOAD_CHUNK_SIZE) + - Updates progress on every chunk + - Supports both queue and callback progress updates + - Creates parent directories automatically + + Args: + progress: Progress tracking object (updated in place) + url: URL to download (gs://, http://, https://) + destination_path: Local file path to save to + download_progress_queue: Optional queue for GUI progress updates + download_progress_callable: Optional callback for CLI progress updates + + Returns: + Path: The path to the downloaded file + + Raises: + ValueError: If URL scheme is unsupported + RuntimeError: If download fails + """ + +def download_available_items( + progress: DownloadProgress, + application_run: Run, + destination_directory: Path, + downloaded_items: set[str], + create_subdirectory_per_item: bool = False, + download_progress_queue: Any | None = None, + download_progress_callable: Callable | None = None, +) -> None: + """Download items that are available and not yet downloaded. + + Features: + - Only downloads TERMINATED items with FULL output + - Skips already downloaded items (tracked via external_id) + - Optional subdirectory per item + - Progress tracking for each item and artifact + + Args: + progress: Progress tracking object + application_run: Run object with results + destination_directory: Directory to save files + downloaded_items: Set of already downloaded external_ids + create_subdirectory_per_item: Create item subdirectories + download_progress_queue: Optional queue for GUI updates + download_progress_callable: Optional callback for CLI updates + """ + +def download_item_artifact( + progress: DownloadProgress, + artifact: Any, + destination_directory: Path, + prefix: str = "", + download_progress_queue: Any | None = None, + download_progress_callable: Callable | None = None, +) -> None: + """Download an artifact of a result item with progress tracking. + + Features: + - CRC32C checksum verification + - Skips download if file exists with correct checksum + - Automatic file extension detection + - Chunked downloads with progress updates + + Raises: + ValueError: If no checksum metadata found or checksum mismatch + requests.HTTPError: If download fails + """ +``` + +**Constants:** + +```python +# From _download.py +APPLICATION_RUN_FILE_READ_CHUNK_SIZE = 1024 * 1024 * 1024 # 1GB (for checksum calculation) +APPLICATION_RUN_DOWNLOAD_CHUNK_SIZE = 1024 * 1024 # 1MB (for streaming downloads) +``` + +**URL Support:** + +The download module supports three URL schemes: +1. **gs://** - Google Cloud Storage (converted to signed URLs via `platform.generate_signed_url()`) +2. **http://** - HTTP URLs (used directly) +3. **https://** - HTTPS URLs (used directly) + +**Progress Update Pattern:** + +```python +def update_progress( + progress: DownloadProgress, + download_progress_callable: Callable | None = None, + download_progress_queue: Any | None = None, +) -> None: + """Update download progress via callback or queue. + + Dual update mechanism: + - Callback: Synchronous update (CLI, blocking) + - Queue: Asynchronous update (GUI, non-blocking) + + Both can be used simultaneously. + """ + if download_progress_callable: + download_progress_callable(progress) + if download_progress_queue: + download_progress_queue.put_nowait(progress) +``` + ## Usage Patterns & Best Practices ### Basic Application Execution @@ -292,29 +491,73 @@ def upload_file(self, file_path: Path, signed_url: str): ### Download with Progress (Actual Pattern) +**Basic download with progress callback:** + ```python -def download_artifact(self, url: str, output_path: Path, progress_callback): - """Download with progress tracking.""" - - response = requests.get(url, stream=True) - total_size = int(response.headers.get("Content-Length", 0)) - - downloaded = 0 - with output_path.open("wb") as f: - for chunk in response.iter_content(chunk_size=APPLICATION_RUN_DOWNLOAD_CHUNK_SIZE): - f.write(chunk) - downloaded += len(chunk) - - # Update progress - progress = DownloadProgress( - status=DownloadProgressState.DOWNLOADING, - artifact_downloaded_chunk_size=len(chunk), - artifact_downloaded_size=downloaded, - artifact_size=total_size - ) +from aignostics.application._download import download_url_to_file_with_progress +from aignostics.application._models import DownloadProgress +from pathlib import Path + +# Create progress object +progress = DownloadProgress() + +# Define progress callback +def on_progress(p: DownloadProgress): + if p.input_slide_size: + percent = (p.input_slide_downloaded_size / p.input_slide_size) * 100 + print(f"Downloaded: {percent:.1f}%") + +# Download from gs://, http://, or https:// +downloaded_file = download_url_to_file_with_progress( + progress=progress, + url="gs://my-bucket/slides/sample.svs", + destination_path=Path("./downloads/sample.svs"), + download_progress_callable=on_progress +) - if progress_callback: - progress_callback(progress) +print(f"Downloaded to: {downloaded_file}") +``` + +**Download with GUI queue (non-blocking):** + +```python +from queue import Queue +from aignostics.application._download import download_url_to_file_with_progress + +# Create queue for GUI updates +progress_queue = Queue() + +# Download in background thread +download_url_to_file_with_progress( + progress=DownloadProgress(), + url="https://example.com/slide.tiff", + destination_path=Path("./slide.tiff"), + download_progress_queue=progress_queue # Non-blocking updates +) + +# In GUI thread, poll queue +while not progress_queue.empty(): + progress = progress_queue.get() + ui.update(progress.artifact_progress_normalized) +``` + +**Download application run results:** + +```python +from aignostics.application._download import download_available_items + +# Track downloaded items to avoid re-downloading +downloaded_items = set() + +# Download all available items +download_available_items( + progress=DownloadProgress(), + application_run=run, + destination_directory=Path("./results"), + downloaded_items=downloaded_items, + create_subdirectory_per_item=True, # Create dirs per item + download_progress_callable=lambda p: print(f"Item {p.item_index}/{p.item_count}") +) ``` ## Testing Strategies (Actual Test Patterns) @@ -480,31 +723,44 @@ with open(file_path, 'rb') as f: ## Module Dependencies -### Internal Dependencies +### Internal Module Organization (NEW in v1.0.0-beta.7) + +The application module is organized into focused submodules: -- `platform` - Client, API operations, and **SDK metadata system** (automatic attachment to all runs) +- **`_service.py`** - High-level orchestration, API integration, run lifecycle management +- **`_models.py`** - Data models (DownloadProgress, DownloadProgressState) +- **`_download.py`** - Download helpers with progress tracking and checksum verification +- **`_utils.py`** - Shared utilities +- **`_cli.py`** - CLI commands +- **`_gui/`** - GUI components (page builders, reactive components) + +### Internal SDK Dependencies + +- `platform` - Client, API operations, **SDK metadata system** (automatic attachment to all runs), signed URLs - `wsi` - WSI file validation - `bucket` - Cloud storage operations - `qupath` - Analysis integration (optional, requires ijson) -- `utils` - Logging and utilities +- `utils` - Logging, sanitization, and base utilities **SDK Metadata Integration:** -Every run submitted through the application module automatically includes SDK metadata from `platform._sdk_metadata.build_sdk_metadata()`. This provides: -- Execution context (script vs CLI vs GUI) -- User and organization information (when authenticated) -- CI/CD environment details (GitHub Actions, pytest) -- Workflow control flags and scheduling information -- User agent with test/CI context +Every run submitted through the application module automatically includes SDK metadata: +- **Run metadata** (v0.0.4): Execution context, user info, CI/CD details, tags, timestamps +- **Item metadata** (v0.0.3): Platform bucket location, tags, timestamps +- Automatic attachment via `platform._sdk_metadata.build_run_sdk_metadata()` See `platform/CLAUDE.md` for detailed SDK metadata documentation and schema. +**Signed URL Generation:** + +The `_download.py` module uses `platform.generate_signed_url()` to convert `gs://` URLs to time-limited signed URLs for downloads. + ### External Dependencies - `semver` - Semantic version validation (using `Version.is_valid()`) -- `google-crc32c` - File integrity checking -- `requests` - HTTP operations -- `pydantic` - Data models with validation +- `google-crc32c` - File integrity checking (CRC32C checksums) +- `requests` - HTTP operations (streaming downloads) +- `pydantic` - Data models with validation and computed fields - `ijson` - Required for QuPath features (optional) ## Performance Notes diff --git a/src/aignostics/platform/CLAUDE.md b/src/aignostics/platform/CLAUDE.md index 606aebae4..7d59f9347 100644 --- a/src/aignostics/platform/CLAUDE.md +++ b/src/aignostics/platform/CLAUDE.md @@ -231,7 +231,7 @@ class Runs: ### SDK Metadata System (`_sdk_metadata.py`) -**NEW FEATURE (as of v1.0.0-beta.7):** The SDK now automatically attaches structured metadata to every application run, providing comprehensive tracking of execution context, user information, and CI/CD environment details. +**ENHANCED FEATURE:** The SDK now automatically attaches structured metadata to every application run and item, providing comprehensive tracking of execution context, user information, CI/CD environment details, tags, and timestamps. **Architecture:** @@ -240,27 +240,38 @@ class Runs: │ SDK Metadata System │ ├────────────────────────────────────────────────────┤ │ Pydantic Models (Validation + Schema Generation) │ -│ ├─ SdkMetadata (root model) │ -│ ├─ SubmissionMetadata (how/when submitted) │ -│ ├─ UserMetadata (organization/user info) │ -│ ├─ CIMetadata (GitHub Actions + pytest) │ -│ ├─ WorkflowMetadata (control flags) │ -│ └─ SchedulingMetadata (due dates/deadlines) │ +│ ├─ RunSdkMetadata (run-level metadata) │ +│ │ ├─ SubmissionMetadata (how/when submitted) │ +│ │ ├─ UserMetadata (organization/user info) │ +│ │ ├─ CIMetadata (GitHub Actions + pytest) │ +│ │ ├─ WorkflowMetadata (control flags) │ +│ │ ├─ SchedulingMetadata (due dates/deadlines) │ +│ │ ├─ tags (set[str]) - NEW │ +│ │ ├─ created_at (timestamp) - NEW │ +│ │ └─ updated_at (timestamp) - NEW │ +│ └─ ItemSdkMetadata (item-level metadata) - NEW │ +│ ├─ PlatformBucketMetadata (storage info) │ +│ ├─ tags (set[str]) │ +│ ├─ created_at (timestamp) │ +│ └─ updated_at (timestamp) │ ├────────────────────────────────────────────────────┤ │ Runtime Functions │ -│ ├─ build_sdk_metadata() → dict │ -│ ├─ validate_sdk_metadata(metadata) → bool │ -│ ├─ validate_sdk_metadata_silent(metadata) → bool │ -│ └─ get_sdk_metadata_json_schema() → dict │ +│ ├─ build_run_sdk_metadata() → dict │ +│ ├─ validate_run_sdk_metadata() → bool │ +│ ├─ get_run_sdk_metadata_json_schema() → dict │ +│ ├─ build_item_sdk_metadata() → dict - NEW │ +│ ├─ validate_item_sdk_metadata() → bool - NEW │ +│ └─ get_item_sdk_metadata_json_schema() → dict │ ├────────────────────────────────────────────────────┤ │ JSON Schema (Versioned) │ -│ └─ Schema version: 0.0.1 │ +│ ├─ Run schema version: 0.0.4 │ +│ └─ Item schema version: 0.0.3 │ │ Published at: docs/source/_static/ │ -│ URL: github.com/.../sdk_metadata_schema_*.json │ +│ URLs: sdk_{run|item}_custom_metadata_schema_* │ └────────────────────────────────────────────────────┘ ``` -**Schema Version:** `0.0.1` +**Schema Versions:** Run `0.0.4`, Item `0.0.3` **Core Pydantic Models:** @@ -318,9 +329,12 @@ class SchedulingMetadata(BaseModel): due_date: str | None # ISO 8601, requested completion time deadline: str | None # ISO 8601, hard deadline -class SdkMetadata(BaseModel): - """Complete SDK metadata schema.""" - schema_version: str # Currently "0.0.1" +class RunSdkMetadata(BaseModel): + """Complete Run SDK metadata schema.""" + schema_version: str # Currently "0.0.4" + created_at: str # ISO 8601 timestamp - NEW + updated_at: str # ISO 8601 timestamp - NEW + tags: set[str] | None # Optional tags - NEW submission: SubmissionMetadata user_agent: str # Enhanced user agent from utils module user: UserMetadata | None # Present if authenticated @@ -329,13 +343,29 @@ class SdkMetadata(BaseModel): workflow: WorkflowMetadata | None # Optional workflow control scheduling: SchedulingMetadata | None # Optional scheduling info + model_config = {"extra": "forbid"} # Strict validation + +class PlatformBucketMetadata(BaseModel): + """Platform bucket storage metadata for items - NEW""" + bucket_name: str # Name of the cloud storage bucket + object_key: str # Object key/path within the bucket + signed_download_url: str # Signed URL for downloading + +class ItemSdkMetadata(BaseModel): + """Complete Item SDK metadata schema - NEW""" + schema_version: str # Currently "0.0.3" + created_at: str # ISO 8601 timestamp + updated_at: str # ISO 8601 timestamp + tags: set[str] | None # Optional item-level tags + platform_bucket: PlatformBucketMetadata | None # Storage location + model_config = {"extra": "forbid"} # Strict validation ``` **Automatic Metadata Generation:** ```python -def build_sdk_metadata() -> dict[str, Any]: +def build_run_sdk_metadata(existing_metadata: dict[str, Any] | None = None) -> dict[str, Any]: """Build SDK metadata automatically attached to runs. Detection Logic: @@ -344,9 +374,13 @@ def build_sdk_metadata() -> dict[str, Any]: - User info: Fetches from Client().me() if authenticated - GitHub CI: Reads GITHUB_* environment variables - Pytest: Reads PYTEST_CURRENT_TEST environment variable + - Preserves created_at and submission.date from existing metadata + + Args: + existing_metadata: Existing SDK metadata to preserve timestamps Returns: - dict with complete metadata structure + dict with complete metadata structure including timestamps """ # Interface detection if "typer" in sys.argv[0] or "aignostics" in sys.argv[0]: @@ -356,21 +390,32 @@ def build_sdk_metadata() -> dict[str, Any]: else: interface = "script" - # Source detection + # Source detection (initiator) if os.environ.get("AIGNOSTICS_BRIDGE_VERSION"): - source = "bridge" + initiator = "bridge" elif os.environ.get("PYTEST_CURRENT_TEST"): - source = "test" + initiator = "test" else: - source = "user" + initiator = "user" + + # Handle timestamps - preserve created_at, always update updated_at + now = datetime.now(UTC).isoformat(timespec="seconds") + existing_sdk = existing_metadata or {} + created_at = existing_sdk.get("created_at", now) + + # Preserve submission.date from existing metadata + existing_submission = existing_sdk.get("submission", {}) + submission_date = existing_submission.get("date", now) # Build metadata structure metadata = { - "schema_version": "0.0.1", + "schema_version": "0.0.4", + "created_at": created_at, # NEW + "updated_at": now, # NEW "submission": { - "date": datetime.now(UTC).isoformat(timespec="seconds"), + "date": submission_date, # Preserved from existing "interface": interface, - "source": source, + "initiator": initiator, # Changed from "source" }, "user_agent": user_agent(), # From utils module } @@ -476,44 +521,79 @@ def _generate_sdk_metadata_schema(session: nox.Session) -> None: **Validation Functions:** ```python -def validate_sdk_metadata(metadata: dict[str, Any]) -> bool: - """Validate SDK metadata and raise ValidationError if invalid.""" +def validate_run_sdk_metadata(metadata: dict[str, Any]) -> bool: + """Validate Run SDK metadata and raise ValidationError if invalid.""" try: - SdkMetadata.model_validate(metadata) + RunSdkMetadata.model_validate(metadata) return True except ValidationError: logger.exception("SDK metadata validation failed") raise -def validate_sdk_metadata_silent(metadata: dict[str, Any]) -> bool: - """Validate SDK metadata without raising exceptions.""" +def validate_run_sdk_metadata_silent(metadata: dict[str, Any]) -> bool: + """Validate Run SDK metadata without raising exceptions.""" try: - SdkMetadata.model_validate(metadata) + RunSdkMetadata.model_validate(metadata) return True except ValidationError: return False -def get_sdk_metadata_json_schema() -> dict[str, Any]: - """Get JSON Schema for SDK metadata with $schema and $id fields.""" - schema = SdkMetadata.model_json_schema() +def get_run_sdk_metadata_json_schema() -> dict[str, Any]: + """Get JSON Schema for Run SDK metadata with $schema and $id fields.""" + schema = RunSdkMetadata.model_json_schema() + schema["$schema"] = "https://json-schema.org/draft/2020-12/schema" + schema["$id"] = ( + f"https://raw.githubusercontent.com/aignostics/python-sdk/main/" + f"docs/source/_static/sdk_run_custom_metadata_schema_v{SDK_METADATA_SCHEMA_VERSION}.json" + ) + return schema + +def build_item_sdk_metadata(existing_metadata: dict[str, Any] | None = None) -> dict[str, Any]: + """Build SDK metadata to attach to individual items - NEW""" + now = datetime.now(UTC).isoformat(timespec="seconds") + existing_sdk = existing_metadata or {} + created_at = existing_sdk.get("created_at", now) + + return { + "schema_version": ITEM_SDK_METADATA_SCHEMA_VERSION, + "created_at": created_at, + "updated_at": now, + } + +def validate_item_sdk_metadata(metadata: dict[str, Any]) -> bool: + """Validate Item SDK metadata - NEW""" + try: + ItemSdkMetadata.model_validate(metadata) + return True + except ValidationError: + logger.exception("Item SDK metadata validation failed") + raise + +def get_item_sdk_metadata_json_schema() -> dict[str, Any]: + """Get JSON Schema for Item SDK metadata - NEW""" + schema = ItemSdkMetadata.model_json_schema() schema["$schema"] = "https://json-schema.org/draft/2020-12/schema" schema["$id"] = ( f"https://raw.githubusercontent.com/aignostics/python-sdk/main/" - f"docs/source/_static/sdk_metadata_schema_v{SDK_METADATA_SCHEMA_VERSION}.json" + f"docs/source/_static/sdk_item_custom_metadata_schema_v{ITEM_SDK_METADATA_SCHEMA_VERSION}.json" ) return schema ``` **Key Features:** -1. **Automatic Attachment** - SDK metadata added to every run submission without user action +1. **Automatic Attachment** - SDK metadata added to every run and item submission without user action 2. **Environment Detection** - Automatically detects script/CLI/GUI, user/test/bridge contexts 3. **CI/CD Integration** - Captures GitHub Actions workflow details and pytest test context 4. **User Agent Integration** - Uses enhanced user_agent() from utils module 5. **Strict Validation** - Pydantic models with `extra="forbid"` ensure data quality -6. **Versioned Schema** - JSON Schema published with semantic versioning +6. **Versioned Schema** - JSON Schema published with semantic versioning (Run: v0.0.4, Item: v0.0.3) 7. **Silent Fallback** - User info and CI data are optional, won't fail if unavailable 8. **Custom Metadata Support** - Users can add custom fields alongside SDK metadata +9. **Tags Support** (NEW) - Associate runs and items with searchable tags (`set[str]`) +10. **Timestamps** (NEW) - Track `created_at` (first submission) and `updated_at` (last modification) +11. **Item Metadata** (NEW) - Separate schema for item-level metadata with platform bucket information +12. **Metadata Updates** (NEW) - Update metadata via CLI (`aignostics application run custom-metadata update`) **Testing:** @@ -717,6 +797,27 @@ def delete(self) -> None: - ✅ `Runs.results()` - Run results (15 sec TTL) - ✅ `Runs.list()` - Run list (15 sec TTL) +**Cache Bypass (NEW):** + +All cached operations now support a `nocache=True` parameter to force fresh API calls: + +```python +# Bypass cache for specific operations +run = client.runs.details(run_id, nocache=True) # Force API call +applications = client.applications.list(nocache=True) # Bypass cache +me = client.me(nocache=True) # Fresh user info + +# Useful in tests to avoid race conditions +def test_run_update(): + run = client.runs.details(run_id, nocache=True) # Always fresh + assert run.output.state == RunState.PROCESSING +``` + +The `nocache` parameter is particularly useful in: +- **Testing**: Avoid race conditions from stale cached data +- **Real-time monitoring**: Ensure latest status in dashboards +- **After mutations**: Get fresh data immediately after updates + **Operations That Clear Cache:** - ❌ `Runs.submit()` - Creates new run diff --git a/tests/CLAUDE.md b/tests/CLAUDE.md index f46be55e3..8c98802a0 100644 --- a/tests/CLAUDE.md +++ b/tests/CLAUDE.md @@ -101,7 +101,7 @@ def test_application_version_formats(): ### SDK Metadata Testing (`platform/sdk_metadata_test.py`) -**NEW FEATURE TESTS (v1.0.0-beta.7):** Comprehensive testing of the SDK metadata system ensuring robust tracking and validation. +**ENHANCED FEATURE TESTS (Run v0.0.4, Item v0.0.3):** Comprehensive testing of the SDK metadata system with separate Run and Item metadata schemas, tags support, and timestamps. **Test Coverage:** @@ -348,8 +348,256 @@ Tested in `application/service_test.py` and `application/cli_test.py` to ensure 6. **Optional Fields**: Verify system works with missing optional fields 7. **Error Cases**: Test validation catches all invalid inputs +### Cache Bypass Testing (`platform/nocache_test.py` - NEW in v1.0.0-beta.7) + +**Comprehensive testing of the nocache parameter** for cache bypass functionality across all cached operations. + +**Test Coverage:** + +1. **Decorator Behavior Tests** - Verify @cached_operation decorator handles nocache correctly +2. **Client Method Tests** - Test nocache on Client.me(), Client.application(), Client.application_version() +3. **Resource Method Tests** - Test nocache on Runs.list(), Run.details(), Applications.list() +4. **Edge Case Tests** - Expired cache entries, multiple consecutive nocache calls, interleaved usage +5. **Cache Clear Integration** - Test interaction between nocache and cache clearing + +**Core Testing Principles:** + +```python +class TestNocacheDecoratorBehavior: + """Test the nocache parameter handling in the cached_operation decorator.""" + + def test_decorator_without_nocache_uses_cache() -> None: + """Verify default behavior uses cache.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # First call - executes function + result1 = test_func() + assert result1 == 1 + assert call_count == 1 + + # Second call - uses cache + result2 = test_func() + assert result2 == 1 # Same value from cache + assert call_count == 1 # Function NOT called again + + def test_decorator_with_nocache_true_skips_reading_cache() -> None: + """Verify nocache=True skips cache read.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # First call - populates cache + result1 = test_func() + assert result1 == 1 + + # Second call with nocache=True - skips cache, executes function + result2 = test_func(nocache=True) + assert result2 == 2 # NEW value, not from cache + assert call_count == 2 # Function called again + + def test_decorator_with_nocache_true_still_writes_to_cache() -> None: + """Verify nocache=True still writes result to cache.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # First call - populates cache + result1 = test_func() + assert result1 == 1 + + # Second call with nocache=True - skips read, writes new value + result2 = test_func(nocache=True) + assert result2 == 2 + + # Third call without nocache - uses value cached by second call + result3 = test_func() + assert result3 == 2 # Uses value from second call + assert call_count == 2 # Function NOT called again + + def test_decorator_nocache_parameter_not_passed_to_function() -> None: + """Verify nocache is intercepted and not passed to decorated function.""" + received_kwargs = {} + + @cached_operation(ttl=60, use_token=False) + def test_func(**kwargs: bool) -> dict: + nonlocal received_kwargs + received_kwargs = kwargs + return {"called": True} + + # Call with nocache=True + test_func(nocache=True) + + # The decorated function should NOT receive nocache in kwargs + assert "nocache" not in received_kwargs +``` + +**Client Method Testing:** + +```python +class TestClientMeNocache: + """Test nocache parameter for Client.me() method.""" + + def test_me_default_uses_cache( + client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Verify me() uses cache by default.""" + mock_me_response = {"user_id": "test-user", "org_id": "test-org"} + mock_api_client.get_me_v1_me_get.return_value = mock_me_response + + # First call + result1 = client_with_mock_api.me() + assert result1 == mock_me_response + assert mock_api_client.get_me_v1_me_get.call_count == 1 + + # Second call - should use cache + result2 = client_with_mock_api.me() + assert result2 == mock_me_response + assert mock_api_client.get_me_v1_me_get.call_count == 1 # No additional call + + def test_me_nocache_true_fetches_fresh_data( + client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Verify me(nocache=True) fetches fresh data.""" + mock_me_response_1 = {"user_id": "user-1"} + mock_me_response_2 = {"user_id": "user-2"} + + # First call - populates cache + mock_api_client.get_me_v1_me_get.return_value = mock_me_response_1 + result1 = client_with_mock_api.me() + assert result1 == mock_me_response_1 + + # Change API response + mock_api_client.get_me_v1_me_get.return_value = mock_me_response_2 + + # Second call with nocache=True - fetches fresh data + result2 = client_with_mock_api.me(nocache=True) + assert result2 == mock_me_response_2 + assert mock_api_client.get_me_v1_me_get.call_count == 2 # Additional call made + + def test_me_nocache_true_updates_cache( + client_with_mock_api: Client, mock_api_client: MagicMock + ) -> None: + """Verify me(nocache=True) updates cache with fresh data.""" + mock_me_response_1 = {"user_id": "user-1"} + mock_me_response_2 = {"user_id": "user-2"} + + # First call - populates cache + mock_api_client.get_me_v1_me_get.return_value = mock_me_response_1 + result1 = client_with_mock_api.me() + + # Change API response + mock_api_client.get_me_v1_me_get.return_value = mock_me_response_2 + + # Second call with nocache=True - fetches and caches new data + result2 = client_with_mock_api.me(nocache=True) + assert result2 == mock_me_response_2 + + # Third call without nocache - uses updated cache + result3 = client_with_mock_api.me() + assert result3 == mock_me_response_2 # Uses new cached value + assert mock_api_client.get_me_v1_me_get.call_count == 2 # No additional call +``` + +**Edge Case Testing:** + +```python +class TestNocacheEdgeCases: + """Test edge cases and special scenarios.""" + + def test_nocache_with_expired_cache_entry() -> None: + """Test nocache behavior when cache entry expired.""" + @cached_operation(ttl=1, use_token=False) # 1 second TTL + def test_func() -> int: + return time.time_ns() + + # First call - populates cache + result1 = test_func() + + # Wait for cache to expire + time.sleep(1.1) + + # Call with nocache=True on expired entry + result2 = test_func(nocache=True) + assert result2 != result1 # Different value + + def test_multiple_consecutive_nocache_calls() -> None: + """Test multiple consecutive calls with nocache=True.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # Multiple calls with nocache=True + assert test_func(nocache=True) == 1 + assert test_func(nocache=True) == 2 + assert test_func(nocache=True) == 3 + assert call_count == 3 + + # Last call without nocache uses cached value from third call + assert test_func() == 3 + assert call_count == 3 + + def test_nocache_interleaved_with_normal_calls() -> None: + """Test interleaving nocache=True with normal cached calls.""" + call_count = 0 + + @cached_operation(ttl=60, use_token=False) + def test_func() -> int: + nonlocal call_count + call_count += 1 + return call_count + + # Normal call - populates cache + assert test_func() == 1 + assert call_count == 1 + + # Normal call - uses cache + assert test_func() == 1 + assert call_count == 1 + + # Nocache call - skips cache, updates it + assert test_func(nocache=True) == 2 + assert call_count == 2 + + # Normal call - uses updated cache + assert test_func() == 2 + assert call_count == 2 ``` +**Key Testing Principles:** + +1. **Cache Read Bypass**: nocache=True skips reading from cache +2. **Cache Write Preserved**: nocache=True still writes to cache +3. **Parameter Interception**: nocache parameter intercepted by decorator, not passed to function +4. **Cache Key Isolation**: nocache respects different cache keys (different function args) +5. **Edge Case Coverage**: Expired entries, multiple consecutive calls, interleaved usage +6. **Integration Testing**: Test across all cached Client and Resource methods +7. **Signature Verification**: Test method signatures include nocache parameter with correct type hints + +**Use Cases Tested:** + +* **Testing**: Avoid race conditions from stale cached data +* **Real-time Monitoring**: Ensure latest status in dashboards +* **After Mutations**: Get fresh data immediately after updates +* **Cache Refresh**: Force cache update without full cache clear + ### Process Management Testing (`dataset/service_test.py`) **Subprocess Cleanup Verification:** diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py index 816760d27..5a1b96ec5 100644 --- a/tests/aignostics/application/cli_test.py +++ b/tests/aignostics/application/cli_test.py @@ -754,6 +754,11 @@ def test_cli_run_update_item_metadata_not_dict(runner: CliRunner) -> None: @pytest.mark.e2e @pytest.mark.timeout(timeout=120) +@pytest.mark.skipif( + (platform.system() == "Linux" and platform.machine() in {"aarch64", "arm64"}) + or (platform.system() in {"Darwin", "Windows"}), + reason="No parallel runners, otherwise race condition on metadata updates", +) @pytest.mark.sequential def test_cli_run_dump_and_update_custom_metadata(runner: CliRunner) -> None: """Test dumping and updating custom metadata via CLI commands.""" @@ -849,6 +854,11 @@ def test_cli_run_dump_and_update_custom_metadata(runner: CliRunner) -> None: @pytest.mark.skip(reason="Waiting for platform API fix to item metadata endpoint which currently returns 404 always") @pytest.mark.e2e @pytest.mark.timeout(timeout=120) +@pytest.mark.skipif( + (platform.system() == "Linux" and platform.machine() in {"aarch64", "arm64"}) + or (platform.system() in {"Darwin", "Windows"}), + reason="No parallel runners, otherwise race condition on metadata updates", +) @pytest.mark.sequential def test_cli_run_dump_and_update_item_custom_metadata(runner: CliRunner) -> None: # noqa: PLR0914, PLR0915 # noqa: PLR0914, PLR0915 """Test dumping and updating item custom metadata via CLI commands."""