From 723c42bc4be44b2ddd2f16cadebe36da73979b8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Pe=CC=81net?= Date: Wed, 7 Apr 2021 16:28:16 +0200 Subject: [PATCH 1/5] Methods to query the metrics (as in DSS Metrics) of a MES or a ME. --- dataikuapi/dss/modelevaluationstore.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index 87c71c19..9c247d46 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -170,6 +170,14 @@ def create_model_evaluation(self, labels=None, prediction_type=None, model_type= run_id = res['id'] return DSSModelEvaluation(self, run_id) + def get_latest_metrics(self): + """ + Get the metrics of the latest model evaluation built + + :return: the metrics, as a JSON object + """ + return self.client._perform_json( + "GET", "/projects/%s/modelevaluationstores/%s/metrics" % (self.project_key, self.mes_id)) class DSSModelEvaluationStoreSettings: """ @@ -271,6 +279,15 @@ def put_file(self, path, f): "POST", "/projects/%s/modelevaluationstores/%s/runs/%s/contents/%s" % (self.project_key, self.mes_id, self.run_id, utils.quote(path)), "", f) + def get_metrics(self): + """ + Get the metrics for this model evaluation + + :return: the metrics, as a JSON object + """ + return self.client._perform_json( + "GET", "/projects/%s/modelevaluationstores/%s/runs/%s/metrics" % (self.project_key, self.mes_id, self.run_id)) + class DSSModelEvaluationSettings: """ A handle on the settings of a model evaluation From d035605095dc86e0dfda3c4b0e9b91209549f620 Mon Sep 17 00:00:00 2001 From: "sourcery-ai[bot]" <58596630+sourcery-ai[bot]@users.noreply.github.com> Date: Fri, 9 Apr 2021 11:39:35 +0200 Subject: [PATCH 2/5] 'Refactored by Sourcery' (#143) Co-authored-by: Sourcery AI <> --- dataikuapi/dss/modelevaluationstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index 9c247d46..d571a0a2 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -118,7 +118,7 @@ def list_model_evaluations(self, as_type=None): :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/modelevaluationstores/%s/runs/" % (self.project_key, self.mes_id)) - if as_type == "objects" or as_type == "object": + if as_type in ["objects", "object"]: return [DSSModelEvaluation(self, item["ref"]["runId"]) for item in items] else: return items From 464864655f5f7d55d92606679f1215e1c2ded760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Pe=CC=81net?= Date: Tue, 13 Apr 2021 16:14:06 +0200 Subject: [PATCH 3/5] Getting MES metrics API closer to dataset and managed folders metrics API. --- dataikuapi/dss/modelevaluationstore.py | 29 +++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index d571a0a2..e96fd337 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -1,3 +1,6 @@ +import json + +from dataikuapi.dss.metrics import ComputedMetrics from .discussion import DSSObjectDiscussions from requests import utils @@ -170,14 +173,34 @@ def create_model_evaluation(self, labels=None, prediction_type=None, model_type= run_id = res['id'] return DSSModelEvaluation(self, run_id) - def get_latest_metrics(self): + ######################################################## + # Metrics + ######################################################## + + def get_last_metric_values(self): """ Get the metrics of the latest model evaluation built - :return: the metrics, as a JSON object + Returns: + a list of metric objects and their value + """ + return ComputedMetrics(self.client._perform_json( + "GET", "/projects/%s/modelevaluationstores/%s/metrics/last" % (self.project_key, self.mes_id))) + + + def get_metric_history(self, metric): + """ + Get the history of the values of the metric on this dataset + + Returns: + an object containing the values of the metric, cast to the appropriate type (double, boolean,...) """ return self.client._perform_json( - "GET", "/projects/%s/modelevaluationstores/%s/metrics" % (self.project_key, self.mes_id)) + "GET", "/projects/%s/modelevaluationstores/%s/metrics/history" % (self.project_key, self.mes_id), + params={'metricLookup': metric if isinstance(metric, str)or isinstance(metric, unicode) + else json.dumps(metric)}) + + class DSSModelEvaluationStoreSettings: """ From c11829690576fd52829536de173455c0620b0658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Pe=CC=81net?= Date: Tue, 13 Apr 2021 18:51:22 +0200 Subject: [PATCH 4/5] Adding compute_metrics method to DSSModelEvaluationStore --- dataikuapi/dss/modelevaluationstore.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index e96fd337..dd8ec9d6 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -173,6 +173,7 @@ def create_model_evaluation(self, labels=None, prediction_type=None, model_type= run_id = res['id'] return DSSModelEvaluation(self, run_id) + ######################################################## # Metrics ######################################################## @@ -200,6 +201,24 @@ def get_metric_history(self, metric): params={'metricLookup': metric if isinstance(metric, str)or isinstance(metric, unicode) else json.dumps(metric)}) + def compute_metrics(self, metric_ids=None, probes=None): + """ + Compute metrics on this managed folder. If the metrics are not specified, the metrics + setup on the managed folder are used. + """ + url = "/projects/%s/modelevaluationstores/%s/actions" % (self.project_key, self.mes_id) + if metric_ids is not None: + return self.client._perform_json( + "POST" , "%s/computeMetricsFromIds" % url, + body={"metricIds" : metric_ids}) + elif probes is not None: + return self.client._perform_json( + "POST" , "%s/computeMetrics" % url, + body=probes) + else: + return self.client._perform_json( + "POST" , "%s/computeMetrics" % url) + class DSSModelEvaluationStoreSettings: From 13782c778c08ba606518a99c98897c13f85fb7a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Pe=CC=81net?= Date: Fri, 11 Jun 2021 16:23:11 +0200 Subject: [PATCH 5/5] Correcting comments. --- dataikuapi/dss/modelevaluationstore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index dd8ec9d6..f1f71910 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -191,7 +191,7 @@ def get_last_metric_values(self): def get_metric_history(self, metric): """ - Get the history of the values of the metric on this dataset + Get the history of the values of the metric on this model evaluation store Returns: an object containing the values of the metric, cast to the appropriate type (double, boolean,...) @@ -203,8 +203,8 @@ def get_metric_history(self, metric): def compute_metrics(self, metric_ids=None, probes=None): """ - Compute metrics on this managed folder. If the metrics are not specified, the metrics - setup on the managed folder are used. + Compute metrics on this model evaluation store. If the metrics are not specified, the metrics + setup on the model evaluation store are used. """ url = "/projects/%s/modelevaluationstores/%s/actions" % (self.project_key, self.mes_id) if metric_ids is not None: