diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index 87c71c19..f1f71910 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -1,3 +1,6 @@ +import json + +from dataikuapi.dss.metrics import ComputedMetrics from .discussion import DSSObjectDiscussions from requests import utils @@ -118,7 +121,7 @@ def list_model_evaluations(self, as_type=None): :rtype: list """ items = self.client._perform_json("GET", "/projects/%s/modelevaluationstores/%s/runs/" % (self.project_key, self.mes_id)) - if as_type == "objects" or as_type == "object": + if as_type in ["objects", "object"]: return [DSSModelEvaluation(self, item["ref"]["runId"]) for item in items] else: return items @@ -171,6 +174,53 @@ def create_model_evaluation(self, labels=None, prediction_type=None, model_type= return DSSModelEvaluation(self, run_id) + ######################################################## + # Metrics + ######################################################## + + def get_last_metric_values(self): + """ + Get the metrics of the latest model evaluation built + + Returns: + a list of metric objects and their value + """ + return ComputedMetrics(self.client._perform_json( + "GET", "/projects/%s/modelevaluationstores/%s/metrics/last" % (self.project_key, self.mes_id))) + + + def get_metric_history(self, metric): + """ + Get the history of the values of the metric on this model evaluation store + + Returns: + an object containing the values of the metric, cast to the appropriate type (double, boolean,...) + """ + return self.client._perform_json( + "GET", "/projects/%s/modelevaluationstores/%s/metrics/history" % (self.project_key, self.mes_id), + params={'metricLookup': metric if isinstance(metric, str)or isinstance(metric, unicode) + else json.dumps(metric)}) + + def compute_metrics(self, metric_ids=None, probes=None): + """ + Compute metrics on this model evaluation store. If the metrics are not specified, the metrics + setup on the model evaluation store are used. + """ + url = "/projects/%s/modelevaluationstores/%s/actions" % (self.project_key, self.mes_id) + if metric_ids is not None: + return self.client._perform_json( + "POST" , "%s/computeMetricsFromIds" % url, + body={"metricIds" : metric_ids}) + elif probes is not None: + return self.client._perform_json( + "POST" , "%s/computeMetrics" % url, + body=probes) + else: + return self.client._perform_json( + "POST" , "%s/computeMetrics" % url) + + + class DSSModelEvaluationStoreSettings: """ A handle on the settings of a model evaluation store @@ -271,6 +321,15 @@ def put_file(self, path, f): "POST", "/projects/%s/modelevaluationstores/%s/runs/%s/contents/%s" % (self.project_key, self.mes_id, self.run_id, utils.quote(path)), "", f) + def get_metrics(self): + """ + Get the metrics for this model evaluation + + :return: the metrics, as a JSON object + """ + return self.client._perform_json( + "GET", "/projects/%s/modelevaluationstores/%s/runs/%s/metrics" % (self.project_key, self.mes_id, self.run_id)) + class DSSModelEvaluationSettings: """ A handle on the settings of a model evaluation