From de8bd9668c68220efbde43e50c49b919ea2322b2 Mon Sep 17 00:00:00 2001 From: Fred Date: Wed, 8 Sep 2021 15:02:11 +0200 Subject: [PATCH 1/8] Added a method to compute data drift on a ME --- dataikuapi/dss/modelevaluationstore.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index 21566397..4dd47659 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -5,6 +5,7 @@ from dataikuapi.dss.metrics import ComputedMetrics from .discussion import DSSObjectDiscussions +from .future import DSSFuture from requests import utils @@ -288,6 +289,15 @@ def delete(self): self.client._perform_json( "DELETE", "/projects/%s/modelevaluationstores/%s/runs/" % (self.project_key, self.mes_id), body=obj) + def compute_data_drift(self, reference_id=None, data_drift_params=None): + future_response = self.client._perform_json( + "POST", "/projects/%s/modelevaluationstores/%s/runs/%s/computeDataDrift" % (self.project_key, self.mes_id, self.run_id), + body={ + "referenceId": reference_id, + "dataDriftParams": data_drift_params + }) + return DSSFuture(self.client, future_response.get('jobId', None), future_response) + def get_metrics(self): """ Get the metrics for this model evaluation. Metrics must be understood here as Metrics in DSS Metrics & Checks From cf46c133ae161af7ca60b1ac71e2be104e27a095 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 10 Sep 2021 15:14:34 +0200 Subject: [PATCH 2/8] Added a basic explanation to DSSModelEvaluation.compute_data_drift() --- dataikuapi/dss/modelevaluationstore.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index 4dd47659..2b8ed7a3 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -290,6 +290,13 @@ def delete(self): "DELETE", "/projects/%s/modelevaluationstores/%s/runs/" % (self.project_key, self.mes_id), body=obj) def compute_data_drift(self, reference_id=None, data_drift_params=None): + """ + Compute input data drift against a reference model or model evaluation. The reference is determined automatically unless specified. + + :param reference_id: model ID or model evaluation ID (optional) + :param data_drift_params: input drift computation settings (optional) + :return: input drift analysis results, as a JSON object + """ future_response = self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/runs/%s/computeDataDrift" % (self.project_key, self.mes_id, self.run_id), body={ From 87d78f8b4730ae7cc00b42e0bc11771b5dfe37ee Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 10 Sep 2021 15:16:04 +0200 Subject: [PATCH 3/8] Wording: input -> data (drift) --- dataikuapi/dss/modelevaluationstore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index 2b8ed7a3..bc91848d 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -291,11 +291,11 @@ def delete(self): def compute_data_drift(self, reference_id=None, data_drift_params=None): """ - Compute input data drift against a reference model or model evaluation. The reference is determined automatically unless specified. + Compute data drift against a reference model or model evaluation. The reference is determined automatically unless specified. :param reference_id: model ID or model evaluation ID (optional) - :param data_drift_params: input drift computation settings (optional) - :return: input drift analysis results, as a JSON object + :param data_drift_params: data drift computation settings (optional) + :return: data drift analysis results, as a JSON object """ future_response = self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/runs/%s/computeDataDrift" % (self.project_key, self.mes_id, self.run_id), From b1c7b11bdec3933f785d6259ea6789720bc1e711 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 17 Sep 2021 17:40:56 +0200 Subject: [PATCH 4/8] Allow both IDs and object instances to be used as reference for drift computation --- dataikuapi/dss/ml.py | 4 ++++ dataikuapi/dss/modelevaluationstore.py | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 31f90d9b..312ebb91 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -2499,6 +2499,10 @@ def get_roc_curve_data(self): return roc + @property + def _full_model_like_id(self): + return self.details["fullModelId"] + def get_performance_metrics(self): """ Returns all performance metrics for this model. diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index bc91848d..dc2e2ea8 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -289,18 +289,27 @@ def delete(self): self.client._perform_json( "DELETE", "/projects/%s/modelevaluationstores/%s/runs/" % (self.project_key, self.mes_id), body=obj) - def compute_data_drift(self, reference_id=None, data_drift_params=None): + @property + def _full_model_like_id(self): + return "ME-%s-%s-%s"%(self.project_key, self.mes_id, self.run_id) + + def compute_data_drift(self, reference=None, data_drift_params=None): """ Compute data drift against a reference model or model evaluation. The reference is determined automatically unless specified. - :param reference_id: model ID or model evaluation ID (optional) + :param reference: saved model version or model evaluation to use as reference (optional) + :type reference: Union[str, DSSModelEvaluation, DSSTrainedPredictionModelDetails] :param data_drift_params: data drift computation settings (optional) :return: data drift analysis results, as a JSON object """ + + if hasattr(reference, '_full_model_like_id'): + reference = reference._full_model_like_id + future_response = self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/runs/%s/computeDataDrift" % (self.project_key, self.mes_id, self.run_id), body={ - "referenceId": reference_id, + "referenceId": reference, "dataDriftParams": data_drift_params }) return DSSFuture(self.client, future_response.get('jobId', None), future_response) From b1a915180355b52a3f255c3f78d5af2fb86c63bc Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 21 Sep 2021 18:18:34 +0200 Subject: [PATCH 5/8] Make full_model_like_id() public & available on both prediction & clustering models --- dataikuapi/dss/ml.py | 8 ++++---- dataikuapi/dss/modelevaluationstore.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 312ebb91..fde3dd0f 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -1804,6 +1804,10 @@ def get_raw(self): """ return self.details + @property + def full_model_like_id(self): + return self.details["fullModelId"] + def get_raw_snippet(self): """ Gets the raw dictionary of trained model snippet. @@ -2499,10 +2503,6 @@ def get_roc_curve_data(self): return roc - @property - def _full_model_like_id(self): - return self.details["fullModelId"] - def get_performance_metrics(self): """ Returns all performance metrics for this model. diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index dc2e2ea8..d2ed2706 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -290,7 +290,7 @@ def delete(self): "DELETE", "/projects/%s/modelevaluationstores/%s/runs/" % (self.project_key, self.mes_id), body=obj) @property - def _full_model_like_id(self): + def full_model_like_id(self): return "ME-%s-%s-%s"%(self.project_key, self.mes_id, self.run_id) def compute_data_drift(self, reference=None, data_drift_params=None): @@ -303,8 +303,8 @@ def compute_data_drift(self, reference=None, data_drift_params=None): :return: data drift analysis results, as a JSON object """ - if hasattr(reference, '_full_model_like_id'): - reference = reference._full_model_like_id + if hasattr(reference, 'full_model_like_id'): + reference = reference.full_model_like_id future_response = self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/runs/%s/computeDataDrift" % (self.project_key, self.mes_id, self.run_id), From ae94b2acb25ae93764147498686905806ac2c678 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 21 Sep 2021 18:35:11 +0200 Subject: [PATCH 6/8] Clarify that compute_data_drift() accepts either a Python object or an ID --- dataikuapi/dss/modelevaluationstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index d2ed2706..610d304d 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -297,7 +297,7 @@ def compute_data_drift(self, reference=None, data_drift_params=None): """ Compute data drift against a reference model or model evaluation. The reference is determined automatically unless specified. - :param reference: saved model version or model evaluation to use as reference (optional) + :param reference: saved model version or model evaluation to use as reference (optional, accepts ID or Python object) :type reference: Union[str, DSSModelEvaluation, DSSTrainedPredictionModelDetails] :param data_drift_params: data drift computation settings (optional) :return: data drift analysis results, as a JSON object From c58f6e158a6d6f123604d7599d25c3c5e8f7fa46 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 21 Sep 2021 18:52:29 +0200 Subject: [PATCH 7/8] full_model_like_id -> full_id --- dataikuapi/dss/ml.py | 2 +- dataikuapi/dss/modelevaluationstore.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index fde3dd0f..2fb31ac1 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -1805,7 +1805,7 @@ def get_raw(self): return self.details @property - def full_model_like_id(self): + def full_id(self): return self.details["fullModelId"] def get_raw_snippet(self): diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index 610d304d..d2bfb63c 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -290,7 +290,7 @@ def delete(self): "DELETE", "/projects/%s/modelevaluationstores/%s/runs/" % (self.project_key, self.mes_id), body=obj) @property - def full_model_like_id(self): + def full_id(self): return "ME-%s-%s-%s"%(self.project_key, self.mes_id, self.run_id) def compute_data_drift(self, reference=None, data_drift_params=None): @@ -303,8 +303,8 @@ def compute_data_drift(self, reference=None, data_drift_params=None): :return: data drift analysis results, as a JSON object """ - if hasattr(reference, 'full_model_like_id'): - reference = reference.full_model_like_id + if hasattr(reference, 'full_id'): + reference = reference.full_id future_response = self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/runs/%s/computeDataDrift" % (self.project_key, self.mes_id, self.run_id), From a7d454c05c404d20470ab30f39ce9281cb0d1755 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 21 Sep 2021 19:06:46 +0200 Subject: [PATCH 8/8] Auto-wait for future results in compute_data_drift() (by default) --- dataikuapi/dss/modelevaluationstore.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index d2bfb63c..2a84b089 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -293,14 +293,16 @@ def delete(self): def full_id(self): return "ME-%s-%s-%s"%(self.project_key, self.mes_id, self.run_id) - def compute_data_drift(self, reference=None, data_drift_params=None): + def compute_data_drift(self, reference=None, data_drift_params=None, wait=True): """ Compute data drift against a reference model or model evaluation. The reference is determined automatically unless specified. - :param reference: saved model version or model evaluation to use as reference (optional, accepts ID or Python object) + :param reference: saved model version (full ID or DSSTrainedPredictionModelDetails) + or model evaluation (full ID or DSSModelEvaluation) to use as reference (optional) :type reference: Union[str, DSSModelEvaluation, DSSTrainedPredictionModelDetails] :param data_drift_params: data drift computation settings (optional) - :return: data drift analysis results, as a JSON object + :param wait: data drift computation settings (optional) + :returns: a `dict` containing data drift analysis results if `wait` is `True`, or a :class:`~dataikuapi.dss.future.DSSFuture` handle otherwise """ if hasattr(reference, 'full_id'): @@ -312,7 +314,8 @@ def compute_data_drift(self, reference=None, data_drift_params=None): "referenceId": reference, "dataDriftParams": data_drift_params }) - return DSSFuture(self.client, future_response.get('jobId', None), future_response) + future = DSSFuture(self.client, future_response.get('jobId', None), future_response) + return future.wait_for_result() if wait else future def get_metrics(self): """