diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index fb3c4947..9ab69a23 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -5,8 +5,6 @@ from .discussion import DSSObjectDiscussions from .future import DSSFuture -from requests import utils - try: basestring except NameError: @@ -169,7 +167,7 @@ def delete_model_evaluations(self, evaluations): def build(self, job_type="NON_RECURSIVE_FORCED_BUILD", wait=True, no_fail=False): """ - Starts a new job to build this Model Evaluation Store and wait for it to complete. + Starts a new job to build this model evaluation store and wait for it to complete. Raises if the job failed. .. code-block:: python @@ -274,6 +272,8 @@ def __init__(self, model_evaluation_store, evaluation_id): def get_full_info(self): """ Retrieve the model evaluation with its performance data + + :return: the model evaluation full info, as a :class:`dataikuapi.dss.DSSModelEvaluationInfo` """ data = self.client._perform_json( "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s" % (self.project_key, self.mes_id, self.evaluation_id)) @@ -301,21 +301,25 @@ def compute_data_drift(self, reference=None, data_drift_params=None, wait=True): :param reference: saved model version (full ID or DSSTrainedPredictionModelDetails) or model evaluation (full ID or DSSModelEvaluation) to use as reference (optional) :type reference: Union[str, DSSModelEvaluation, DSSTrainedPredictionModelDetails] - :param data_drift_params: data drift computation settings (optional) + :param data_drift_params: data drift computation settings as a :class:`dataikuapi.dss.modelevaluationstore.DataDriftParams` (optional) + :type data_drift_params: DataDriftParams :param wait: data drift computation settings (optional) - :returns: a `dict` containing data drift analysis results if `wait` is `True`, or a :class:`~dataikuapi.dss.future.DSSFuture` handle otherwise + :returns: a :class:`dataikuapi.dss.modelevaluationstore.DataDriftResult` containing data drift analysis results if `wait` is `True`, or a :class:`~dataikuapi.dss.future.DSSFuture` handle otherwise """ if hasattr(reference, 'full_id'): reference = reference.full_id + if data_drift_params: + data_drift_params = data_drift_params.data + future_response = self.client._perform_json( "POST", "/projects/%s/modelevaluationstores/%s/evaluations/%s/computeDataDrift" % (self.project_key, self.mes_id, self.evaluation_id), body={ "referenceId": reference, "dataDriftParams": data_drift_params }) - future = DSSFuture(self.client, future_response.get('jobId', None), future_response) + future = DSSFuture(self.client, future_response.get('jobId', None), future_response, result_wrapper=DataDriftResult) return future.wait_for_result() if wait else future def get_metrics(self): @@ -361,39 +365,190 @@ class DSSModelEvaluationFullInfo: def __init__(self, model_evaluation, full_info): self.model_evaluation = model_evaluation self.full_info = full_info + self.metrics = self.full_info["metrics"] # type: dict + """The performance and data drift metric, if any.""" + self.creation_date = self.full_info["evaluation"]["created"] # type: int + """The date and time of the creation of the model evaluation, as an epoch.""" + self.full_id = self.full_info["evaluation"]["ref"]["fullId"] # type: str + self.model_full_id = self.full_info["evaluation"]["modelRef"]["fullId"] # type: str + self.prediction_type = self.full_info["evaluation"]["predictionType"] # type: str + self.prediction_variable = self.full_info["evaluation"]["predictionVariable"] # type: str + self.target_variable = self.full_info["evaluation"]["targetVariable"] # type: str + self.user_meta = self.full_info["evaluation"]["userMeta"] # type: dict + """The user-accessible metadata (name, labels) + Returns the original object, not a copy. Changes to the returned object are persisted to DSS by calling :meth:`save_user_meta`.""" def get_raw(self): return self.full_info - def get_metrics(self): + def save_user_meta(self): + return self.model_evaluation.client._perform_text( + "PUT", "/projects/%s/modelevaluationstores/%s/evaluations/%s/user-meta" % + (self.model_evaluation.project_key, self.model_evaluation.mes_id, self.model_evaluation.evaluation_id), body=self.user_meta) + + +class DataDriftParams(object): + """ + Object that represents parameters for data drift computation. + Do not create this object directly, use :meth:`dataikuapi.dss.modelevaluationstore.DataDriftParams.from_params` instead. + """ + def __init__(self, data): + self.data = data + + def __repr__(self): + return u"{}({})".format(self.__class__.__name__, self.data) + + @staticmethod + def from_params(per_column_settings, nb_bins=10, compute_histograms=True, confidence_level=0.95): """ - Get the metrics evaluated, if any. + Creates parameters for data drift computation from columns, number of bins, compute histograms and confidence level + + :param dict per_column_settings: A dict representing the per column settings. + You should use a :class:`~dataikuapi.dss.modelevaluationstore.PerColumnDriftParamBuilder` to build it. + :param int nb_bins: (optional) Nb. bins in histograms (apply to all columns) - default: 10 + :param bool compute_histograms: (optional) Enable/disable histograms - default: True + :param float confidence_level: (optional) Used to compute confidence interval on drift's model accuracy - default: 0.95 - :return: a dict containing the performance and data drift metric, if any + :rtype: :class:`dataikuapi.dss.modelevaluationstore.DataDriftParams` """ - return self.full_info["metrics"] + return DataDriftParams({ + "columns": per_column_settings, + "nbBins": nb_bins, + "computeHistograms": compute_histograms, + "confidenceLevel": confidence_level + }) - def get_labels(self): + +class PerColumnDriftParamBuilder(object): + """ + Builder for a map of per column drift params settings. + Used as a helper before computing data drift to build columns param expected in + :meth:`dataikuapi.dss.modelevaluationstore.DataDriftParams.from_params`. + """ + def __init__(self): + self.columns = {} + + def build(self): + """Returns the built dict for per column drift params settings""" + return self.columns + + def with_column_drift_param(self, name, handling="AUTO", enabled=True): """ - Get the labels of the Model Evaluation + Sets the drift params settings for given column name. - :return: a dict containing the labels + :param: string name: The name of the column + :param: string handling: (optional) The column type, should be either NUMERICAL, CATEGORICAL or AUTO (default: AUTO) + :param: bool enabled: (optional) False means the column is ignored in drift computation (default: True) """ - return self.full_info["evaluation"]["labels"] + self.columns[name] = { + "handling": handling, + "enabled": enabled + } + return self + + +class DataDriftResult(object): + """ + A handle on the data drift result of a model evaluation. + + Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSModelEvaluation.compute_data_drift` + """ + def __init__(self, data): + self.data = data + self.drift_model_result = DriftModelResult(self.data["driftModelResult"]) + """Drift analysis based on drift modeling.""" + self.univariate_drift_result = UnivariateDriftResult(self.data["univariateDriftResult"]) + """Per-column drift analysis based on pairwise comparison of distributions.""" + self.per_column_settings = [ColumnSettings(cs) for cs in self.data["perColumnSettings"]] + """Information about column handling that has been used (errors, types, etc).""" - def get_evaluation_parameters(self): + def get_raw(self): + """ + :return: the raw data drift result + :rtype: dict """ - Get info on the evaluation parameters, most noticeably the evaluation metric (evaluationMetric field - of the returned dict) + return self.data + - :return: a dict +class DriftModelResult(object): + """ + A handle on the drift model result. + + Do not create this class directly, instead use :attr:`dataikuapi.dss.modelevaluationstore.DataDriftResult.drift_model_result` + """ + def __init__(self, data): + self.data = data + self.drift_model_accuracy = DriftModelAccuracy(self.data["driftModelAccuracy"]) + self.feature_drift_importance = self.data["driftVersusImportance"] # type: dict + + def get_raw(self): """ - return self.full_info["evaluation"]["metricParams"] + :return: the raw drift model result + :rtype: dict + """ + return self.data + + +class UnivariateDriftResult(object): + """ + A handle on the univariate data drift. + + Do not create this class directly, instead use :attr:`dataikuapi.dss.modelevaluationstore.DataDriftResult.univariate_drift_result` + """ + def __init__(self, data): + self.data = data + self.per_column_drift_data = self.data["columns"] # type: dict + """Drift data per column, as a dict of column name -> drift data.""" + + def get_raw(self): + """ + :return: the raw univariate data drift + :rtype: dict + """ + return self.data + - def get_creation_date(self): +class ColumnSettings(object): + """ + A handle on column handling information. + + Do not create this class directly, instead use :meth:`dataikuapi.dss.modelevaluationstore.DataDriftResult.get_per_column_settings` + """ + def __init__(self, data): + self.data = data + self.name = self.data["name"] # type: str + self.actual_column_handling = self.data["actualHandling"] # type: str + """The actual column handling (either forced via drift params or inferred from model evaluation preprocessings). + It can be any of NUMERICAL, CATEGORICAL, or IGNORED.""" + self.default_column_handling = self.data["defaultHandling"] # type: str + """The default column handling (based on model evaluation preprocessing only). + It can be any of NUMERICAL, CATEGORICAL, or IGNORED.""" + self.error_message = self.data.get("errorMessage", None) + + def get_raw(self): + """ + :return: the raw column handling information + :rtype: dict """ - Return the date and time of the creation of the Model Evaluation + return self.data + - :return: the date and time, as an epoch +class DriftModelAccuracy(object): + """ + A handle on the drift model accuracy. + + Do not create this class directly, instead use :attr:`dataikuapi.dss.modelevaluationstore.DriftModelResult.drift_model_accuracy` + """ + def __init__(self, data): + self.data = data + self.value = self.data["value"] # type: float + self.lower_confidence_interval = self.data["lower"] # type: float + self.upper_confidence_interval = self.data["upper"] # type: float + self.pvalue = self.data["pvalue"] # type: float + + def get_raw(self): + """ + :return: the drift model accuracy data + :rtype: dict """ - return self.full_info["evaluation"]["created"] \ No newline at end of file + return self.data