diff --git a/dataikuapi/dss/modelcomparison.py b/dataikuapi/dss/modelcomparison.py new file mode 100644 index 00000000..8562140a --- /dev/null +++ b/dataikuapi/dss/modelcomparison.py @@ -0,0 +1,164 @@ +from dataikuapi.dss.discussion import DSSObjectDiscussions + + +class DSSModelComparison(object): + """ + A handle to interact with a model comparison on the DSS instance + + Do not create this directly, use :meth:`dataikuapi.dss.DSSProject.get_model_comparison` + """ + def __init__(self, client, project_key, mec_id): + self.client = client + self.project = client.get_project(project_key) + self.project_key = project_key + self.mec_id = mec_id + + @property + def id(self): + return self.mec_id + + def get_settings(self): + """ + Returns the settings of this model comparison + + :rtype: :class:`dataikuapi.dss.modelcomparison.DSSModelComparisonSettings` + """ + data = self.client._perform_json( + "GET", "/projects/%s/modelcomparisons/%s" % (self.project_key, self.mec_id)) + return DSSModelComparisonSettings(self, data) + + def get_object_discussions(self): + """ + Get a handle to manage discussions on the model comparison + + :returns: the handle to manage discussions + :rtype: :class:`dataikuapi.discussion.DSSObjectDiscussions` + """ + return DSSObjectDiscussions(self.client, self.project_key, "MODEL_COMPARISON", self.mec_id) + + ######################################################## + # Deletion + ######################################################## + + def delete(self): + """ + Delete the model comparison + + """ + return self.client._perform_empty("DELETE", "/projects/%s/modelcomparisons/%s" % (self.project_key, self.mec_id)) + + +class DSSModelComparisonSettings(object): + """ + A handle on the settings of a model comparison + + A model comparison has: + - a display name ; + - a prediction type ; + - a list of full ids of items to compare + + The prediction type can be: + - BINARY_CLASSIFICATION, + - REGRESSION, + - MULTICLASS + + The full ids are: + - the model id of a Lab Model, + - the model id of a saved model version, + - the model evaluation id of a model evaluation. + + + Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSModelComparison.get_settings` + """ + def __init__(self, model_comparison, settings): + self.model_comparison = model_comparison + self.settings = settings + + def get_raw(self): + """ + Get raw settings of a model comparison + + :return: the raw settings of comparison, as a dict. Modifications made to the returned object + are reflected when saving + :rtype: dict + """ + return self.settings + + def add_compared_item(self, full_id): + """ + Add an item to the list of compared items + + :param full_id: full id of the item (lab model, saved model version, model evaluation) to add + """ + if "comparedModels" not in self.settings: + self.settings["comparedModels"] = [] + self.settings["comparedModels"].append({ + "refId": full_id + }) + + def remove_compared_item(self, full_id): + """ + Remove an item from the list of compared items + + :param full_id: full id of the item (lab model, saved model version, model evaluation) to remove + """ + if not self.settings["comparedModels"]: + return + self.settings["comparedModels"] = filter(lambda x: x["refId"] != full_id, self.settings["comparedModels"]) + + def get_compared_items(self): + """ + Get the full ids of items compared in this comparison + + :return: the list of the full ids of compared items + :rtype: list[str] + """ + if not self.settings["comparedModels"]: + return [] + return [x["refId"] for x in self.settings["comparedModels"]] + + @property + def prediction_type(self): + """ + Get the prediction type of this comparison + + :return: str + """ + return self.settings["predictionType"] + + @prediction_type.setter + def prediction_type(self, prediction_type): + """ + Set the prediction type of this comparison. Must be consistent + with the prediction types of compared items. + + :param prediction_type: + """ + self.settings["predictionType"] = prediction_type + + @property + def display_name(self): + """ + Human readable name of this comparison + + :return: str + """ + return self.settings["displayName"] + + @display_name.setter + def display_name(self, display_name): + """ + Set the human readable name of this comparison + + :param display_name: + """ + self.settings["displayName"] = display_name + + def save(self): + """ + Save settings modifications + """ + self.model_comparison.client._perform_empty( + "PUT", "/projects/%s/modelcomparisons/%s" % (self.model_comparison.project_key, self.model_comparison.mec_id), + body=self.settings) + diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index 170d3651..3f7de2c7 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -278,6 +278,9 @@ def get_full_info(self): "GET", "/projects/%s/modelevaluationstores/%s/runs/%s" % (self.project_key, self.mes_id, self.run_id)) return DSSModelEvaluationFullInfo(self, data) + def get_full_id(self): + return "ME-{}-{}-{}".format(self.project_key, self.mes_id, self.run_id) + def delete(self): """ Remove this model evaluation diff --git a/dataikuapi/dss/project.py b/dataikuapi/dss/project.py index df7279ce..4a5f8842 100644 --- a/dataikuapi/dss/project.py +++ b/dataikuapi/dss/project.py @@ -1,5 +1,6 @@ import time, warnings, sys, os.path as osp from .dataset import DSSDataset, DSSDatasetListItem, DSSManagedDatasetCreationHelper +from .modelcomparison import DSSModelComparison from .jupyternotebook import DSSJupyterNotebook, DSSJupyterNotebookListItem from .notebook import DSSNotebook from .streaming_endpoint import DSSStreamingEndpoint, DSSStreamingEndpointListItem, DSSManagedStreamingEndpointCreationHelper @@ -783,17 +784,15 @@ def get_model_evaluation_store(self, mes_id): """ return DSSModelEvaluationStore(self.client, self.project_key, mes_id) - def create_model_evaluation_store(self, name, mes_id=None): + def create_model_evaluation_store(self, name): """ Create a new model evaluation store in the project, and return a handle to interact with it. :param string name: the name for the new model evaluation store - :param string mes_id optional: the id for the new model evaluation store - + :returns: A :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluationStore` model evaluation store handle """ obj = { - "id" : mes_id, "projectKey" : self.project_key, "name" : name } @@ -802,6 +801,51 @@ def create_model_evaluation_store(self, name, mes_id=None): mes_id = res['id'] return DSSModelEvaluationStore(self.client, self.project_key, mes_id) + ######################################################## + # Model comparisons + ######################################################## + + def list_model_comparisons(self): + """ + List the model comparisons in this project. + + :returns: The list of the model comparisons + :rtype: list + """ + items = self.client._perform_json("GET", "/projects/%s/modelcomparisons/" % self.project_key) + return [DSSModelComparison(self.client, self.project_key, item["id"]) for item in items] + + def get_model_comparison(self, mec_id): + """ + Get a handle to interact with a specific model comparison + + :param string mec_id: the id of the desired model comparison + + :returns: A handle on a model comparison + :rtype: :class:`dataikuapi.dss.modelcomparison.DSSModelComparison` + """ + return DSSModelComparison(self.client, self.project_key, mec_id) + + def create_model_comparison(self, name, prediction_type): + """ + Create a new model comparison in the project, and return a handle to interact with it. + + :param string name: the name for the new model comparison + :param string prediction_type: one of BINARY_CLASSIFICATION, REGRESSION and MULTICLASS + + :returns: A handle on a new model comparison + :rtype: :class:`dataikuapi.dss.modelcomparison.DSSModelComparison` + """ + obj = { + "projectKey": self.project_key, + "displayName": name, + "predictionType": prediction_type + } + res = self.client._perform_json("POST", "/projects/%s/modelcomparisons/" % self.project_key, + body = obj) + mec_id = res['id'] + return DSSModelComparison(self.client, self.project_key, mec_id) + ######################################################## # Jobs ########################################################