From 46c45ea0e4bcdf53ed76f4caa3e91674e0a8aa1b Mon Sep 17 00:00:00 2001 From: Clement Date: Wed, 22 Sep 2021 16:36:17 +0200 Subject: [PATCH 01/50] Selectable code env for MLFlow models --- dataikuapi/dss/savedmodel.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dataikuapi/dss/savedmodel.py b/dataikuapi/dss/savedmodel.py index 84a0ba43..2cc32be0 100644 --- a/dataikuapi/dss/savedmodel.py +++ b/dataikuapi/dss/savedmodel.py @@ -117,7 +117,7 @@ def get_origin_ml_task(self): if fmi is not None: return DSSMLTask.from_full_model_id(self.client, fmi, project_key=self.project_key) - def import_mlflow_version_from_path(self, version_id, path): + def import_mlflow_version_from_path(self, version_id, path, code_env_name = "INHERIT"): """ Create a new version for this saved model from a path containing a MLFlow model. @@ -125,7 +125,9 @@ def import_mlflow_version_from_path(self, version_id, path): :param str version_id: Identifier of the version to create :param str path: An absolute path on the local filesystem. Must be a folder, and must contain a MLFlow model - + :param str code_env_name: Name of the code env to use for this model version. The code env must contain at least + mlflow and the package(s) corresponding to the used MLFlow-compatible frameworks. + If value is "INHERIT", the default active code env of the project will be used :return a :class:MLFlowVersionHandler in order to interact with the new MLFlow model version """ # TODO: Add a check that it's indeed a MLFlow model folder @@ -135,7 +137,7 @@ def import_mlflow_version_from_path(self, version_id, path): shutil.make_archive("tmpmodel", "zip", path) #[, root_dir[, base_dir[, verbose[, dry_run[, owner[, group[, logger]]]]]]]) with open("tmpmodel.zip", "rb") as fp: - self.client._perform_empty("POST", "/projects/%s/savedmodels/%s/versions/%s" % (self.project_key, self.sm_id, version_id), + self.client._perform_empty("POST", "/projects/%s/savedmodels/%s/versions/%s?codeEnvName=%s" % (self.project_key, self.sm_id, version_id, code_env_name), files={"file":("tmpmodel.zip", fp)}) return self.get_mlflow_version_handler(version_id) From e76cc908c37d3cbc55cdcf544d476247e2df425d Mon Sep 17 00:00:00 2001 From: Clement Date: Thu, 23 Sep 2021 14:26:26 +0200 Subject: [PATCH 02/50] Class to interact with MLFlow model metrics params --- dataikuapi/dss/savedmodel.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dataikuapi/dss/savedmodel.py b/dataikuapi/dss/savedmodel.py index 2cc32be0..6c804477 100644 --- a/dataikuapi/dss/savedmodel.py +++ b/dataikuapi/dss/savedmodel.py @@ -234,6 +234,22 @@ def delete(self): """ return self.client._perform_empty("DELETE", "/projects/%s/savedmodels/%s" % (self.project_key, self.sm_id)) +class MLFlowVersionSettings: + """Handle for the settings of an imported MLFlow model version""" + + def __init__(self, version_handler, data): + self.version_handler = version_handler + self.data = data + + @property + def raw(self): + return self.data + + def save(self): + self.version_handler.saved_model.client._perform_empty("PUT", + "/projects/%s/savedmodels/%s/versions/%s/external-ml/metadata" % (self.version_handler.saved_model.project_key, self.version_handler.saved_model.sm_id, self.version_handler.version_id), + body=self.data) + class MLFlowVersionHandler: """Handler to interact with an imported MLFlow model version""" def __init__(self, saved_model, version_id): @@ -241,6 +257,10 @@ def __init__(self, saved_model, version_id): self.saved_model = saved_model self.version_id = version_id + def get_settings(self): + metadata = self.saved_model.client._perform_json("GET", "/projects/%s/savedmodels/%s/versions/%s/external-ml/metadata" % (self.saved_model.project_key, self.saved_model.sm_id, self.version_id)) + return MLFlowVersionSettings(self, metadata) + def set_core_metadata(self, target_column_name, class_labels = None, get_features_from_dataset=None, features_list = None, From a8243cc070f34575f32a125b81c5f90c2c9c8af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Pe=CC=81net?= Date: Tue, 28 Sep 2021 13:32:05 +0200 Subject: [PATCH 03/50] Specify the name when creating a SM for an MLFlow model, rather than the id. ID is autogenerated. --- dataikuapi/dss/project.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dataikuapi/dss/project.py b/dataikuapi/dss/project.py index 5a459a4b..f9dc7b1a 100644 --- a/dataikuapi/dss/project.py +++ b/dataikuapi/dss/project.py @@ -709,22 +709,22 @@ def get_saved_model(self, sm_id): """ return DSSSavedModel(self.client, self.project_key, sm_id) - def create_mlflow_pyfunc_model(self, id, prediction_type = None): + def create_mlflow_pyfunc_model(self, name, prediction_type = None): """ Creates a new external saved model for storing and managing MLFlow models - :param string id: Identifier for the new saved model in the flow + :param string name: Human readable name for the new saved model in the flow :param string prediction_type: Optional (but needed for most operations). One of BINARY_CLASSIFICATION, MULTICLASS or REGRESSION """ - if len(id) != 8: - raise ValueError("model id must be 8 characters long") + if not name: + raise ValueError("name can not be empty") model = { - "id": id, "savedModelType" : "MLFLOW_PYFUNC", - "predictionType" : prediction_type + "predictionType" : prediction_type, + "name": name } - self.client._perform_empty("POST", "/projects/%s/savedmodels/" % self.project_key, body = model) + id = self.client._perform_json("POST", "/projects/%s/savedmodels/" % self.project_key, body = model)["id"] return self.get_saved_model(id) ######################################################## From 7306afc09f7768b7de929d491cafebadf7a13db8 Mon Sep 17 00:00:00 2001 From: Valentin Thorey Date: Thu, 7 Oct 2021 11:40:11 +0200 Subject: [PATCH 04/50] Clean mlflow tmp file --- dataikuapi/dss/savedmodel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dataikuapi/dss/savedmodel.py b/dataikuapi/dss/savedmodel.py index 6c804477..9caa059d 100644 --- a/dataikuapi/dss/savedmodel.py +++ b/dataikuapi/dss/savedmodel.py @@ -134,11 +134,13 @@ def import_mlflow_version_from_path(self, version_id, path, code_env_name = "INH # TODO: Put it in a proper temp folder # TODO: cleanup the archive import shutil + import os shutil.make_archive("tmpmodel", "zip", path) #[, root_dir[, base_dir[, verbose[, dry_run[, owner[, group[, logger]]]]]]]) with open("tmpmodel.zip", "rb") as fp: self.client._perform_empty("POST", "/projects/%s/savedmodels/%s/versions/%s?codeEnvName=%s" % (self.project_key, self.sm_id, version_id, code_env_name), files={"file":("tmpmodel.zip", fp)}) + os.remove("tmpmodel.zip") return self.get_mlflow_version_handler(version_id) From 45cde3a7f3f5bc1142082698ec2561d2272b3f77 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 16 Apr 2021 12:06:57 +0200 Subject: [PATCH 05/50] Added FMClient and get_cloud_credentials --- dataikuapi/__init__.py | 3 +- dataikuapi/fm/tenant.py | 18 +++++++++ dataikuapi/fmclient.py | 81 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 dataikuapi/fm/tenant.py create mode 100644 dataikuapi/fmclient.py diff --git a/dataikuapi/__init__.py b/dataikuapi/__init__.py index 89b5119a..569b1017 100644 --- a/dataikuapi/__init__.py +++ b/dataikuapi/__init__.py @@ -1,8 +1,9 @@ from .dssclient import DSSClient +from .fmclient import FMClient from .apinode_client import APINodeClient from .apinode_admin_client import APINodeAdminClient from .dss.recipe import GroupingRecipeCreator, JoinRecipeCreator, StackRecipeCreator, WindowRecipeCreator, SyncRecipeCreator, SamplingRecipeCreator, SQLQueryRecipeCreator, CodeRecipeCreator, SplitRecipeCreator, SortRecipeCreator, TopNRecipeCreator, DistinctRecipeCreator, DownloadRecipeCreator, PredictionScoringRecipeCreator, ClusteringScoringRecipeCreator -from .dss.admin import DSSUserImpersonationRule, DSSGroupImpersonationRule \ No newline at end of file +from .dss.admin import DSSUserImpersonationRule, DSSGroupImpersonationRule diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py new file mode 100644 index 00000000..4fe5a194 --- /dev/null +++ b/dataikuapi/fm/tenant.py @@ -0,0 +1,18 @@ +class FMCloudCredentials(object): + """ + A Tenant Cloud Credentials in the FM instance + """ + def __init__(self, client, tenant_id, cloud_credentials): + self.client = client + self.tenant_id = tenant_id + self.cloud_credentials = cloud_credentials + + def set_cmk_key(self, cmk_key_id): + self.cloud_credentials['awsCMKId'] = cmk_key_id + + def save(self): + """Saves back the settings to the project""" + + self.client._perform_empty("PUT", "/tenants/%s/cloud-credentials" % (self.tenant_id), + body = self.cloud_credentials) + diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py new file mode 100644 index 00000000..b3275397 --- /dev/null +++ b/dataikuapi/fmclient.py @@ -0,0 +1,81 @@ +import json +from requests import Session +from requests import exceptions +from requests.auth import HTTPBasicAuth + +import os.path as osp +from .utils import DataikuException + +from .fm.tenant import FMCloudCredentials +from .fm.tenant import FMCloudTags + +class FMClient(object): + """Entry point for the FM API client""" + + def __init__(self, host, api_key_id, api_key_secret, extra_headers = None): + """ + Instantiate a new FM API client on the given host with the given API key. + + API keys can be managed in FM on the project page or in the global settings. + + The API key will define which operations are allowed for the client. + """ + self.api_key_id = api_key_id + self.api_key_secret = api_key_secret + self.host = host + self._session = Session() + + if self.api_key_id is not None and self.api_key_secret is not None: + self._session.auth = HTTPBasicAuth(self.api_key_id, self.api_key_secret) + else: + raise ValueError("API Key ID and API Key secret are required") + + if extra_headers is not None: + self._session.headers.update(extra_headers) + + ######################################################## + # Tenant + ######################################################## + + def get_cloud_credentials(self, tenant_id): + """ + Get Tenant's Cloud Credential + + :param string tenant_id + + :return: tenant's cloud credentials + :rtype: :class:`dataikuapi.fm.tenant.FMCloudCredentials` + """ + creds = self._perform_json("GET", "/tenants/%s/cloud-credentials" % tenant_id) + return FMCloudCredentials(self, tenant_id, creds) + + ######################################################## + # Internal Request handling + ######################################################## + + def _perform_http(self, method, path, params=None, body=None, stream=False, files=None, raw_body=None): + if body is not None: + body = json.dumps(body) + if raw_body is not None: + body = raw_body + + try: + http_res = self._session.request( + method, "%s/api/public%s" % (self.host, path), + params=params, data=body, + files = files, + stream = stream) + http_res.raise_for_status() + return http_res + except exceptions.HTTPError: + try: + ex = http_res.json() + except ValueError: + ex = {"message": http_res.text} + raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) + + def _perform_empty(self, method, path, params=None, body=None, files = None, raw_body=None): + self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body) + + def _perform_json(self, method, path, params=None, body=None,files=None, raw_body=None): + return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).json() From 287fc3153f7d0f51bd19cab4b840f18dc0620919 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 27 May 2021 18:17:24 +0200 Subject: [PATCH 06/50] Set tenant_id in FMClient --- dataikuapi/fm/tenant.py | 5 ++--- dataikuapi/fmclient.py | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index 4fe5a194..95f9aefa 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -2,9 +2,8 @@ class FMCloudCredentials(object): """ A Tenant Cloud Credentials in the FM instance """ - def __init__(self, client, tenant_id, cloud_credentials): + def __init__(self, client, cloud_credentials): self.client = client - self.tenant_id = tenant_id self.cloud_credentials = cloud_credentials def set_cmk_key(self, cmk_key_id): @@ -13,6 +12,6 @@ def set_cmk_key(self, cmk_key_id): def save(self): """Saves back the settings to the project""" - self.client._perform_empty("PUT", "/tenants/%s/cloud-credentials" % (self.tenant_id), + self.client._perform_tenant_empty("PUT", "/cloud-credentials", body = self.cloud_credentials) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index b3275397..41ce010a 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -7,12 +7,11 @@ from .utils import DataikuException from .fm.tenant import FMCloudCredentials -from .fm.tenant import FMCloudTags class FMClient(object): """Entry point for the FM API client""" - def __init__(self, host, api_key_id, api_key_secret, extra_headers = None): + def __init__(self, host, api_key_id, api_key_secret, tenant_id, extra_headers = None): """ Instantiate a new FM API client on the given host with the given API key. @@ -23,6 +22,7 @@ def __init__(self, host, api_key_id, api_key_secret, extra_headers = None): self.api_key_id = api_key_id self.api_key_secret = api_key_secret self.host = host + self.__tenant_id = tenant_id self._session = Session() if self.api_key_id is not None and self.api_key_secret is not None: @@ -37,17 +37,15 @@ def __init__(self, host, api_key_id, api_key_secret, extra_headers = None): # Tenant ######################################################## - def get_cloud_credentials(self, tenant_id): + def get_cloud_credentials(self): """ - Get Tenant's Cloud Credential + Get Cloud Credential - :param string tenant_id - - :return: tenant's cloud credentials + :return: Cloud credentials :rtype: :class:`dataikuapi.fm.tenant.FMCloudCredentials` """ - creds = self._perform_json("GET", "/tenants/%s/cloud-credentials" % tenant_id) - return FMCloudCredentials(self, tenant_id, creds) + creds = self._perform_tenant_json("GET", "/cloud-credentials") + return FMCloudCredentials(self, creds) ######################################################## # Internal Request handling @@ -79,3 +77,9 @@ def _perform_empty(self, method, path, params=None, body=None, files = None, raw def _perform_json(self, method, path, params=None, body=None,files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).json() + + def _perform_tenant_json(self, method, path, params=None, body=None,files=None, raw_body=None): + return self._perform_json(method, "/tenants/%s%s" % ( self.__tenant_id, path ), params=params, body=body, files=files, raw_body=raw_body) + + def _perform_tenant_empty(self, method, path, params=None, body=None, files = None, raw_body=None): + self._perform_empty(method, "/tenants/%s%s" % ( self.__tenant_id, path ), params=params, body=body, files=files, raw_body=raw_body) From be994f2ddef16de9aadf380283d6391661541f01 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 27 May 2021 18:18:31 +0200 Subject: [PATCH 07/50] Added FMVirtualNetwork --- dataikuapi/fm/virtualnetworks.py | 5 +++++ dataikuapi/fmclient.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 dataikuapi/fm/virtualnetworks.py diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py new file mode 100644 index 00000000..455bd9b2 --- /dev/null +++ b/dataikuapi/fm/virtualnetworks.py @@ -0,0 +1,5 @@ +class FMVirtualNetwork(object): + def __init__(self, client, virtual_network_settings): + self.client = client + self.settings = virtual_network_settings + self.id = self.settings['id'] diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 41ce010a..a555b6aa 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -7,6 +7,7 @@ from .utils import DataikuException from .fm.tenant import FMCloudCredentials +from .fm.virtualnetworks import FMVirtualNetwork class FMClient(object): """Entry point for the FM API client""" @@ -47,6 +48,34 @@ def get_cloud_credentials(self): creds = self._perform_tenant_json("GET", "/cloud-credentials") return FMCloudCredentials(self, creds) + + ######################################################## + # VirtualNetwork + ######################################################## + + def get_virtual_networks(self): + """ + Get Virtual Networks + + :return: list of virtual networks + :rtype: list of :class:`dataikuapi.fm.tenant.FMVirtualNetwork` + """ + vns = self._perform_tenant_json("GET", "/virtual-networks") + return [ FMVirtualNetwork(self, x) for x in vns] + + def get_virtual_network(self, virtual_network_id): + """ + Get a Virtual Network + + :param str virtual_network_id + + :return: requested virtual network + :rtype: :class:`dataikuapi.fm.tenant.FMVirtualNetwork` + """ + template = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) + return FMVirtualNetwork(self, template) + + ######################################################## # Internal Request handling ######################################################## From f7c241acca31decc12a4386cde9f221d37b2528a Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 27 May 2021 18:23:05 +0200 Subject: [PATCH 08/50] Added FMInstanceSettingsTemplate --- dataikuapi/fm/instancesettingstemplates.py | 6 +++++ dataikuapi/fmclient.py | 31 ++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 dataikuapi/fm/instancesettingstemplates.py diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py new file mode 100644 index 00000000..d9d640ee --- /dev/null +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -0,0 +1,6 @@ +class FMInstanceSettingsTemplate(object): + def __init__(self, client, ist_data): + self.client = client + self.id = ist_data['id'] + self.ist_data = ist_data + diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index a555b6aa..16a7c21c 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -8,6 +8,7 @@ from .fm.tenant import FMCloudCredentials from .fm.virtualnetworks import FMVirtualNetwork +from .fm.instancesettingstemplates import FMInstanceSettingsTemplate class FMClient(object): """Entry point for the FM API client""" @@ -34,6 +35,7 @@ def __init__(self, host, api_key_id, api_key_secret, tenant_id, extra_headers = if extra_headers is not None: self._session.headers.update(extra_headers) + ######################################################## # Tenant ######################################################## @@ -76,6 +78,35 @@ def get_virtual_network(self, virtual_network_id): return FMVirtualNetwork(self, template) + ######################################################## + # Instance settings template + ######################################################## + + def get_instance_templates(self): + """ + Get Instance Settings Templates + + :return: list of instance settings template + :rtype: list of :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` + """ + templates = self._perform_tenant_json("GET", "/instance-settings-templates") + return [ FMInstanceSettingsTemplate(self, x) for x in templates] + + def get_instance_template(self, template_id): + """ + Get an Instance + + :param str template_id + + :return: requested instance settings template + :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + """ + template = self._perform_tenant_json("GET", "/instance-settings-templates/%s" % template_id) + return FMInstanceSettingsTemplate(self, template) + + + + ######################################################## # Internal Request handling ######################################################## From 0fda43b12a3b52b05f7b4ece6dc171f87f56aed1 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 27 May 2021 18:24:04 +0200 Subject: [PATCH 09/50] Added FMInstance --- dataikuapi/fm/instances.py | 11 +++++++ dataikuapi/fmclient.py | 62 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 dataikuapi/fm/instances.py diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py new file mode 100644 index 00000000..9d272a04 --- /dev/null +++ b/dataikuapi/fm/instances.py @@ -0,0 +1,11 @@ +from enum import Enum + +class FMInstance(object): + def __init__(self, client): + self.client = client + +class FMInstanceEncryptionMode(Enum): + NONE = "NONE" + DEFAULT_KEY = "DEFAULT_KEY" + TENANT = "TENANT" + CUSTOM = "CUSTOM" diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 16a7c21c..7a756151 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -8,6 +8,7 @@ from .fm.tenant import FMCloudCredentials from .fm.virtualnetworks import FMVirtualNetwork +from .fm.instances import FMInstance, FMInstanceEncryptionMode from .fm.instancesettingstemplates import FMInstanceSettingsTemplate class FMClient(object): @@ -105,6 +106,67 @@ def get_instance_template(self, template_id): return FMInstanceSettingsTemplate(self, template) + ######################################################## + # Instance + ######################################################## + + def get_instances(self): + """ + Get Instances + + :return: list of instances + :rtype: list of :class:`dataikuapi.fm.tenant.FMInstance` + """ + instances = self._perform_tenant_json("GET", "/instances") + return [ FMInstance(self, **x) for x in instances] + + def get_instance(self, instance_id): + """ + Get an Instance + + :param str instance_id + + :return: Instance + :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + """ + instance = self._perform_json("GET", "/instances/%s" % instance_id) + return FMInstance(self, **instance) + + def create_instance(self, instance_settings_template, virtual_network, label, + dss_node_type="design", image_id=None, + cloud_instance_type=None, data_volume_type=None, data_volume_size=None, + data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=FMInstanceEncryptionMode.NONE, + data_volume_encryption_key=None, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None, + cloud_tags=None, fm_tags=None): + """ + Create an Instance + + :param str instance_id + + :return: Instance + :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + """ + data = { + "virtualNetworkId": virtual_network.id, + "instanceSettingsTemplateId": instance_settings_template.id, + "label": label, + "dssNodeType": dss_node_type, + "imageId": image_id, + "cloudInstanceType": cloud_instance_type, + "dataVolumeType": data_volume_type, + "dataVolumeSizeGB": data_volume_size, + "dataVolumeSizeMaxGB": data_volume_size_max, + "dataVolumeIOPS": data_volume_IOPS, + "dataVolumeEncryption": str(data_volume_encryption), + "dataVolumeEncryptionKey": data_volume_encryption_key, + "awsRootVolumeSizeGB": aws_root_volume_size, + "awsRootVolumeType": aws_root_volume_type, + "awsRootVolumeIOPS": aws_root_volume_IOPS, + "cloudTags": cloud_tags, + "fmTags": fm_tags + } + instance = self._perform_tenant_json("POST", "/instances", body=data) + return FMInstance(self, instance) ######################################################## From 7dcf67a878741087671e00ed287ef9729dae88cf Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 1 Jun 2021 17:19:14 +0200 Subject: [PATCH 10/50] fm public api: Add instance actions --- dataikuapi/fm/instances.py | 18 +++++++++++++++++- dataikuapi/fmclient.py | 6 +++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 9d272a04..69b9ee55 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,8 +1,24 @@ from enum import Enum class FMInstance(object): - def __init__(self, client): + def __init__(self, client, instance_data): self.client = client + self.instance_data = instance_data + print(instance_data) + self.id = instance_data['id'] + + def reprovision(self): + status = self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) + print(status) + + def deprovision(self): + status = self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) + print(status) + + def status(self): + status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) + return status + class FMInstanceEncryptionMode(Enum): NONE = "NONE" diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 7a756151..ed6b081e 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -129,8 +129,8 @@ def get_instance(self, instance_id): :return: Instance :rtype: :class:`dataikuapi.fm.tenant.FMInstance` """ - instance = self._perform_json("GET", "/instances/%s" % instance_id) - return FMInstance(self, **instance) + instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) + return FMInstance(self, instance) def create_instance(self, instance_settings_template, virtual_network, label, dss_node_type="design", image_id=None, @@ -157,7 +157,7 @@ def create_instance(self, instance_settings_template, virtual_network, label, "dataVolumeSizeGB": data_volume_size, "dataVolumeSizeMaxGB": data_volume_size_max, "dataVolumeIOPS": data_volume_IOPS, - "dataVolumeEncryption": str(data_volume_encryption), + "dataVolumeEncryption": data_volume_encryption.value, "dataVolumeEncryptionKey": data_volume_encryption_key, "awsRootVolumeSizeGB": aws_root_volume_size, "awsRootVolumeType": aws_root_volume_type, From e6210ad40aa94492534e680cf1bf97f2298d09cb Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 2 Jun 2021 09:55:48 +0200 Subject: [PATCH 11/50] fm: Add FMInstanceStatus --- dataikuapi/fm/instances.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 69b9ee55..c63afcfe 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -15,9 +15,9 @@ def deprovision(self): status = self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) print(status) - def status(self): + def get_status(self): status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) - return status + return FMInstanceStatus(status) class FMInstanceEncryptionMode(Enum): @@ -25,3 +25,12 @@ class FMInstanceEncryptionMode(Enum): DEFAULT_KEY = "DEFAULT_KEY" TENANT = "TENANT" CUSTOM = "CUSTOM" + + +class FMInstanceStatus(dict): + """A class holding read-only information about an Instance. + This class should not be created directly. Instead, use :meth:`FMInstance.get_info` + """ + def __init__(self, data): + """Do not call this directly, use :meth:`FMInstance.get_status`""" + super(FMInstanceStatus, self).__init__(data) From a53fbfe5048f613b55d33ae0afb3770097649a8d Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 2 Jun 2021 15:37:55 +0200 Subject: [PATCH 12/50] fm: Save and restart_dss --- dataikuapi/fm/instances.py | 32 +++++++++++++++++++++++++++----- dataikuapi/fmclient.py | 12 ++++++------ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index c63afcfe..af228243 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,19 +1,41 @@ from enum import Enum class FMInstance(object): + """ + A handle to interact with a DSS instance. + Do not create this directly, use :meth:`FMClient.get_instance` or :meth: `FMClient.create_instance` + """ def __init__(self, client, instance_data): self.client = client self.instance_data = instance_data - print(instance_data) self.id = instance_data['id'] def reprovision(self): - status = self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) - print(status) + """ + Reprovision the DSS physical instance + """ + self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) def deprovision(self): - status = self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) - print(status) + """ + Deprovision the DSS physical instance + """ + self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) + + def restart_dss(self): + """ + Restart the DSS running on the physical instance + """ + self.client._perform_tenant_json("GET", "/instances/%s/actions/restart-dss" % self.id) + + def save(self): + """ + Update the DSS Instance. + """ + self.client._perform_tenant_empty("PUT", "/instances/%s" % self.id, body=self.instance_data) + self.instance_data = self.client._perform_tenant_json("GET", "/instances/%s" % self.id) + + def get_status(self): status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index ed6b081e..f7981599 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -58,7 +58,7 @@ def get_cloud_credentials(self): def get_virtual_networks(self): """ - Get Virtual Networks + List all Virtual Networks :return: list of virtual networks :rtype: list of :class:`dataikuapi.fm.tenant.FMVirtualNetwork` @@ -85,7 +85,7 @@ def get_virtual_network(self, virtual_network_id): def get_instance_templates(self): """ - Get Instance Settings Templates + List all Instance Settings Templates :return: list of instance settings template :rtype: list of :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` @@ -95,7 +95,7 @@ def get_instance_templates(self): def get_instance_template(self, template_id): """ - Get an Instance + Get an Instance Template :param str template_id @@ -112,7 +112,7 @@ def get_instance_template(self, template_id): def get_instances(self): """ - Get Instances + List all DSS Instances :return: list of instances :rtype: list of :class:`dataikuapi.fm.tenant.FMInstance` @@ -122,7 +122,7 @@ def get_instances(self): def get_instance(self, instance_id): """ - Get an Instance + Get a DSS Instance :param str instance_id @@ -139,7 +139,7 @@ def create_instance(self, instance_settings_template, virtual_network, label, data_volume_encryption_key=None, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None, cloud_tags=None, fm_tags=None): """ - Create an Instance + Create a DSS Instance :param str instance_id From b49c8e36040607628feb054d1e621ca8d4de0235 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 2 Jun 2021 21:47:46 +0200 Subject: [PATCH 13/50] fm: create instance template --- dataikuapi/fm/instances.py | 11 +-- dataikuapi/fm/instancesettingstemplates.py | 41 +++++++++++ dataikuapi/fmclient.py | 84 ++++++++++++++++++++-- 3 files changed, 126 insertions(+), 10 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index af228243..ab4ba2b0 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -12,13 +12,13 @@ def __init__(self, client, instance_data): def reprovision(self): """ - Reprovision the DSS physical instance + Reprovision the physical DSS instance """ self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) def deprovision(self): """ - Deprovision the DSS physical instance + Deprovision the physical DSS instance """ self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) @@ -30,14 +30,15 @@ def restart_dss(self): def save(self): """ - Update the DSS Instance. + Update the Instance. """ self.client._perform_tenant_empty("PUT", "/instances/%s" % self.id, body=self.instance_data) self.instance_data = self.client._perform_tenant_json("GET", "/instances/%s" % self.id) - - def get_status(self): + """ + Get the physical DSS instance's status + """ status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) return FMInstanceStatus(status) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index d9d640ee..6dd0782e 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -1,6 +1,47 @@ +from enum import Enum +import json + class FMInstanceSettingsTemplate(object): def __init__(self, client, ist_data): self.client = client self.id = ist_data['id'] self.ist_data = ist_data + def save(self): + """ + Update the Instance Settings Template. + """ + self.client._perform_tenant_empty("PUT", "/instance-settings-templates/%s" % self.id, body=self.ist_data) + self.ist_data = self.client._perform_tenant_json("GET", "/instance-settings-templates/%s" % self.id) + +class FMSetupAction(dict): + def __init__(self, setupActionType, params=None): + """ + param: object setupActionType: the type (`:class: FMSetupActionType`) of the SetupAction + param: str params: the parameters of the SetupAction in a json-encoded string + """ + data = { + "type": setupActionType.value, + } + if params: + data['params'] = params + + super(FMSetupAction, self).__init__(data) + +class FMSetupActionAddAuthorizedKey(FMSetupAction): + def __init__(self, ssh_key): + super(FMSetupActionAddAuthorizedKey, self).__init__(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key }) + +class FMSetupActionType(Enum): + RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" + INSTALL_SYSTEM_PACKAGES="INSTALL_SYSTEM_PACKAGES" + INSTALL_DSS_PLUGINS_FROM_STORE="INSTALL_DSS_PLUGINS_FROM_STORE" + SETUP_K8S_AND_SPARK="SETUP_K8S_AND_SPARK" + SETUP_RUNTIME_DATABASE="SETUP_RUNTIME_DATABASE" + SETUP_MUS_AUTOCREATE="SETUP_MUS_AUTOCREATE" + SETUP_UI_CUSTOMIZATION="SETUP_UI_CUSTOMIZATION" + SETUP_MEMORY_SETTINGS="SETUP_MEMORY_SETTINGS" + SETUP_GRAPHICS_EXPORT="SETUP_GRAPHICS_EXPORT" + ADD_AUTHORIZED_KEY="ADD_AUTHORIZED_KEY" + INSTALL_JDBC_DRIVER="INSTALL_JDBC_DRIVER" + SETUP_ADVANCED_SECURITY="SETUP_ADVANCED_SECURITY" diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index f7981599..247f0a66 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -100,12 +100,69 @@ def get_instance_template(self, template_id): :param str template_id :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + :rtype: :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` """ template = self._perform_tenant_json("GET", "/instance-settings-templates/%s" % template_id) return FMInstanceSettingsTemplate(self, template) + def create_instance_template(self, label, + setupActions=None, license=None, + awsKeyPairName=None, startupInstanceProfileArn=None, runtimeInstanceProfileArn=None, + restrictAwsMetadataServerAccess=True, dataikuAwsAPIAccessMode="NONE", dataikuAwsKeypairStorageMode=None, + dataikuAwsAccessKeyId=None, dataikuAwsSecretAccessKey=None, + dataikuAwsSecretAccessKeyAwsSecretName=None, awsSecretsManagerRegion=None, + azureSshKey=None, startupManagedIdentity=None, runtimeManagedIdentity=None): + """ + Create an Instance Template + + :param str label: The label of the Instance Settings Template + + :param list setupActions: Optional, a list of `:class: FMSetupAction` to be played on an instance + :param str license: Optional, overrides the license set in Cloud Setup + + :param str awsKeyPairName: Optional, AWS Only, the name of an AWS key pair to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. + :param str startupInstanceProfileArn: Optional, AWS Only, the ARN of the Instance profile assigned to the DSS instance at startup time + :param str runtimeInstanceProfileArn: Optional, AWS Only, the ARN of the Instance profile assigned to the DSS instance at runtime + :param boolean restrictAwsMetadataServerAccess: Optional, AWS Only, If true, restrict the access to the metadata server access. Defaults to true + :param str dataikuAwsAPIAccessMode: Optional, AWS Only, the access mode DSS is using to connect to the AWS API. If "NONE" DSS will use the Instance Profile, If "KEYPAIR", an AWS access key id and secret will be securely given to the dataiku account. + :param str dataikuAwsKeypairStorageMode: Optional, AWS Only, the storage mode of the AWS api key. Accepts "NONE", "INLINE_ENCRYPTED" or "AWS_SECRETS_MANAGER" + :param str dataikuAwsAccessKeyId: Optional, AWS Only, AWS Access Key ID. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" + :param str dataikuAwsSecretAccessKey: Optional, AWS Only, AWS Access Key Secret. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" and dataikuAwsKeypairStorageMode is "INLINE_ENCRYPTED" + :param str dataikuAwsSecretAccessKeyAwsSecretName: Optional, AWS Only, ASM secret name. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" and dataikuAwsKeypairStorageMode is "AWS_SECRET_MANAGER" + :param str awsSecretsManagerRegion: Optional, AWS Only + + :param str azureSshKey: Optional, Azure Only, the ssh public key to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. + :param str startupManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at startup time + :param str runtimeManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at runtime + + :return: requested instance settings template + :rtype: :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` + """ + + data = { + "label": label, + "setupActions": setupActions, + "license": license, + "awsKeyPairName": awsKeyPairName, + "startupInstanceProfileArn": startupInstanceProfileArn, + "runtimeInstanceProfileArn": runtimeInstanceProfileArn, + "restrictAwsMetadataServerAccess": restrictAwsMetadataServerAccess, + "dataikuAwsAPIAccessMode": "dataikuAwsAPIAccessMode", + "dataikuAwsKeypairStorageMode": dataikuAwsKeypairStorageMode, + "dataikuAwsAccessKeyId": dataikuAwsAccessKeyId, + "dataikuAwsSecretAccessKey": dataikuAwsSecretAccessKey, + "dataikuAwsSecretAccessKeyAwsSecretName": dataikuAwsSecretAccessKeyAwsSecretName, + "awsSecretsManagerRegion": awsSecretsManagerRegion, + "azureSshKey": azureSshKey, + "startupManagedIdentity": startupManagedIdentity, + "runtimeManagedIdentity": runtimeManagedIdentity + } + + template = self._perform_tenant_json("POST", "/instance-settings-templates", body=data) + return FMInstanceSettingsTemplate(self, template) + + ######################################################## # Instance ######################################################## @@ -132,8 +189,8 @@ def get_instance(self, instance_id): instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return FMInstance(self, instance) - def create_instance(self, instance_settings_template, virtual_network, label, - dss_node_type="design", image_id=None, + def create_instance(self, instance_settings_template, virtual_network, label, image_id, + dss_node_type="design", cloud_instance_type=None, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=FMInstanceEncryptionMode.NONE, data_volume_encryption_key=None, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None, @@ -141,7 +198,24 @@ def create_instance(self, instance_settings_template, virtual_network, label, """ Create a DSS Instance - :param str instance_id + :param str instance_settings_template: The instance settings template id this instance should be based on + :param str virtual_network: The virtual network where the instance should be spawned + :param str label: The label of the instance + :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) + + :param str dss_node_type: Optional , the type of the dss node to create. Defaults to "design" + :param str cloud_instance_type: Optional, Machine type + :param str data_volume_type: Optional, Data volume type + :param int data_volume_size: Optional, Data volume initial size + :param int data_volume_size_max: Optional, Data volume maximum size + :param int data_volume_IOPS: Optional, Data volume IOPS + :param object data_volume_encryption: Optional, a :class:`FMInstanceEncryptionMode` setting the encryption mode of the data volume + :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM + :param int aws_root_volume_size: Optional, the root volume size + :param str aws_root_volume_type: Optional, the root volume type + :param int aws_root_volume_IOPS: Optional, the root volume IOPS + :param dict cloud_tags: Optional, a key value dictionary of tags to be applied on the cloud resources + :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager :return: Instance :rtype: :class:`dataikuapi.fm.tenant.FMInstance` @@ -178,7 +252,7 @@ def _perform_http(self, method, path, params=None, body=None, stream=False, file body = json.dumps(body) if raw_body is not None: body = raw_body - + print(body) try: http_res = self._session.request( method, "%s/api/public%s" % (self.host, path), From 9825ca7347263e673d772a736f485fb5180db861 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 3 Jun 2021 12:49:30 +0200 Subject: [PATCH 14/50] FM: future --- dataikuapi/fm/future.py | 85 ++++++++++++++++++++++++++++++++++++++ dataikuapi/fm/instances.py | 19 +++++++-- dataikuapi/fmclient.py | 18 ++++---- 3 files changed, 110 insertions(+), 12 deletions(-) create mode 100644 dataikuapi/fm/future.py diff --git a/dataikuapi/fm/future.py b/dataikuapi/fm/future.py new file mode 100644 index 00000000..60d31b57 --- /dev/null +++ b/dataikuapi/fm/future.py @@ -0,0 +1,85 @@ +import sys, time + +class FMFuture(object): + """ + A future on the DSS instance + """ + def __init__(self, client, job_id, state=None, result_wrapper=lambda result: result): + self.client = client + self.job_id = job_id + self.state = state + self.state_is_peek = True + self.result_wrapper = result_wrapper + + @staticmethod + def from_resp(client, resp,result_wrapper=lambda result: result): + """Creates a DSSFuture from a parsed JSON response""" + return FMFuture(client, resp.get('jobId', None), state=resp, result_wrapper=result_wrapper) + + @classmethod + def get_result_wait_if_needed(cls, client, ret): + if 'jobId' in ret: + future = FMFuture(client, ret["jobId"], ret) + future.wait_for_result() + return future.get_result() + else: + return ret['result'] + + def abort(self): + """ + Abort the future + """ + return self.client._perform_tenant_empty("DELETE", "/futures/%s" % self.job_id) + + def get_state(self): + """ + Get the status of the future, and its result if it's ready + """ + self.state = self.client._perform_tenant_json( + "GET", "/futures/%s" % self.job_id, params={'peek' : False}) + self.state_is_peek = False + return self.state + + def peek_state(self): + """ + Get the status of the future, and its result if it's ready + """ + self.state = self.client._perform_tenant_json( + "GET", "/futures/%s" % self.job_id, params={'peek' : True}) + self.state_is_peek = True + return self.state + + def get_result(self): + """ + Get the future result if it's ready, raises an Exception otherwise + """ + if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: + self.get_state() + if self.state.get('hasResult', False): + return self.result_wrapper(self.state.get('result', None)) + else: + raise Exception("Result not ready") + + def has_result(self): + """ + Checks whether the future has a result ready + """ + if self.state is None or not self.state.get('hasResult', False): + self.get_state() + return self.state.get('hasResult', False) + + def wait_for_result(self): + """ + Wait and get the future result + """ + if self.state.get('hasResult', False): + return self.result_wrapper(self.state.get('result', None)) + if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: + self.get_state() + while not self.state.get('hasResult', False): + time.sleep(5) + self.get_state() + if self.state.get('hasResult', False): + return self.result_wrapper(self.state.get('result', None)) + else: + raise Exception("No result") diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index ab4ba2b0..41d41374 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,4 +1,5 @@ from enum import Enum +from .future import FMFuture class FMInstance(object): """ @@ -13,20 +14,32 @@ def __init__(self, client, instance_data): def reprovision(self): """ Reprovision the physical DSS instance + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the reprovision process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) + future = self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) + return FMFuture.from_resp(self.client, future) def deprovision(self): """ Deprovision the physical DSS instance + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deprovision process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) + future = self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) + return FMFuture.from_resp(self.client, future) def restart_dss(self): """ Restart the DSS running on the physical instance + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the restart process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - self.client._perform_tenant_json("GET", "/instances/%s/actions/restart-dss" % self.id) + future = self.client._perform_tenant_json("GET", "/instances/%s/actions/restart-dss" % self.id) + return FMFuture.from_resp(self.client, future) def save(self): """ diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 247f0a66..410c9794 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -61,7 +61,7 @@ def get_virtual_networks(self): List all Virtual Networks :return: list of virtual networks - :rtype: list of :class:`dataikuapi.fm.tenant.FMVirtualNetwork` + :rtype: list of :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ vns = self._perform_tenant_json("GET", "/virtual-networks") return [ FMVirtualNetwork(self, x) for x in vns] @@ -73,7 +73,7 @@ def get_virtual_network(self, virtual_network_id): :param str virtual_network_id :return: requested virtual network - :rtype: :class:`dataikuapi.fm.tenant.FMVirtualNetwork` + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ template = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) return FMVirtualNetwork(self, template) @@ -100,7 +100,7 @@ def get_instance_template(self, template_id): :param str template_id :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ template = self._perform_tenant_json("GET", "/instance-settings-templates/%s" % template_id) return FMInstanceSettingsTemplate(self, template) @@ -118,7 +118,7 @@ def create_instance_template(self, label, :param str label: The label of the Instance Settings Template - :param list setupActions: Optional, a list of `:class: FMSetupAction` to be played on an instance + :param list setupActions: Optional, a list of :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` to be played on an instance :param str license: Optional, overrides the license set in Cloud Setup :param str awsKeyPairName: Optional, AWS Only, the name of an AWS key pair to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. @@ -137,7 +137,7 @@ def create_instance_template(self, label, :param str runtimeManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at runtime :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ data = { @@ -172,7 +172,7 @@ def get_instances(self): List all DSS Instances :return: list of instances - :rtype: list of :class:`dataikuapi.fm.tenant.FMInstance` + :rtype: list of :class:`dataikuapi.fm.instances.FMInstance` """ instances = self._perform_tenant_json("GET", "/instances") return [ FMInstance(self, **x) for x in instances] @@ -184,7 +184,7 @@ def get_instance(self, instance_id): :param str instance_id :return: Instance - :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + :rtype: :class:`dataikuapi.fm.instances.FMInstance` """ instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return FMInstance(self, instance) @@ -209,7 +209,7 @@ def create_instance(self, instance_settings_template, virtual_network, label, im :param int data_volume_size: Optional, Data volume initial size :param int data_volume_size_max: Optional, Data volume maximum size :param int data_volume_IOPS: Optional, Data volume IOPS - :param object data_volume_encryption: Optional, a :class:`FMInstanceEncryptionMode` setting the encryption mode of the data volume + :param object data_volume_encryption: Optional, a :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode` setting the encryption mode of the data volume :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM :param int aws_root_volume_size: Optional, the root volume size :param str aws_root_volume_type: Optional, the root volume type @@ -218,7 +218,7 @@ def create_instance(self, instance_settings_template, virtual_network, label, im :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager :return: Instance - :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + :rtype: :class:`dataikuapi.fm.instances.FMInstance` """ data = { "virtualNetworkId": virtual_network.id, From d1ee8d6e0e605d25ec805156a1b0808cb3b1637e Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 3 Jun 2021 13:27:02 +0200 Subject: [PATCH 15/50] fm: delete instance --- dataikuapi/fm/instances.py | 10 ++++++++++ dataikuapi/fmclient.py | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 41d41374..915ae874 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -55,6 +55,16 @@ def get_status(self): status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) return FMInstanceStatus(status) + def delete(self): + """ + Delete the DSS instance + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` + """ + future = self.client._perform_tenant_json("GET", "/instances/%s/actions/delete" % self.id) + return FMFuture.from_resp(self.client, future) + class FMInstanceEncryptionMode(Enum): NONE = "NONE" diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 410c9794..8a3af799 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -252,7 +252,6 @@ def _perform_http(self, method, path, params=None, body=None, stream=False, file body = json.dumps(body) if raw_body is not None: body = raw_body - print(body) try: http_res = self._session.request( method, "%s/api/public%s" % (self.host, path), From a221e17ff66fde4b1118b22949668ca790396808 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 3 Jun 2021 13:47:45 +0200 Subject: [PATCH 16/50] fm: Add delete instance settings template --- dataikuapi/fm/instancesettingstemplates.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 6dd0782e..3d2439cb 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -1,5 +1,6 @@ from enum import Enum import json +from dataikuapi.fm.future import FMFuture class FMInstanceSettingsTemplate(object): def __init__(self, client, ist_data): @@ -14,6 +15,16 @@ def save(self): self.client._perform_tenant_empty("PUT", "/instance-settings-templates/%s" % self.id, body=self.ist_data) self.ist_data = self.client._perform_tenant_json("GET", "/instance-settings-templates/%s" % self.id) + def delete(self): + """ + Delete the DSS Instance Settings Template. + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` + """ + future = self.client._perform_tenant_json("DELETE", "/instance-settings-templates/%s" % self.id) + return FMFuture.from_resp(self.client, future) + class FMSetupAction(dict): def __init__(self, setupActionType, params=None): """ From 2e62ecf698cb6baf13defa50fc019dae9f50c1f6 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 3 Jun 2021 16:30:01 +0200 Subject: [PATCH 17/50] fm: Virtual network management --- dataikuapi/fm/virtualnetworks.py | 124 ++++++++++++++++++++++++++++++- dataikuapi/fmclient.py | 50 ++++++++++++- 2 files changed, 169 insertions(+), 5 deletions(-) diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index 455bd9b2..b1748d0f 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -1,5 +1,123 @@ +from dataikuapi.fm.future import FMFuture + class FMVirtualNetwork(object): - def __init__(self, client, virtual_network_settings): + def __init__(self, client, vn_data): self.client = client - self.settings = virtual_network_settings - self.id = self.settings['id'] + self.vn_data = vn_data + self.id = self.vn_data['id'] + + def save(self): + """ + Update the Virtual Network. + """ + self.client._perform_tenant_empty("PUT", "/virtual-networks/%s" % self.id, body=self.vn_data) + self.vn_data = self.client._perform_tenant_json("GET", "/virtual-networks/%s" % self.id) + + def delete(self): + """ + Delete the DSS Instance Settings Template. + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` + """ + future = self.client._perform_tenant_json("DELETE", "/virtual-networks/%s" % self.id) + return FMFuture.from_resp(self.client, future) + + def set_fleet_management(self, enable, event_server=None, deployer_management="NO_MANAGED_DEPLOYER"): + """ + When enabled, all instances in this virtual network know each other and can centrally manage deployer and logs centralization + + :param boolean enable: Enable or not the Fleet Management + + :param str event_server: Optional, Node name of the node that should act as the centralized event server for logs concentration. Audit logs of all design, deployer and automation nodes will automatically be sent there. + :param str deployer_management: Optional, Accepts: + - "NO_MANAGED_DEPLOYER": Do not manage the the deployer. This is the default mode. + - "CENTRAL_DEPLOYER": Central deployer. Recommanded if you have more than one design node or may have more than one design node in the future. + - "EACH_DESIGN_NODE": Deployer from design. Recommanded if you have a single design node and want a simpler setup. + """ + + self.vn_data['managedNodesDirectory'] = enable + self.vn_data['eventServerNodeLabel'] = event_server + self.vn_data['nodesDirectoryDeployerMode'] = deployer_management + self.save() + + def set_dns_strategy(self, assign_domain_name, aws_private_ip_zone53_id=None, aws_public_ip_zone53_id=None, azure_dns_zone_id=None): + """ + Set the DNS strategy for this virtual network + + :param boolean assign_domain_name: If false, don't assign domain names, use ip_only + :param str aws_private_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for private ip + :param str aws_public_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for public ip + :param str azure_dns_zone_id: Optional, Azure Only, the ID of the Azure DNS zone to use + """ + + if assign_domain_name: + self.vn_data['dnsStrategy'] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" + self.vn_data['awsRoute53PrivateIPZoneId'] = aws_private_ip_zone53_id + self.vn_data['awsRoute53PublicIPZoneId'] = aws_public_ip_zone53_id + self.vn_data['azureDnsZoneId'] = azure_dns_zone_id + else : + self.vn_data['dnsStrategy'] = "NONE" + + self.save() + + def set_https_strategy(self, https_strategy): + """ + Set the HTTPS strategy for this virtual network + + :param object: a :class:`dataikuapi.fm.virtualnetworks.FMHTTPSStrategy` + """ + self.vn_data.update(https_strategy) + self.save() + +class FMHTTPSStrategy(dict): + def __init__(self, data, https_strategy, http_redirect=False): + """ + A class holding HTTPS Strategy for Virtual Network + + Do not create this directly, use: + - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.disable` to use HTTP only + - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.self_signed` to use self-signed certificates + - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.custom_cert` to use custom certificates + - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.lets_encrypt` to use Let's Encrypt + """ + super(FMHTTPSStrategy, self).__init__(data) + self['httpsStrategy'] = https_strategy + if http_redirect: + self['httpStrategy'] = "REDIRECT" + else: + self['httpStrategy'] = "DISABLE" + + @staticmethod + def disable(): + """ + Use HTTP only + """ + return FMHTTPSStrategy({}, "NONE", False) + + @staticmethod + def self_signed(http_redirect): + """ + Use self-signed certificates + + :param bool http_redirect: If true, HTTP is redirected to HTTPS. If false, HTTP is disabled. Defaults to false + """ + return FMHTTPSStrategy({}, "SELF_SIGNED", http_redirect) + + @staticmethod + def custom_cert(http_redirect): + """ + Use a custom certificate for each instance + + :param bool http_redirect: If true, HTTP is redirected to HTTPS. If false, HTTP is disabled. Defaults to false + """ + return FMHTTPSStrategy({}, "CUSTOM_CERTIFICATE", http_redirect) + + @staticmethod + def lets_encrypt(contact_mail): + """ + Use Let's Encrypt to generate https certificates + + :param str contact_mail: The contact email provided to Let's Encrypt + """ + return FMHTTPSStrategy({"contactMail": contact_mail}, "LETSENCRYPT", True) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 8a3af799..93e8f0dc 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -75,8 +75,54 @@ def get_virtual_network(self, virtual_network_id): :return: requested virtual network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ - template = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) - return FMVirtualNetwork(self, template) + vn = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) + return FMVirtualNetwork(self, vn) + + def create_virtual_network(self, + label, + awsVpcId=None, + awsSubnetId=None, + awsAutoCreateSecurityGroups=False, + awsSecurityGroups=None, + azureVnId=None, + azureSubnetId=None, + azureAutoUpdateSecurityGroups=None, + internetAccessMode = "YES"): + """ + Create a Virtual Network + + :param str label: The label of the Virtual Network + + :param str awsVpcId: AWS Only, ID of the VPC to use + :param str awsSubnetId: AWS Only, ID of the subnet to use + :param boolean awsAutoCreateSecurityGroups: Optional, AWS Only, If false, do not create security groups automatically. Defaults to false + :param list awsSecurityGroups: Optional, AWS Only, A list of up to 5 security group ids to assign to the instances created in this virtual network. Ignored if awsAutoCreateSecurityGroups is true + + :param str azureVnId: Azure Only, ID of the Azure Virtual Network to use + :param str azureSubnetId: Azure Only, ID of the subnet to use + :param boolean azureAutoUpdateSecurityGroups: Azure Only, Auto update the subnet security group + + :param str internetAccessMode: Optional, The internet access mode of the instances created in this virtual network. Accepts "YES", "NO", "EGRESS_ONLY". Defaults to "YES" + + :return: requested instance settings template + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + """ + + data = { + "label": label, + "awsVpcId": awsVpcId, + "awsSubnetId": awsSubnetId, + "awsAutoCreateSecurityGroups": awsAutoCreateSecurityGroups, + "awsSecurityGroups": awsSecurityGroups, + "azureVnId": azureVnId, + "azureSubnetId": azureSubnetId, + "azureAutoUpdateSecurityGroups": azureAutoUpdateSecurityGroups, + "internetAccessMode": internetAccessMode, + "mode": "EXISTING_MONOTENANT" + } + + vn = self._perform_tenant_json("POST", "/virtual-networks", body=data) + return FMVirtualNetwork(self, vn) ######################################################## From 4627c9cffb84fd1b230ca64c18bd85662c938c1a Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 3 Jun 2021 17:53:21 +0200 Subject: [PATCH 18/50] fm: Instance settings update --- dataikuapi/fm/instances.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 915ae874..b6f95cb2 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -65,6 +65,43 @@ def delete(self): future = self.client._perform_tenant_json("GET", "/instances/%s/actions/delete" % self.id) return FMFuture.from_resp(self.client, future) + def set_automated_snapshots(self, enable, period, keep=0): + """ + Set the automated snapshots policy for this instance + + :param boolean enable: Enable the automated snapshots + :param int period: The time period between 2 snapshot in hours + :param int keep: Optional, the number of snapshot to keep. Use 0 to keep all snapshots. Defaults to 0. + """ + self.instance_data['enableAutomatedSnapshot'] = enable + self.instance_data['automatedSnapshotPeriod'] = period + self.instance_data['automatedSnapshotRetention'] = keep + self.save() + + def set_elastic_ip(self, enable, elasticip_allocation_id): + """ + Set a public elastic ip for this instance + + :param boolan enable: Enable the elastic ip allocation + :param str elaticip_allocation_id: The AWS ElasticIP allocation ID or the Azure Public IP ID + """ + self.instance_data['awsAssignElasticIP'] = enable + self.instance_data['awsElasticIPAllocationId'] = elasticip_allocation_id + self.instance_data['azureAssignElasticIP'] = enable + self.instance_data['azurePublicIPId'] = elasticip_allocation_id + self.save() + + def set_custom_certificate(self, pem_data): + """ + Set the custom certificate for this instance + + Only needed when Virtual Network HTTPS Strategy is set to Custom Certificate + + param: str pem_data: The SSL certificate + """ + self.instance_data['sslCertificatePEM'] = pem_data + self.save() + class FMInstanceEncryptionMode(Enum): NONE = "NONE" From dfe7a85c0af283d86d95490482f2d571d1dd1897 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 1 Sep 2021 14:28:49 +0200 Subject: [PATCH 19/50] Add __init__.py in fm module --- dataikuapi/fm/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 dataikuapi/fm/__init__.py diff --git a/dataikuapi/fm/__init__.py b/dataikuapi/fm/__init__.py new file mode 100644 index 00000000..e69de29b From 3a484b5a54d7119c0cbde38047f3aa57dbb14510 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 1 Sep 2021 16:15:46 +0200 Subject: [PATCH 20/50] FM Tenant: Update license --- dataikuapi/fm/tenant.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index 95f9aefa..4cafd8e6 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -1,3 +1,4 @@ +import json class FMCloudCredentials(object): """ A Tenant Cloud Credentials in the FM instance @@ -9,6 +10,36 @@ def __init__(self, client, cloud_credentials): def set_cmk_key(self, cmk_key_id): self.cloud_credentials['awsCMKId'] = cmk_key_id + def set_static_license(self, license_file=None, license_string=None): + """ + Set a default static license for the DSS instances + + Requires either a license_file or a license_string + + :param str license_file: Optional, load the license from a json file + :param str license_string: Optional, load the license from a json string + """ + if license_file is not None: + with open(license_file) as json_file: + license = json.load(json_file) + elif license_string is not None: + license = json.load(license_string) + else: + raise ValueError("a valid license_file or license_string needs to be provided") + self.cloud_credentials['licenseMode'] = 'STATIC' + self.cloud_credentials['license'] = json.dumps(license, indent=2) + + def set_automatically_updated_license(self, license_token): + """ + Set an automatically updated license for the DSS instances + + :param str license_token: License token + """ + if license_token is None: + raise ValueError("a valid license_token needs to be provided") + self.cloud_credentials['licenseMode'] = 'AUTO_UPDATE' + self.cloud_credentials['licenseToken'] = license_token + def save(self): """Saves back the settings to the project""" From 0fbbf2e960266c8608323fdad8c3a1735ae0139f Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 1 Sep 2021 17:35:17 +0200 Subject: [PATCH 21/50] FM: Add helper to set cloud credentials --- dataikuapi/fm/tenant.py | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index 4cafd8e6..67c97dcd 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -9,6 +9,7 @@ def __init__(self, client, cloud_credentials): def set_cmk_key(self, cmk_key_id): self.cloud_credentials['awsCMKId'] = cmk_key_id + self.save() def set_static_license(self, license_file=None, license_string=None): """ @@ -28,6 +29,7 @@ def set_static_license(self, license_file=None, license_string=None): raise ValueError("a valid license_file or license_string needs to be provided") self.cloud_credentials['licenseMode'] = 'STATIC' self.cloud_credentials['license'] = json.dumps(license, indent=2) + self.save() def set_automatically_updated_license(self, license_token): """ @@ -39,6 +41,16 @@ def set_automatically_updated_license(self, license_token): raise ValueError("a valid license_token needs to be provided") self.cloud_credentials['licenseMode'] = 'AUTO_UPDATE' self.cloud_credentials['licenseToken'] = license_token + self.save() + + def set_authentication(self, authentication): + """ + Set the authentication for the tenant + + :param object: a :class:`dataikuapi.fm.tenant.FMCloudAuthentication` + """ + self.cloud_credentials.update(authentication) + self.save() def save(self): """Saves back the settings to the project""" @@ -46,3 +58,60 @@ def save(self): self.client._perform_tenant_empty("PUT", "/cloud-credentials", body = self.cloud_credentials) + +class FMCloudAuthentication(dict): + def __init__(self, data): + """ + A class holding the Cloud Authentication information + + Do not create this directly, use: + - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_same_as_fm` to use the same authentication as Fleet Manager + - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_iam_role` to use a custom IAM Role + - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_keypair` to use a AWS Access key ID and AWS Secret Access Key pair + """ + super(FMCloudAuthentication, self).__init__(data) + + @staticmethod + def aws_same_as_fm(): + """ + AWS Only: Use the same authentication as Fleet Manager + """ + return FMCloudAuthentication({"awsAuthenticationMode": "SAME_AS_FM"}) + + @staticmethod + def aws_iam_role(role_arn): + """ + AWS Only: Use an IAM Role + + params: str role_arn: ARN of the IAM Role + """ + return FMCloudAuthentication({"awsAuthenticationMode": "IAM_ROLE", "awsIAMRoleARN": role_arn}) + + @staticmethod + def aws_keypair(access_key_id, secret_access_key): + """ + AWS Only: Use an AWS Access Key + + :param str access_key_id: AWS Access Key ID + :param str secret_access_key: AWS Secret Access Key + """ + return FMCloudAuthentication({"awsAuthenticationMode": "KEYPAIR", "awsAccessKeyId": access_key_id, "awsSecretAccessKey": secret_access_key}) + + @staticmethod + def azure(subscription, tenant_id, environment, client_id): + """ + Azure Only + + :param str subscription: Azure Subscription + :param str tenant_id: Azure Tenant Id + :param str environment: Azure Environment + :param str client_id: Azure Client Id + """ + data = { + "azureSubscription": subscription, + "azureTenantId": tenant_id, + "azureEnvironment": environment, + "azureFMAppClientId": client_id + } + + return FMCloudAuthentication(data) From 412c2d08e70eec2540d8d100a38c7423fbf8928b Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 1 Sep 2021 17:40:12 +0200 Subject: [PATCH 22/50] FM: Review comments --- dataikuapi/fmclient.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 93e8f0dc..92cf7a01 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -14,7 +14,7 @@ class FMClient(object): """Entry point for the FM API client""" - def __init__(self, host, api_key_id, api_key_secret, tenant_id, extra_headers = None): + def __init__(self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None): """ Instantiate a new FM API client on the given host with the given API key. @@ -43,7 +43,7 @@ def __init__(self, host, api_key_id, api_key_secret, tenant_id, extra_headers = def get_cloud_credentials(self): """ - Get Cloud Credential + Get Cloud Credentials :return: Cloud credentials :rtype: :class:`dataikuapi.fm.tenant.FMCloudCredentials` @@ -56,7 +56,7 @@ def get_cloud_credentials(self): # VirtualNetwork ######################################################## - def get_virtual_networks(self): + def list_virtual_networks(self): """ List all Virtual Networks @@ -129,7 +129,7 @@ def create_virtual_network(self, # Instance settings template ######################################################## - def get_instance_templates(self): + def list_instance_templates(self): """ List all Instance Settings Templates @@ -213,7 +213,7 @@ def create_instance_template(self, label, # Instance ######################################################## - def get_instances(self): + def list_instances(self): """ List all DSS Instances From 0ba05b7b4109436acfa55af3e04cfc000a1e23e8 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 3 Sep 2021 16:44:57 +0200 Subject: [PATCH 23/50] Add run_ansible_task & install_system_packages --- dataikuapi/fm/instancesettingstemplates.py | 50 +++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 3d2439cb..9df21882 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -25,11 +25,24 @@ def delete(self): future = self.client._perform_tenant_json("DELETE", "/instance-settings-templates/%s" % self.id) return FMFuture.from_resp(self.client, future) + def add_setup_action(self, setup_action): + """ + Add a setup_action + + :param object setup_action: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` + """ + self.ist_data['setupActions'].append(setup_action) + self.save() + + class FMSetupAction(dict): def __init__(self, setupActionType, params=None): """ - param: object setupActionType: the type (`:class: FMSetupActionType`) of the SetupAction - param: str params: the parameters of the SetupAction in a json-encoded string + A class representing a SetupAction + + Do not create this directly, use: + - :meth:`dataikuapi.fm.instancesettingstemplates.FMSetupAction.add_authorized_key` + """ data = { "type": setupActionType.value, @@ -39,9 +52,31 @@ def __init__(self, setupActionType, params=None): super(FMSetupAction, self).__init__(data) -class FMSetupActionAddAuthorizedKey(FMSetupAction): - def __init__(self, ssh_key): - super(FMSetupActionAddAuthorizedKey, self).__init__(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key }) + @staticmethod + def add_authorized_key(ssh_key): + """ + Return a ADD_AUTHORIZED_KEY SetupAction + """ + return FMSetupAction(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key }) + + @staticmethod + def run_ansible_task(stage, yaml_string): + """ + Return a RUN_ANSIBLE_TASK SetupAction + + :params object stage: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` + :params str yaml_string: a yaml encoded string defining the ansibles tasks to run + """ + return FMSetupAction(FMSetupActionType.RUN_ANSIBLE_TASKS, {"stage": stage.value, "ansibleTasks": yaml_string }) + + @staticmethod + def install_system_packages(packages): + """ + Return an INSTALL_SYSTEM_PACKAGES SetupAction + + :params list packages: List of packages to install + """ + return FMSetupAction(FMSetupActionType.INSTALL_SYSTEM_PACKAGES, {"packages": packages }) class FMSetupActionType(Enum): RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" @@ -56,3 +91,8 @@ class FMSetupActionType(Enum): ADD_AUTHORIZED_KEY="ADD_AUTHORIZED_KEY" INSTALL_JDBC_DRIVER="INSTALL_JDBC_DRIVER" SETUP_ADVANCED_SECURITY="SETUP_ADVANCED_SECURITY" + +class FMSetupActionStage(Enum): + after_dss_startup="after_dss_startup" + after_install="after_install" + before_install="before_install" From 791e53e5813cad44f0124c7b880916ea403ba86c Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 3 Sep 2021 16:59:37 +0200 Subject: [PATCH 24/50] FM: Setup Advanced Security SetupAction --- dataikuapi/fm/instancesettingstemplates.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 9df21882..95b9da21 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -64,8 +64,8 @@ def run_ansible_task(stage, yaml_string): """ Return a RUN_ANSIBLE_TASK SetupAction - :params object stage: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` - :params str yaml_string: a yaml encoded string defining the ansibles tasks to run + :param object stage: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` + :param str yaml_string: a yaml encoded string defining the ansibles tasks to run """ return FMSetupAction(FMSetupActionType.RUN_ANSIBLE_TASKS, {"stage": stage.value, "ansibleTasks": yaml_string }) @@ -74,10 +74,20 @@ def install_system_packages(packages): """ Return an INSTALL_SYSTEM_PACKAGES SetupAction - :params list packages: List of packages to install + :param list packages: List of packages to install """ return FMSetupAction(FMSetupActionType.INSTALL_SYSTEM_PACKAGES, {"packages": packages }) + @staticmethod + def setup_advanced_security(basic_headers = True, hsts = False): + """ + Return an SETUP_ADVANCED_SECURITY SetupAction + + :param boolean basic_headers: Optional, Prevent browsers to render Web content served by DSS to be embedded into a frame, iframe, embed or object tag. Defaults to True + :param boolean hsts: Optional, Enforce HTTP Strict Transport Security. Defaults to False + """ + return FMSetupAction(FMSetupActionType.SETUP_ADVANCED_SECURITY, {"basic_headers": basic_headers, "hsts": hsts}) + class FMSetupActionType(Enum): RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" INSTALL_SYSTEM_PACKAGES="INSTALL_SYSTEM_PACKAGES" From 381030f3de8d4f885478c6f1d6acf37ddab743b8 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 3 Sep 2021 17:25:52 +0200 Subject: [PATCH 25/50] FM: Install JDBC Driver SetupAction --- dataikuapi/fm/instancesettingstemplates.py | 32 +++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 95b9da21..bb6bde6a 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -55,14 +55,14 @@ def __init__(self, setupActionType, params=None): @staticmethod def add_authorized_key(ssh_key): """ - Return a ADD_AUTHORIZED_KEY SetupAction + Return a ADD_AUTHORIZED_KEY FMSetupAction """ return FMSetupAction(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key }) @staticmethod def run_ansible_task(stage, yaml_string): """ - Return a RUN_ANSIBLE_TASK SetupAction + Return a RUN_ANSIBLE_TASK FMSetupAction :param object stage: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` :param str yaml_string: a yaml encoded string defining the ansibles tasks to run @@ -72,7 +72,7 @@ def run_ansible_task(stage, yaml_string): @staticmethod def install_system_packages(packages): """ - Return an INSTALL_SYSTEM_PACKAGES SetupAction + Return an INSTALL_SYSTEM_PACKAGES FMSetupAction :param list packages: List of packages to install """ @@ -81,13 +81,28 @@ def install_system_packages(packages): @staticmethod def setup_advanced_security(basic_headers = True, hsts = False): """ - Return an SETUP_ADVANCED_SECURITY SetupAction + Return an SETUP_ADVANCED_SECURITY FMSetupAction :param boolean basic_headers: Optional, Prevent browsers to render Web content served by DSS to be embedded into a frame, iframe, embed or object tag. Defaults to True :param boolean hsts: Optional, Enforce HTTP Strict Transport Security. Defaults to False """ return FMSetupAction(FMSetupActionType.SETUP_ADVANCED_SECURITY, {"basic_headers": basic_headers, "hsts": hsts}) + @staticmethod + def install_jdbc_driver(database_type, url, paths_in_archive=None, http_headers=None, http_username=None, http_password=None, datadir_subdirectory=None): + """ + Return a INSTALL_JDBC_DRIVER FMSetupAction + + :param object database_type: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionAddJDBCDriverDatabaseType` + :param str url: The full address to the driver. Supports http(s)://, s3://, abs:// or file:// endpoints + :param list paths_in_archive: Optional, must be used when the driver is shipped as a tarball or a ZIP file. Add here all the paths to find the JAR files in the driver archive. Paths are relative to the top of the archive. Wildcards are supported. + :param dict http_headers: Optional, If you download the driver from a HTTP(S) endpoint, add here the headers you want to add to the query. This setting is ignored for any other type of download. + :param str http_username: Optional, If the HTTP(S) endpoint expect a Basic Authentication, add here the username. To explicitely specify which Assigned Identity use if the machine have several, set the client_id here. To authenticate with a SAS Token on Azure Blob Storage (not recommended), use "token" as the value here. + :param str http_password: Optional, If the HTTP(S) endpoint expect a Basic Authentication, add here the password. To authenticate with a SAS Token on Azure Blob Storage (not recommended), store the token in this field. + :param str datadir_subdirectory: Optional, Some drivers are shipped with a high number of JAR files along with them. In that case, you might want to install them under an additional level in the DSS data directory. Set the name of this subdirectory here. Not required for most drivers. + """ + return FMSetupAction(FMSetupActionType.INSTALL_JDBC_DRIVER, {"url": url, "dbType": database_type.value, "pathsInArchive": paths_in_archive, "headers": http_headers, "username": http_username, "password": http_password, "subpathInDatadir": datadir_subdirectory}) + class FMSetupActionType(Enum): RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" INSTALL_SYSTEM_PACKAGES="INSTALL_SYSTEM_PACKAGES" @@ -106,3 +121,12 @@ class FMSetupActionStage(Enum): after_dss_startup="after_dss_startup" after_install="after_install" before_install="before_install" + +class FMSetupActionAddJDBCDriverDatabaseType(Enum): + mysql="mysql" + mssql="mssql" + oracle="oracle" + mariadb="mariadb" + snowflake="snowflake" + athena="athena" + bigquery="bigquery" From 3e93db831c185c30755f7cbaaa60022b999baf22 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 3 Sep 2021 17:40:45 +0200 Subject: [PATCH 26/50] FM: setup_k8s_and_spark SetupAction --- dataikuapi/fm/instancesettingstemplates.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index bb6bde6a..5eaa4a49 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -103,6 +103,13 @@ def install_jdbc_driver(database_type, url, paths_in_archive=None, http_headers= """ return FMSetupAction(FMSetupActionType.INSTALL_JDBC_DRIVER, {"url": url, "dbType": database_type.value, "pathsInArchive": paths_in_archive, "headers": http_headers, "username": http_username, "password": http_password, "subpathInDatadir": datadir_subdirectory}) + @staticmethod + def setup_k8s_and_spark(): + """ + Return a SETUP_K8S_AND_SPARK FMSetupAction + """ + return FMSetupAction(FMSetupActionType.SETUP_K8S_AND_SPARK) + class FMSetupActionType(Enum): RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" INSTALL_SYSTEM_PACKAGES="INSTALL_SYSTEM_PACKAGES" From 66a7a9ece66047ba63732ae1b2b00e1d1db2e5c4 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 7 Sep 2021 17:23:13 +0200 Subject: [PATCH 27/50] FM: FMInstanceCreator --- dataikuapi/fm/instances.py | 98 ++++++++++++++++++++++++++++++++++++++ dataikuapi/fmclient.py | 11 ++++- 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index b6f95cb2..3a4e7f7b 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,6 +1,104 @@ from enum import Enum from .future import FMFuture +class FMInstanceCreator(object): + def __init__(self, client, label, instance_settings_template_id, virtual_network_id, image_id): + """ + Helper to create a DSS Instance + + :param str instance_settings_template: The instance settings template id this instance should be based on + :param str virtual_network: The virtual network where the instance should be spawned + :param str label: The label of the instance + :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) + """ + self.client = client + self.data = {} + self.data["label"] = label + self.data["instanceSettingsTemplateId"] = instance_settings_template_id + self.data["virtualNetworkId"] = virtual_network_id + self.data["imageId"] = image_id + + # Set the default value for dssNodeType + self.data["dssNodeType"] = "design" + + def create(self): + """ + Create the DSS instance + + :return: Instance + :rtype: :class:`dataikuapi.fm.instances.FMInstance` + """ + instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) + return FMInstance(self.client, instance) + + def set_dss_node_type(self, dss_node_type): + """ + Set the DSS Node type of the instance to create + + :param str dss_node_type: Optional , the type of the dss node to create. Supports "design", "automation ordeployer". Defaults to "design" + """ + if dss_node_type not in ["design", "automation", "deployer"]: + raise ValueError("Only \"design\", \"automation\" or \"deployer\" dss_node_type are supported") + self.data["dssNodeType"] = dss_node_type + + def set_cloud_instance_type(self, cloud_instance_type): + """ + Set the machine type for the DSS Instance + """ + self.data["cloudInstanceType"] = cloud_instance_type + + def set_data_volume_options(self, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=None, data_volume_encryption_key=None): + """ + Set the options of the data volume to use with the DSS Instance + + :param str data_volume_type: Optional, Data volume type + :param int data_volume_size: Optional, Data volume initial size + :param int data_volume_size_max: Optional, Data volume maximum size + :param int data_volume_IOPS: Optional, Data volume IOPS + :param object data_volume_encryption: Optional, a :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode` setting the encryption mode of the data volume + :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM + """ + if type(data_volume_encryption) is not FMInstanceEncryptionMode: + raise TypeError("data_volume encryption needs to be of type FMInstanceEncryptionMode") + + self.data["dataVolumeType"] = data_volume_type + self.data["dataVolumeSizeGB"] = data_volume_size + self.data["dataVolumeSizeMaxGB"] = data_volume_size_max + self.data["dataVolumeIOPS"] = data_volume_IOPS + self.data["dataVolumeEncryption"] = data_volume_encryption.value + self.data["dataVolumeEncryptionKey"] = data_volume_encryption_key + + def set_aws_root_volume_options(self, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None): + """ + AWS Only: Set the options of the root volume of the DSS Instance + + :param int aws_root_volume_size: Optional, the root volume size + :param str aws_root_volume_type: Optional, the root volume type + :param int aws_root_volume_IOPS: Optional, the root volume IOPS + """ + if self.client.cloud != "AWS": + raise BaseException("set_aws_root_volume_options is only usable on AWS tenants") + self.data["awsRootVolumeSizeGB"] = aws_root_volume_size + self.data["awsRootVolumeType"] = aws_root_volume_type + self.data["awsRootVolumeIOPS"] = aws_root_volume_IOPS + + def set_cloud_tags(self, cloud_tags): + """ + Set the tags to be applied to the cloud resources created for this DSS instance + + :param dict cloud_tags: a key value dictionary of tags to be applied on the cloud resources + """ + self.data["cloudTags"] = cloud_tags + + def set_fm_tags(self, fm_tags): + """ + A list of tags to add on the DSS Instance in Fleet Manager + + :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager + """ + self.data["fmTags"] = fm_tags + + class FMInstance(object): """ A handle to interact with a DSS instance. diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 92cf7a01..87d1210b 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -2,8 +2,9 @@ from requests import Session from requests import exceptions from requests.auth import HTTPBasicAuth - import os.path as osp + +from enum import Enum from .utils import DataikuException from .fm.tenant import FMCloudCredentials @@ -14,17 +15,23 @@ class FMClient(object): """Entry point for the FM API client""" - def __init__(self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None): + def __init__(self, host, api_key_id, api_key_secret, cloud, tenant_id="main", extra_headers=None): """ Instantiate a new FM API client on the given host with the given API key. API keys can be managed in FM on the project page or in the global settings. The API key will define which operations are allowed for the client. + + :param str host: Full url of the FM + """ self.api_key_id = api_key_id self.api_key_secret = api_key_secret self.host = host + if cloud not in ["AWS", "Azure"]: + raise ValueError("cloud should be either \"AWS\" or \"Azure\"") + self.cloud = cloud self.__tenant_id = tenant_id self._session = Session() From 2f09e1360e4831c1d2cafaaec6160052c532f218 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 8 Sep 2021 14:57:18 +0200 Subject: [PATCH 28/50] FM: Split Instance and InstanceCreator per cloud --- dataikuapi/fm/instances.py | 112 +++++++++++++++++++++++++------------ dataikuapi/fmclient.py | 54 +++--------------- 2 files changed, 84 insertions(+), 82 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 3a4e7f7b..fec296d1 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -6,9 +6,10 @@ def __init__(self, client, label, instance_settings_template_id, virtual_network """ Helper to create a DSS Instance + :param object client: :class:`dataikuapi.fm.fmclient` + :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned - :param str label: The label of the instance :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) """ self.client = client @@ -21,33 +22,29 @@ def __init__(self, client, label, instance_settings_template_id, virtual_network # Set the default value for dssNodeType self.data["dssNodeType"] = "design" - def create(self): - """ - Create the DSS instance - - :return: Instance - :rtype: :class:`dataikuapi.fm.instances.FMInstance` - """ - instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) - return FMInstance(self.client, instance) - - def set_dss_node_type(self, dss_node_type): + def with_dss_node_type(self, dss_node_type): """ Set the DSS Node type of the instance to create :param str dss_node_type: Optional , the type of the dss node to create. Supports "design", "automation ordeployer". Defaults to "design" + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if dss_node_type not in ["design", "automation", "deployer"]: raise ValueError("Only \"design\", \"automation\" or \"deployer\" dss_node_type are supported") self.data["dssNodeType"] = dss_node_type + return self - def set_cloud_instance_type(self, cloud_instance_type): + def with_cloud_instance_type(self, cloud_instance_type): """ Set the machine type for the DSS Instance + + :param str cloud_instance_type + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ self.data["cloudInstanceType"] = cloud_instance_type + return self - def set_data_volume_options(self, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=None, data_volume_encryption_key=None): + def with_data_volume_options(self, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=None, data_volume_encryption_key=None): """ Set the options of the data volume to use with the DSS Instance @@ -57,6 +54,7 @@ def set_data_volume_options(self, data_volume_type=None, data_volume_size=None, :param int data_volume_IOPS: Optional, Data volume IOPS :param object data_volume_encryption: Optional, a :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode` setting the encryption mode of the data volume :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if type(data_volume_encryption) is not FMInstanceEncryptionMode: raise TypeError("data_volume encryption needs to be of type FMInstanceEncryptionMode") @@ -67,42 +65,71 @@ def set_data_volume_options(self, data_volume_type=None, data_volume_size=None, self.data["dataVolumeIOPS"] = data_volume_IOPS self.data["dataVolumeEncryption"] = data_volume_encryption.value self.data["dataVolumeEncryptionKey"] = data_volume_encryption_key + return self + + def with_cloud_tags(self, cloud_tags): + """ + Set the tags to be applied to the cloud resources created for this DSS instance - def set_aws_root_volume_options(self, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None): + :param dict cloud_tags: a key value dictionary of tags to be applied on the cloud resources + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ - AWS Only: Set the options of the root volume of the DSS Instance + self.data["cloudTags"] = cloud_tags + return self + + def with_fm_tags(self, fm_tags): + """ + A list of tags to add on the DSS Instance in Fleet Manager + + :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` + """ + self.data["fmTags"] = fm_tags + return self + + +class FMAWSInstanceCreator(FMInstanceCreator): + def with_aws_root_volume_options(self, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None): + """ + Set the options of the root volume of the DSS Instance :param int aws_root_volume_size: Optional, the root volume size :param str aws_root_volume_type: Optional, the root volume type :param int aws_root_volume_IOPS: Optional, the root volume IOPS + :rtype: :class:`dataikuapi.fm.instances.FMAWSInstanceCreator` """ - if self.client.cloud != "AWS": - raise BaseException("set_aws_root_volume_options is only usable on AWS tenants") self.data["awsRootVolumeSizeGB"] = aws_root_volume_size self.data["awsRootVolumeType"] = aws_root_volume_type self.data["awsRootVolumeIOPS"] = aws_root_volume_IOPS + return self - def set_cloud_tags(self, cloud_tags): + def create(self): """ - Set the tags to be applied to the cloud resources created for this DSS instance + Create the DSS instance - :param dict cloud_tags: a key value dictionary of tags to be applied on the cloud resources + :return: Created DSS Instance + :rtype: :class:`dataikuapi.fm.instances.FMAWSInstance` """ - self.data["cloudTags"] = cloud_tags + instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) + return FMAWSInstance(self.client, instance) - def set_fm_tags(self, fm_tags): + +class FMAzureInstanceCreator(FMInstanceCreator): + def create(self): """ - A list of tags to add on the DSS Instance in Fleet Manager + Create the DSS instance - :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager + :return: Created DSS Instance + :rtype: :class:`dataikuapi.fm.instances.FMAzureInstance` """ - self.data["fmTags"] = fm_tags + instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) + return FMAzureInstance(self.client, instance) class FMInstance(object): """ A handle to interact with a DSS instance. - Do not create this directly, use :meth:`FMClient.get_instance` or :meth: `FMClient.create_instance` + Do not create this directly, use :meth:`FMClient.get_instance` or :meth: `FMClient.new_instance_creator` """ def __init__(self, client, instance_data): self.client = client @@ -176,28 +203,41 @@ def set_automated_snapshots(self, enable, period, keep=0): self.instance_data['automatedSnapshotRetention'] = keep self.save() + def set_custom_certificate(self, pem_data): + """ + Set the custom certificate for this instance + + Only needed when Virtual Network HTTPS Strategy is set to Custom Certificate + + param: str pem_data: The SSL certificate + """ + self.instance_data['sslCertificatePEM'] = pem_data + self.save() + + +class FMAWSInstance(FMInstance): def set_elastic_ip(self, enable, elasticip_allocation_id): """ Set a public elastic ip for this instance :param boolan enable: Enable the elastic ip allocation - :param str elaticip_allocation_id: The AWS ElasticIP allocation ID or the Azure Public IP ID + :param str elaticip_allocation_id: AWS ElasticIP allocation ID """ self.instance_data['awsAssignElasticIP'] = enable self.instance_data['awsElasticIPAllocationId'] = elasticip_allocation_id - self.instance_data['azureAssignElasticIP'] = enable - self.instance_data['azurePublicIPId'] = elasticip_allocation_id self.save() - def set_custom_certificate(self, pem_data): - """ - Set the custom certificate for this instance - Only needed when Virtual Network HTTPS Strategy is set to Custom Certificate +class FMAzureInstance(FMInstance): + def set_elastic_ip(self, enable, public_ip_id): + """ + Set a public elastic ip for this instance - param: str pem_data: The SSL certificate + :param boolan enable: Enable the elastic ip allocation + :param str public_ip_id: Azure Public IP ID """ - self.instance_data['sslCertificatePEM'] = pem_data + self.instance_data['azureAssignElasticIP'] = enable + self.instance_data['azurePublicIPId'] = public_ip_id self.save() diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 87d1210b..123688a0 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -11,6 +11,7 @@ from .fm.virtualnetworks import FMVirtualNetwork from .fm.instances import FMInstance, FMInstanceEncryptionMode from .fm.instancesettingstemplates import FMInstanceSettingsTemplate +from dataikuapi.fm.instances import FMAWSInstanceCreator, FMAzureInstanceCreator class FMClient(object): """Entry point for the FM API client""" @@ -242,58 +243,19 @@ def get_instance(self, instance_id): instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return FMInstance(self, instance) - def create_instance(self, instance_settings_template, virtual_network, label, image_id, - dss_node_type="design", - cloud_instance_type=None, data_volume_type=None, data_volume_size=None, - data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=FMInstanceEncryptionMode.NONE, - data_volume_encryption_key=None, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None, - cloud_tags=None, fm_tags=None): + def new_instance_creator(self, label, instance_settings_template_id, virtual_network_id, image_id): """ - Create a DSS Instance + Instantiate a new instance creator + :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned - :param str label: The label of the instance :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) - - :param str dss_node_type: Optional , the type of the dss node to create. Defaults to "design" - :param str cloud_instance_type: Optional, Machine type - :param str data_volume_type: Optional, Data volume type - :param int data_volume_size: Optional, Data volume initial size - :param int data_volume_size_max: Optional, Data volume maximum size - :param int data_volume_IOPS: Optional, Data volume IOPS - :param object data_volume_encryption: Optional, a :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode` setting the encryption mode of the data volume - :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM - :param int aws_root_volume_size: Optional, the root volume size - :param str aws_root_volume_type: Optional, the root volume type - :param int aws_root_volume_IOPS: Optional, the root volume IOPS - :param dict cloud_tags: Optional, a key value dictionary of tags to be applied on the cloud resources - :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager - - :return: Instance - :rtype: :class:`dataikuapi.fm.instances.FMInstance` """ - data = { - "virtualNetworkId": virtual_network.id, - "instanceSettingsTemplateId": instance_settings_template.id, - "label": label, - "dssNodeType": dss_node_type, - "imageId": image_id, - "cloudInstanceType": cloud_instance_type, - "dataVolumeType": data_volume_type, - "dataVolumeSizeGB": data_volume_size, - "dataVolumeSizeMaxGB": data_volume_size_max, - "dataVolumeIOPS": data_volume_IOPS, - "dataVolumeEncryption": data_volume_encryption.value, - "dataVolumeEncryptionKey": data_volume_encryption_key, - "awsRootVolumeSizeGB": aws_root_volume_size, - "awsRootVolumeType": aws_root_volume_type, - "awsRootVolumeIOPS": aws_root_volume_IOPS, - "cloudTags": cloud_tags, - "fmTags": fm_tags - } - instance = self._perform_tenant_json("POST", "/instances", body=data) - return FMInstance(self, instance) + if self.cloud == "AWS": + return FMAWSInstanceCreator(self, label, instance_settings_template_id, virtual_network_id, image_id) + elif self.cloud == "Azure": + return FMAzureInstanceCreator(self, label, instance_settings_template_id, virtual_network_id, image_id) ######################################################## From 68e400501c07a7d4ac10c8d13bcd54c40b62d845 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 17 Sep 2021 17:51:18 +0200 Subject: [PATCH 29/50] FM InstanceSettingsTemplateCreator --- dataikuapi/fm/instancesettingstemplates.py | 145 +++++++++++++++++++++ dataikuapi/fmclient.py | 67 ++-------- 2 files changed, 156 insertions(+), 56 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 5eaa4a49..437da3c9 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -2,6 +2,151 @@ import json from dataikuapi.fm.future import FMFuture +class FMInstanceSettingsTemplateCreator(object): + def __init__(self, client, label): + """ + Create an Instance Template + + :param str azureSshKey: Optional, Azure Only, the ssh public key to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. + :param str startupManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at startup time + :param str runtimeManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at runtime + + :return: requested instance settings template + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + """ + + self.data = {} + self.data["label"] = label + self.client = client + + def create(self): + template = self.client._perform_tenant_json("POST", "/instance-settings-templates", body=self.data) + return FMInstanceSettingsTemplate(self, template) + + def with_setup_actions(self, setup_actions): + """ + Add setup actions + + :param list setup_actions: List of :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` to be played on an instance + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplateCreator` + """ + self.data["setupActions"] = setup_actions + return self + + def with_license(self, license): + self.data["license"] = license + return self + + +class FMAWSInstanceSettingsTemplateCreator(FMInstanceSettingsTemplateCreator): + def with_aws_keypair(self, aws_keypair_name): + """ + Add an AWS Keypair to the DSS instance. + Needed to get SSH access to the DSS instance, using the centos user. + + :param str aws_keypair_name: Name of an AWS key pair to add to the instance. + """ + self.data["awsKeyPairName"] = aws_keypair_name + return self + + def with_startup_instance_profile(self, startup_instance_profile_arn): + """ + Add a Instance Profile to be assign to the DSS instance on startup + + :param str startup_instance_profile_arn: ARN of the Instance profile assigned to the DSS instance at startup time + """ + self.data["startupInstanceProfileArn"] = startup_instance_profile_arn + return self + + def with_runtime_instance_profile(self, runtime_instance_profile_arn): + """ + Add a Instance Profile to be assign to the DSS instance when running + + :param str runtime_instance_profile_arn: ARN of the Instance profile assigned to the DSS instance during runtime + """ + self.data["runtimeInstanceProfileArn"] = runtime_instance_profile_arn + return self + + def with_restrict_aws_metadata_server_access(self, restrict_aws_metadata_server_access = True): + """ + Restrict AWS metadata server access on the DSS instance. + + :param boolean restrict_aws_metadata_server_access: Optional, If true, restrict the access to the metadata server access. Defaults to true + """ + self.data["restrictAwsMetadataServerAccess"] = restrict_aws_metadata_server_access + return self + + def with_default_aws_api_access_mode(self): + """ + The DSS Instance will use the Runtime Instance Profile to access AWS API. + """ + self.data["dataikuAwsAPIAccessMode"] = "NONE" + return self + + def with_keypair_aws_api_access_mode(self, aws_access_key_id, aws_keypair_storage_mode="NONE", aws_secret_access_key=None, aws_secret_access_key_aws_secret_name=None, aws_secrets_manager_region=None): + """ + DSS Instance will use an Access Key to authenticate against the AWS API. + + :param str aws_access_key_id: AWS Access Key ID. + :param str aws_keypair_storage_mode: Optional, the storage mode of the AWS api key. Accepts "NONE", "INLINE_ENCRYPTED" or "AWS_SECRETS_MANAGER". Defaults to "NONE" + :param str aws_secret_access_key: Optional, AWS Access Key Secret. Only needed if keypair_storage_mode is "INLINE_ENCRYPTED" + :param str aws_secret_access_key_aws_secret_name: Optional, ASM secret name. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" + :param str aws_secrets_manager_region: Optional, Secret Manager region to use. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" + """ + if aws_keypair_storage_mode not in ["NONE", "INLINE_ENCRYPTED", "AWS_SECRETS_MANAGER"]: + raise ValueError("aws_keypair_storage_mode should be either \"NONE\", \"INLINE_ENCRYPTED\" or \"AWS_SECRET_MANAGER\"") + + self.data["dataikuAwsAPIAccessMode"] = "KEYPAIR" + self.data["dataikuAwsKeypairStorageMode"] = aws_keypair_storage_mode + + if aws_keypair_storage_mode == "NONE": + return self + + self.data["dataikuAwsAccessKeyId"] = aws_access_key_id + + if aws_keypair_storage_mode == "INLINE_ENCRYPTED": + if aws_secret_access_key == None: + raise ValueError("When aws_keypair_storage_mode is \"INLINE_ENCRYPTED\", aws_secret_access_key should be provided") + self.data["dataikuAwsSecretAccessKey"] = aws_secret_access_key + elif aws_keypair_storage_mode == "AWS_SECRETS_MANAGER": + if aws_secret_access_key_aws_secret_name == None: + raise ValueError("When aws_keypair_storage_mode is \"AWS_SECRETS_MANAGER\", aws_secret_access_key_aws_secret_name should be provided") + self.data["dataikuAwsSecretAccessKeyAwsSecretName"] = aws_secret_access_key_aws_secret_name + self.data["awsSecretsManagerRegion"] = aws_secrets_manager_region + + return self + + +class FMAzureInstanceSettingsTemplateCreator(FMInstanceSettingsTemplateCreator): + def with_ssh_key(self, ssh_public_key): + """ + Add an SSH public key to the DSS Instance. + Needed to access it through SSH, using the centos user. + + :param str ssh_public_key: The content of the public key to add to the instance. + """ + self.data["azureSshKey"] = ssh_public_key + return self + + def with_startup_managed_identity(self, startup_managed_identity): + """ + Add a managed identity to be assign to the DSS instance on startup + + :param str startup_managed_identity: Managed Identity ID + """ + self.data["startupManagedIdentity"] = startup_managed_identity + return self + + def with_runtime_managed_identity(self, runtime_managed_identity): + """ + Add a managed identity to be assign to the DSS instance when running + + :param str runtime_managed_identity: Managed Identity ID + """ + self.data["runtimeManagedIdentity"] = runtime_managed_identity + return self + + class FMInstanceSettingsTemplate(object): def __init__(self, client, ist_data): self.client = client diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 123688a0..2857c406 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -9,9 +9,8 @@ from .fm.tenant import FMCloudCredentials from .fm.virtualnetworks import FMVirtualNetwork -from .fm.instances import FMInstance, FMInstanceEncryptionMode -from .fm.instancesettingstemplates import FMInstanceSettingsTemplate -from dataikuapi.fm.instances import FMAWSInstanceCreator, FMAzureInstanceCreator +from .fm.instances import FMInstance, FMInstanceEncryptionMode, FMAWSInstanceCreator, FMAzureInstanceCreator +from .fm.instancesettingstemplates import FMInstanceSettingsTemplate, FMAWSInstanceSettingsTemplateCreator, FMAzureInstanceSettingsTemplateCreator class FMClient(object): """Entry point for the FM API client""" @@ -159,62 +158,17 @@ def get_instance_template(self, template_id): template = self._perform_tenant_json("GET", "/instance-settings-templates/%s" % template_id) return FMInstanceSettingsTemplate(self, template) - - def create_instance_template(self, label, - setupActions=None, license=None, - awsKeyPairName=None, startupInstanceProfileArn=None, runtimeInstanceProfileArn=None, - restrictAwsMetadataServerAccess=True, dataikuAwsAPIAccessMode="NONE", dataikuAwsKeypairStorageMode=None, - dataikuAwsAccessKeyId=None, dataikuAwsSecretAccessKey=None, - dataikuAwsSecretAccessKeyAwsSecretName=None, awsSecretsManagerRegion=None, - azureSshKey=None, startupManagedIdentity=None, runtimeManagedIdentity=None): + def new_instance_template_creator(self, label): """ - Create an Instance Template - - :param str label: The label of the Instance Settings Template - - :param list setupActions: Optional, a list of :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` to be played on an instance - :param str license: Optional, overrides the license set in Cloud Setup - - :param str awsKeyPairName: Optional, AWS Only, the name of an AWS key pair to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. - :param str startupInstanceProfileArn: Optional, AWS Only, the ARN of the Instance profile assigned to the DSS instance at startup time - :param str runtimeInstanceProfileArn: Optional, AWS Only, the ARN of the Instance profile assigned to the DSS instance at runtime - :param boolean restrictAwsMetadataServerAccess: Optional, AWS Only, If true, restrict the access to the metadata server access. Defaults to true - :param str dataikuAwsAPIAccessMode: Optional, AWS Only, the access mode DSS is using to connect to the AWS API. If "NONE" DSS will use the Instance Profile, If "KEYPAIR", an AWS access key id and secret will be securely given to the dataiku account. - :param str dataikuAwsKeypairStorageMode: Optional, AWS Only, the storage mode of the AWS api key. Accepts "NONE", "INLINE_ENCRYPTED" or "AWS_SECRETS_MANAGER" - :param str dataikuAwsAccessKeyId: Optional, AWS Only, AWS Access Key ID. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" - :param str dataikuAwsSecretAccessKey: Optional, AWS Only, AWS Access Key Secret. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" and dataikuAwsKeypairStorageMode is "INLINE_ENCRYPTED" - :param str dataikuAwsSecretAccessKeyAwsSecretName: Optional, AWS Only, ASM secret name. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" and dataikuAwsKeypairStorageMode is "AWS_SECRET_MANAGER" - :param str awsSecretsManagerRegion: Optional, AWS Only + Instantiate a new instance template creator - :param str azureSshKey: Optional, Azure Only, the ssh public key to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. - :param str startupManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at startup time - :param str runtimeManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at runtime - - :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + :param str label: The label of the instance + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplateCreator` """ - - data = { - "label": label, - "setupActions": setupActions, - "license": license, - "awsKeyPairName": awsKeyPairName, - "startupInstanceProfileArn": startupInstanceProfileArn, - "runtimeInstanceProfileArn": runtimeInstanceProfileArn, - "restrictAwsMetadataServerAccess": restrictAwsMetadataServerAccess, - "dataikuAwsAPIAccessMode": "dataikuAwsAPIAccessMode", - "dataikuAwsKeypairStorageMode": dataikuAwsKeypairStorageMode, - "dataikuAwsAccessKeyId": dataikuAwsAccessKeyId, - "dataikuAwsSecretAccessKey": dataikuAwsSecretAccessKey, - "dataikuAwsSecretAccessKeyAwsSecretName": dataikuAwsSecretAccessKeyAwsSecretName, - "awsSecretsManagerRegion": awsSecretsManagerRegion, - "azureSshKey": azureSshKey, - "startupManagedIdentity": startupManagedIdentity, - "runtimeManagedIdentity": runtimeManagedIdentity - } - - template = self._perform_tenant_json("POST", "/instance-settings-templates", body=data) - return FMInstanceSettingsTemplate(self, template) + if self.cloud == "AWS": + return FMAWSInstanceSettingsTemplateCreator(self, label) + elif self.cloud == "Azure": + return FMAzureInstanceSettingsTemplateCreator(self, label) ######################################################## @@ -251,6 +205,7 @@ def new_instance_creator(self, label, instance_settings_template_id, virtual_net :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if self.cloud == "AWS": return FMAWSInstanceCreator(self, label, instance_settings_template_id, virtual_network_id, image_id) From e257c5b3fe3ac88e7e0eed2ef3eef1906b72bb9c Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 21 Sep 2021 18:17:22 +0200 Subject: [PATCH 30/50] FM: InstanceSettingsTemplate format + doc --- dataikuapi/fm/instancesettingstemplates.py | 168 ++++++++++++++------- 1 file changed, 116 insertions(+), 52 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 437da3c9..4fe035c6 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -2,17 +2,13 @@ import json from dataikuapi.fm.future import FMFuture + class FMInstanceSettingsTemplateCreator(object): def __init__(self, client, label): """ - Create an Instance Template - - :param str azureSshKey: Optional, Azure Only, the ssh public key to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. - :param str startupManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at startup time - :param str runtimeManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at runtime + A builder class to create an Instance Settings Template - :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + :param str label: The label of the Virtual Network """ self.data = {} @@ -20,7 +16,16 @@ def __init__(self, client, label): self.client = client def create(self): - template = self.client._perform_tenant_json("POST", "/instance-settings-templates", body=self.data) + """ + Create the Instance Settings Template + + :return: Created InstanceSettingsTemplate + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + """ + + template = self.client._perform_tenant_json( + "POST", "/instance-settings-templates", body=self.data + ) return FMInstanceSettingsTemplate(self, template) def with_setup_actions(self, setup_actions): @@ -67,13 +72,17 @@ def with_runtime_instance_profile(self, runtime_instance_profile_arn): self.data["runtimeInstanceProfileArn"] = runtime_instance_profile_arn return self - def with_restrict_aws_metadata_server_access(self, restrict_aws_metadata_server_access = True): + def with_restrict_aws_metadata_server_access( + self, restrict_aws_metadata_server_access=True + ): """ Restrict AWS metadata server access on the DSS instance. :param boolean restrict_aws_metadata_server_access: Optional, If true, restrict the access to the metadata server access. Defaults to true """ - self.data["restrictAwsMetadataServerAccess"] = restrict_aws_metadata_server_access + self.data[ + "restrictAwsMetadataServerAccess" + ] = restrict_aws_metadata_server_access return self def with_default_aws_api_access_mode(self): @@ -83,7 +92,14 @@ def with_default_aws_api_access_mode(self): self.data["dataikuAwsAPIAccessMode"] = "NONE" return self - def with_keypair_aws_api_access_mode(self, aws_access_key_id, aws_keypair_storage_mode="NONE", aws_secret_access_key=None, aws_secret_access_key_aws_secret_name=None, aws_secrets_manager_region=None): + def with_keypair_aws_api_access_mode( + self, + aws_access_key_id, + aws_keypair_storage_mode="NONE", + aws_secret_access_key=None, + aws_secret_access_key_aws_secret_name=None, + aws_secrets_manager_region=None, + ): """ DSS Instance will use an Access Key to authenticate against the AWS API. @@ -93,8 +109,14 @@ def with_keypair_aws_api_access_mode(self, aws_access_key_id, aws_keypair_storag :param str aws_secret_access_key_aws_secret_name: Optional, ASM secret name. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" :param str aws_secrets_manager_region: Optional, Secret Manager region to use. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" """ - if aws_keypair_storage_mode not in ["NONE", "INLINE_ENCRYPTED", "AWS_SECRETS_MANAGER"]: - raise ValueError("aws_keypair_storage_mode should be either \"NONE\", \"INLINE_ENCRYPTED\" or \"AWS_SECRET_MANAGER\"") + if aws_keypair_storage_mode not in [ + "NONE", + "INLINE_ENCRYPTED", + "AWS_SECRETS_MANAGER", + ]: + raise ValueError( + 'aws_keypair_storage_mode should be either "NONE", "INLINE_ENCRYPTED" or "AWS_SECRET_MANAGER"' + ) self.data["dataikuAwsAPIAccessMode"] = "KEYPAIR" self.data["dataikuAwsKeypairStorageMode"] = aws_keypair_storage_mode @@ -106,12 +128,18 @@ def with_keypair_aws_api_access_mode(self, aws_access_key_id, aws_keypair_storag if aws_keypair_storage_mode == "INLINE_ENCRYPTED": if aws_secret_access_key == None: - raise ValueError("When aws_keypair_storage_mode is \"INLINE_ENCRYPTED\", aws_secret_access_key should be provided") + raise ValueError( + 'When aws_keypair_storage_mode is "INLINE_ENCRYPTED", aws_secret_access_key should be provided' + ) self.data["dataikuAwsSecretAccessKey"] = aws_secret_access_key elif aws_keypair_storage_mode == "AWS_SECRETS_MANAGER": if aws_secret_access_key_aws_secret_name == None: - raise ValueError("When aws_keypair_storage_mode is \"AWS_SECRETS_MANAGER\", aws_secret_access_key_aws_secret_name should be provided") - self.data["dataikuAwsSecretAccessKeyAwsSecretName"] = aws_secret_access_key_aws_secret_name + raise ValueError( + 'When aws_keypair_storage_mode is "AWS_SECRETS_MANAGER", aws_secret_access_key_aws_secret_name should be provided' + ) + self.data[ + "dataikuAwsSecretAccessKeyAwsSecretName" + ] = aws_secret_access_key_aws_secret_name self.data["awsSecretsManagerRegion"] = aws_secrets_manager_region return self @@ -149,16 +177,20 @@ def with_runtime_managed_identity(self, runtime_managed_identity): class FMInstanceSettingsTemplate(object): def __init__(self, client, ist_data): - self.client = client - self.id = ist_data['id'] + self.client = client + self.id = ist_data["id"] self.ist_data = ist_data def save(self): """ Update the Instance Settings Template. """ - self.client._perform_tenant_empty("PUT", "/instance-settings-templates/%s" % self.id, body=self.ist_data) - self.ist_data = self.client._perform_tenant_json("GET", "/instance-settings-templates/%s" % self.id) + self.client._perform_tenant_empty( + "PUT", "/instance-settings-templates/%s" % self.id, body=self.ist_data + ) + self.ist_data = self.client._perform_tenant_json( + "GET", "/instance-settings-templates/%s" % self.id + ) def delete(self): """ @@ -167,7 +199,9 @@ def delete(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("DELETE", "/instance-settings-templates/%s" % self.id) + future = self.client._perform_tenant_json( + "DELETE", "/instance-settings-templates/%s" % self.id + ) return FMFuture.from_resp(self.client, future) def add_setup_action(self, setup_action): @@ -176,7 +210,7 @@ def add_setup_action(self, setup_action): :param object setup_action: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ - self.ist_data['setupActions'].append(setup_action) + self.ist_data["setupActions"].append(setup_action) self.save() @@ -193,7 +227,7 @@ def __init__(self, setupActionType, params=None): "type": setupActionType.value, } if params: - data['params'] = params + data["params"] = params super(FMSetupAction, self).__init__(data) @@ -202,7 +236,7 @@ def add_authorized_key(ssh_key): """ Return a ADD_AUTHORIZED_KEY FMSetupAction """ - return FMSetupAction(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key }) + return FMSetupAction(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key}) @staticmethod def run_ansible_task(stage, yaml_string): @@ -212,7 +246,10 @@ def run_ansible_task(stage, yaml_string): :param object stage: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` :param str yaml_string: a yaml encoded string defining the ansibles tasks to run """ - return FMSetupAction(FMSetupActionType.RUN_ANSIBLE_TASKS, {"stage": stage.value, "ansibleTasks": yaml_string }) + return FMSetupAction( + FMSetupActionType.RUN_ANSIBLE_TASKS, + {"stage": stage.value, "ansibleTasks": yaml_string}, + ) @staticmethod def install_system_packages(packages): @@ -221,20 +258,33 @@ def install_system_packages(packages): :param list packages: List of packages to install """ - return FMSetupAction(FMSetupActionType.INSTALL_SYSTEM_PACKAGES, {"packages": packages }) + return FMSetupAction( + FMSetupActionType.INSTALL_SYSTEM_PACKAGES, {"packages": packages} + ) @staticmethod - def setup_advanced_security(basic_headers = True, hsts = False): + def setup_advanced_security(basic_headers=True, hsts=False): """ Return an SETUP_ADVANCED_SECURITY FMSetupAction :param boolean basic_headers: Optional, Prevent browsers to render Web content served by DSS to be embedded into a frame, iframe, embed or object tag. Defaults to True :param boolean hsts: Optional, Enforce HTTP Strict Transport Security. Defaults to False """ - return FMSetupAction(FMSetupActionType.SETUP_ADVANCED_SECURITY, {"basic_headers": basic_headers, "hsts": hsts}) + return FMSetupAction( + FMSetupActionType.SETUP_ADVANCED_SECURITY, + {"basic_headers": basic_headers, "hsts": hsts}, + ) @staticmethod - def install_jdbc_driver(database_type, url, paths_in_archive=None, http_headers=None, http_username=None, http_password=None, datadir_subdirectory=None): + def install_jdbc_driver( + database_type, + url, + paths_in_archive=None, + http_headers=None, + http_username=None, + http_password=None, + datadir_subdirectory=None, + ): """ Return a INSTALL_JDBC_DRIVER FMSetupAction @@ -246,7 +296,18 @@ def install_jdbc_driver(database_type, url, paths_in_archive=None, http_headers= :param str http_password: Optional, If the HTTP(S) endpoint expect a Basic Authentication, add here the password. To authenticate with a SAS Token on Azure Blob Storage (not recommended), store the token in this field. :param str datadir_subdirectory: Optional, Some drivers are shipped with a high number of JAR files along with them. In that case, you might want to install them under an additional level in the DSS data directory. Set the name of this subdirectory here. Not required for most drivers. """ - return FMSetupAction(FMSetupActionType.INSTALL_JDBC_DRIVER, {"url": url, "dbType": database_type.value, "pathsInArchive": paths_in_archive, "headers": http_headers, "username": http_username, "password": http_password, "subpathInDatadir": datadir_subdirectory}) + return FMSetupAction( + FMSetupActionType.INSTALL_JDBC_DRIVER, + { + "url": url, + "dbType": database_type.value, + "pathsInArchive": paths_in_archive, + "headers": http_headers, + "username": http_username, + "password": http_password, + "subpathInDatadir": datadir_subdirectory, + }, + ) @staticmethod def setup_k8s_and_spark(): @@ -255,30 +316,33 @@ def setup_k8s_and_spark(): """ return FMSetupAction(FMSetupActionType.SETUP_K8S_AND_SPARK) + class FMSetupActionType(Enum): - RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" - INSTALL_SYSTEM_PACKAGES="INSTALL_SYSTEM_PACKAGES" - INSTALL_DSS_PLUGINS_FROM_STORE="INSTALL_DSS_PLUGINS_FROM_STORE" - SETUP_K8S_AND_SPARK="SETUP_K8S_AND_SPARK" - SETUP_RUNTIME_DATABASE="SETUP_RUNTIME_DATABASE" - SETUP_MUS_AUTOCREATE="SETUP_MUS_AUTOCREATE" - SETUP_UI_CUSTOMIZATION="SETUP_UI_CUSTOMIZATION" - SETUP_MEMORY_SETTINGS="SETUP_MEMORY_SETTINGS" - SETUP_GRAPHICS_EXPORT="SETUP_GRAPHICS_EXPORT" - ADD_AUTHORIZED_KEY="ADD_AUTHORIZED_KEY" - INSTALL_JDBC_DRIVER="INSTALL_JDBC_DRIVER" - SETUP_ADVANCED_SECURITY="SETUP_ADVANCED_SECURITY" + RUN_ANSIBLE_TASKS = "RUN_ANSIBLE_TASKS" + INSTALL_SYSTEM_PACKAGES = "INSTALL_SYSTEM_PACKAGES" + INSTALL_DSS_PLUGINS_FROM_STORE = "INSTALL_DSS_PLUGINS_FROM_STORE" + SETUP_K8S_AND_SPARK = "SETUP_K8S_AND_SPARK" + SETUP_RUNTIME_DATABASE = "SETUP_RUNTIME_DATABASE" + SETUP_MUS_AUTOCREATE = "SETUP_MUS_AUTOCREATE" + SETUP_UI_CUSTOMIZATION = "SETUP_UI_CUSTOMIZATION" + SETUP_MEMORY_SETTINGS = "SETUP_MEMORY_SETTINGS" + SETUP_GRAPHICS_EXPORT = "SETUP_GRAPHICS_EXPORT" + ADD_AUTHORIZED_KEY = "ADD_AUTHORIZED_KEY" + INSTALL_JDBC_DRIVER = "INSTALL_JDBC_DRIVER" + SETUP_ADVANCED_SECURITY = "SETUP_ADVANCED_SECURITY" + class FMSetupActionStage(Enum): - after_dss_startup="after_dss_startup" - after_install="after_install" - before_install="before_install" + after_dss_startup = "after_dss_startup" + after_install = "after_install" + before_install = "before_install" + class FMSetupActionAddJDBCDriverDatabaseType(Enum): - mysql="mysql" - mssql="mssql" - oracle="oracle" - mariadb="mariadb" - snowflake="snowflake" - athena="athena" - bigquery="bigquery" + mysql = "mysql" + mssql = "mssql" + oracle = "oracle" + mariadb = "mariadb" + snowflake = "snowflake" + athena = "athena" + bigquery = "bigquery" From 16c23251be271527fe547db75c5a4e0b7eb6d305 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 21 Sep 2021 18:19:11 +0200 Subject: [PATCH 31/50] fm: FMVirtualNetworkCreator --- dataikuapi/fm/virtualnetworks.py | 182 +++++++++++++++++++++++++++---- dataikuapi/fmclient.py | 53 ++------- 2 files changed, 168 insertions(+), 67 deletions(-) diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index b1748d0f..ca5a3ced 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -1,17 +1,123 @@ from dataikuapi.fm.future import FMFuture + +class FMVirtualNetworkCreator(object): + def __init__(self, client, label): + """ + A builder class to create a Virtual Network + + :param str label: The label of the Virtual Network + """ + self.client = client + self.data = {} + self.data["label"] = label + self.data["mode"] = "EXISTING_MONOTENANT" + + def with_internet_access_mode(self, internet_access_mode): + """ + :param str internet_access_mode: The internet access mode of the instances created in this virtual network. Accepts "YES", "NO", "EGRESS_ONLY". Defaults to "YES" + """ + if internet_access_mode not in ["YES", "NO", "EGRESS_ONLY"]: + raise ValueError( + 'internet_access_mode should be either "YES", "NO", or "EGRESS_ONLY"' + ) + + self.data["internetAccessMode"] = internet_access_mode + return self + + +class FMAWSVirtualNetworkCreator(FMVirtualNetworkCreator): + def with_vpc(self, aws_vpc_id, aws_subnet_id): + """ + Setup the VPC and Subnet to with the VirtualNetwork + + :param str aws_vpc_id: ID of the VPC to use + :param str aws_subnet_id: ID of the subnet to use + """ + self.data["awsVpcId"] = aws_vpc_id + self.data["awsSubnetId"] = aws_subnet_id + return self + + def with_auto_create_security_groups(self): + """ + Automatically create the AWS Security Groups when creating this VirtualNetwork + """ + self.data["awsAutoCreateSecurityGroups"] = True + return self + + def with_aws_security_groups(self, aws_security_groups): + """ + Use pre-created AWS Security Groups + + :param list aws_security_groups: A list of up to 5 security group ids to assign to the instances created in this virtual network. + """ + self.data["awsAutoCreateSecurityGroups"] = False + self.data["awsSecurityGroups"] = aws_security_groups + return self + + def create(self): + """ + Create the VirtualNetwork + + :return: Created VirtualNetwork + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAWSVirtualNetwork` + """ + vn = self.client._perform_tenant_json( + "POST", "/virtual-networks", body=self.data + ) + return FMAWSVirtualNetwork(self, vn) + + +class FMAzureVirtualNetworkCreator(FMVirtualNetworkCreator): + def with_virtual_network(self, azure_vn_id, azure_subnet_id): + """ + Setup the Azure Virtual Network and Subnet to with the VirtualNetwork + + :param str azure_vn_id: Resource ID of the Azure Virtual Network to use + :param str azure_subnet_id: Resource ID of the subnet to use + """ + self.data["azureVnId"] = azure_vn_id + self.data["azureSubnetId"] = azure_subnet_id + return self + + def with_auto_update_security_groups(self, auto_update_security_groups=True): + """ + Auto update the security groups of the Azure Virtual Network + + :param boolean auto_update_security_groups: Optional, Auto update the subnet security group. Defaults to True + """ + self.data["azureAutoUpdateSecurityGroups"] = auto_update_security_groups + return self + + def create(self): + """ + Create the VirtualNetwork + + :return: Created VirtualNetwork + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAzureVirtualNetwork` + """ + vn = self.client._perform_tenant_json( + "POST", "/virtual-networks", body=self.data + ) + return FMAzureVirtualNetwork(self, vn) + + class FMVirtualNetwork(object): def __init__(self, client, vn_data): - self.client = client + self.client = client self.vn_data = vn_data - self.id = self.vn_data['id'] + self.id = self.vn_data["id"] def save(self): """ Update the Virtual Network. """ - self.client._perform_tenant_empty("PUT", "/virtual-networks/%s" % self.id, body=self.vn_data) - self.vn_data = self.client._perform_tenant_json("GET", "/virtual-networks/%s" % self.id) + self.client._perform_tenant_empty( + "PUT", "/virtual-networks/%s" % self.id, body=self.vn_data + ) + self.vn_data = self.client._perform_tenant_json( + "GET", "/virtual-networks/%s" % self.id + ) def delete(self): """ @@ -20,10 +126,14 @@ def delete(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("DELETE", "/virtual-networks/%s" % self.id) + future = self.client._perform_tenant_json( + "DELETE", "/virtual-networks/%s" % self.id + ) return FMFuture.from_resp(self.client, future) - def set_fleet_management(self, enable, event_server=None, deployer_management="NO_MANAGED_DEPLOYER"): + def set_fleet_management( + self, enable, event_server=None, deployer_management="NO_MANAGED_DEPLOYER" + ): """ When enabled, all instances in this virtual network know each other and can centrally manage deployer and logs centralization @@ -36,40 +146,64 @@ def set_fleet_management(self, enable, event_server=None, deployer_management="N - "EACH_DESIGN_NODE": Deployer from design. Recommanded if you have a single design node and want a simpler setup. """ - self.vn_data['managedNodesDirectory'] = enable - self.vn_data['eventServerNodeLabel'] = event_server - self.vn_data['nodesDirectoryDeployerMode'] = deployer_management + self.vn_data["managedNodesDirectory"] = enable + self.vn_data["eventServerNodeLabel"] = event_server + self.vn_data["nodesDirectoryDeployerMode"] = deployer_management + self.save() + + def set_https_strategy(self, https_strategy): + """ + Set the HTTPS strategy for this virtual network + + :param object: a :class:`dataikuapi.fm.virtualnetworks.FMHTTPSStrategy` + """ + self.vn_data.update(https_strategy) self.save() - def set_dns_strategy(self, assign_domain_name, aws_private_ip_zone53_id=None, aws_public_ip_zone53_id=None, azure_dns_zone_id=None): + +class FMAWSVirtualNetwork(FMVirtualNetwork): + def set_dns_strategy( + self, + assign_domain_name, + aws_private_ip_zone53_id=None, + aws_public_ip_zone53_id=None, + ): """ Set the DNS strategy for this virtual network :param boolean assign_domain_name: If false, don't assign domain names, use ip_only :param str aws_private_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for private ip :param str aws_public_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for public ip - :param str azure_dns_zone_id: Optional, Azure Only, the ID of the Azure DNS zone to use """ if assign_domain_name: - self.vn_data['dnsStrategy'] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" - self.vn_data['awsRoute53PrivateIPZoneId'] = aws_private_ip_zone53_id - self.vn_data['awsRoute53PublicIPZoneId'] = aws_public_ip_zone53_id - self.vn_data['azureDnsZoneId'] = azure_dns_zone_id - else : - self.vn_data['dnsStrategy'] = "NONE" + self.vn_data["dnsStrategy"] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" + self.vn_data["awsRoute53PrivateIPZoneId"] = aws_private_ip_zone53_id + self.vn_data["awsRoute53PublicIPZoneId"] = aws_public_ip_zone53_id + else: + self.vn_data["dnsStrategy"] = "NONE" self.save() - def set_https_strategy(self, https_strategy): + +class FMAzureVirtualNetwork(FMVirtualNetwork): + def set_dns_strategy(self, assign_domain_name, azure_dns_zone_id=None): """ - Set the HTTPS strategy for this virtual network + Set the DNS strategy for this virtual network - :param object: a :class:`dataikuapi.fm.virtualnetworks.FMHTTPSStrategy` + :param boolean assign_domain_name: If false, don't assign domain names, use ip_only + :param str azure_dns_zone_id: Optional, Azure Only, the ID of the Azure DNS zone to use """ - self.vn_data.update(https_strategy) + + if assign_domain_name: + self.vn_data["dnsStrategy"] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" + self.vn_data["azureDnsZoneId"] = azure_dns_zone_id + else: + self.vn_data["dnsStrategy"] = "NONE" + self.save() + class FMHTTPSStrategy(dict): def __init__(self, data, https_strategy, http_redirect=False): """ @@ -82,11 +216,11 @@ def __init__(self, data, https_strategy, http_redirect=False): - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.lets_encrypt` to use Let's Encrypt """ super(FMHTTPSStrategy, self).__init__(data) - self['httpsStrategy'] = https_strategy + self["httpsStrategy"] = https_strategy if http_redirect: - self['httpStrategy'] = "REDIRECT" + self["httpStrategy"] = "REDIRECT" else: - self['httpStrategy'] = "DISABLE" + self["httpStrategy"] = "DISABLE" @staticmethod def disable(): diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 2857c406..0dbef6fc 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -8,7 +8,7 @@ from .utils import DataikuException from .fm.tenant import FMCloudCredentials -from .fm.virtualnetworks import FMVirtualNetwork +from .fm.virtualnetworks import FMVirtualNetwork, FMAWSVirtualNetworkCreator, FMAzureVirtualNetworkCreator from .fm.instances import FMInstance, FMInstanceEncryptionMode, FMAWSInstanceCreator, FMAzureInstanceCreator from .fm.instancesettingstemplates import FMInstanceSettingsTemplate, FMAWSInstanceSettingsTemplateCreator, FMAzureInstanceSettingsTemplateCreator @@ -85,51 +85,18 @@ def get_virtual_network(self, virtual_network_id): vn = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) return FMVirtualNetwork(self, vn) - def create_virtual_network(self, - label, - awsVpcId=None, - awsSubnetId=None, - awsAutoCreateSecurityGroups=False, - awsSecurityGroups=None, - azureVnId=None, - azureSubnetId=None, - azureAutoUpdateSecurityGroups=None, - internetAccessMode = "YES"): - """ - Create a Virtual Network - - :param str label: The label of the Virtual Network - - :param str awsVpcId: AWS Only, ID of the VPC to use - :param str awsSubnetId: AWS Only, ID of the subnet to use - :param boolean awsAutoCreateSecurityGroups: Optional, AWS Only, If false, do not create security groups automatically. Defaults to false - :param list awsSecurityGroups: Optional, AWS Only, A list of up to 5 security group ids to assign to the instances created in this virtual network. Ignored if awsAutoCreateSecurityGroups is true - :param str azureVnId: Azure Only, ID of the Azure Virtual Network to use - :param str azureSubnetId: Azure Only, ID of the subnet to use - :param boolean azureAutoUpdateSecurityGroups: Azure Only, Auto update the subnet security group - - :param str internetAccessMode: Optional, The internet access mode of the instances created in this virtual network. Accepts "YES", "NO", "EGRESS_ONLY". Defaults to "YES" - - :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + def new_virtual_network_creator(self, label): """ + Instantiate a new virtual network creator - data = { - "label": label, - "awsVpcId": awsVpcId, - "awsSubnetId": awsSubnetId, - "awsAutoCreateSecurityGroups": awsAutoCreateSecurityGroups, - "awsSecurityGroups": awsSecurityGroups, - "azureVnId": azureVnId, - "azureSubnetId": azureSubnetId, - "azureAutoUpdateSecurityGroups": azureAutoUpdateSecurityGroups, - "internetAccessMode": internetAccessMode, - "mode": "EXISTING_MONOTENANT" - } - - vn = self._perform_tenant_json("POST", "/virtual-networks", body=data) - return FMVirtualNetwork(self, vn) + :param str label: The label of the + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetworkCreator` + """ + if self.cloud == "AWS": + return FMAWSVirtualNetworkCreator(self, label) + elif self.cloud == "Azure": + return FMAzureVirtualNetworkCreator(self, label) ######################################################## From 0611d8d76e7ece9a0dcb2512aca0b2cb28026049 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 21 Sep 2021 18:21:25 +0200 Subject: [PATCH 32/50] FM: format --- dataikuapi/fm/future.py | 62 +++++++++------ dataikuapi/fm/instances.py | 88 +++++++++++++++------ dataikuapi/fm/tenant.py | 36 ++++++--- dataikuapi/fmclient.py | 156 ++++++++++++++++++++++++++++--------- 4 files changed, 247 insertions(+), 95 deletions(-) diff --git a/dataikuapi/fm/future.py b/dataikuapi/fm/future.py index 60d31b57..c52b9a04 100644 --- a/dataikuapi/fm/future.py +++ b/dataikuapi/fm/future.py @@ -1,29 +1,35 @@ import sys, time + class FMFuture(object): """ A future on the DSS instance """ - def __init__(self, client, job_id, state=None, result_wrapper=lambda result: result): - self.client = client - self.job_id = job_id - self.state = state - self.state_is_peek = True - self.result_wrapper = result_wrapper + + def __init__( + self, client, job_id, state=None, result_wrapper=lambda result: result + ): + self.client = client + self.job_id = job_id + self.state = state + self.state_is_peek = True + self.result_wrapper = result_wrapper @staticmethod - def from_resp(client, resp,result_wrapper=lambda result: result): + def from_resp(client, resp, result_wrapper=lambda result: result): """Creates a DSSFuture from a parsed JSON response""" - return FMFuture(client, resp.get('jobId', None), state=resp, result_wrapper=result_wrapper) + return FMFuture( + client, resp.get("jobId", None), state=resp, result_wrapper=result_wrapper + ) @classmethod def get_result_wait_if_needed(cls, client, ret): - if 'jobId' in ret: + if "jobId" in ret: future = FMFuture(client, ret["jobId"], ret) future.wait_for_result() return future.get_result() else: - return ret['result'] + return ret["result"] def abort(self): """ @@ -36,7 +42,8 @@ def get_state(self): Get the status of the future, and its result if it's ready """ self.state = self.client._perform_tenant_json( - "GET", "/futures/%s" % self.job_id, params={'peek' : False}) + "GET", "/futures/%s" % self.job_id, params={"peek": False} + ) self.state_is_peek = False return self.state @@ -45,7 +52,8 @@ def peek_state(self): Get the status of the future, and its result if it's ready """ self.state = self.client._perform_tenant_json( - "GET", "/futures/%s" % self.job_id, params={'peek' : True}) + "GET", "/futures/%s" % self.job_id, params={"peek": True} + ) self.state_is_peek = True return self.state @@ -53,10 +61,14 @@ def get_result(self): """ Get the future result if it's ready, raises an Exception otherwise """ - if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: + if ( + self.state is None + or not self.state.get("hasResult", False) + or self.state_is_peek + ): self.get_state() - if self.state.get('hasResult', False): - return self.result_wrapper(self.state.get('result', None)) + if self.state.get("hasResult", False): + return self.result_wrapper(self.state.get("result", None)) else: raise Exception("Result not ready") @@ -64,22 +76,26 @@ def has_result(self): """ Checks whether the future has a result ready """ - if self.state is None or not self.state.get('hasResult', False): + if self.state is None or not self.state.get("hasResult", False): self.get_state() - return self.state.get('hasResult', False) + return self.state.get("hasResult", False) def wait_for_result(self): """ Wait and get the future result """ - if self.state.get('hasResult', False): - return self.result_wrapper(self.state.get('result', None)) - if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: + if self.state.get("hasResult", False): + return self.result_wrapper(self.state.get("result", None)) + if ( + self.state is None + or not self.state.get("hasResult", False) + or self.state_is_peek + ): self.get_state() - while not self.state.get('hasResult', False): + while not self.state.get("hasResult", False): time.sleep(5) self.get_state() - if self.state.get('hasResult', False): - return self.result_wrapper(self.state.get('result', None)) + if self.state.get("hasResult", False): + return self.result_wrapper(self.state.get("result", None)) else: raise Exception("No result") diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index fec296d1..8e70c543 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,8 +1,11 @@ from enum import Enum from .future import FMFuture + class FMInstanceCreator(object): - def __init__(self, client, label, instance_settings_template_id, virtual_network_id, image_id): + def __init__( + self, client, label, instance_settings_template_id, virtual_network_id, image_id + ): """ Helper to create a DSS Instance @@ -30,7 +33,9 @@ def with_dss_node_type(self, dss_node_type): :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if dss_node_type not in ["design", "automation", "deployer"]: - raise ValueError("Only \"design\", \"automation\" or \"deployer\" dss_node_type are supported") + raise ValueError( + 'Only "design", "automation" or "deployer" dss_node_type are supported' + ) self.data["dssNodeType"] = dss_node_type return self @@ -44,7 +49,15 @@ def with_cloud_instance_type(self, cloud_instance_type): self.data["cloudInstanceType"] = cloud_instance_type return self - def with_data_volume_options(self, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=None, data_volume_encryption_key=None): + def with_data_volume_options( + self, + data_volume_type=None, + data_volume_size=None, + data_volume_size_max=None, + data_volume_IOPS=None, + data_volume_encryption=None, + data_volume_encryption_key=None, + ): """ Set the options of the data volume to use with the DSS Instance @@ -57,7 +70,9 @@ def with_data_volume_options(self, data_volume_type=None, data_volume_size=None, :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if type(data_volume_encryption) is not FMInstanceEncryptionMode: - raise TypeError("data_volume encryption needs to be of type FMInstanceEncryptionMode") + raise TypeError( + "data_volume encryption needs to be of type FMInstanceEncryptionMode" + ) self.data["dataVolumeType"] = data_volume_type self.data["dataVolumeSizeGB"] = data_volume_size @@ -89,7 +104,12 @@ def with_fm_tags(self, fm_tags): class FMAWSInstanceCreator(FMInstanceCreator): - def with_aws_root_volume_options(self, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None): + def with_aws_root_volume_options( + self, + aws_root_volume_size=None, + aws_root_volume_type=None, + aws_root_volume_IOPS=None, + ): """ Set the options of the root volume of the DSS Instance @@ -110,7 +130,9 @@ def create(self): :return: Created DSS Instance :rtype: :class:`dataikuapi.fm.instances.FMAWSInstance` """ - instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) + instance = self.client._perform_tenant_json( + "POST", "/instances", body=self.data + ) return FMAWSInstance(self.client, instance) @@ -122,7 +144,9 @@ def create(self): :return: Created DSS Instance :rtype: :class:`dataikuapi.fm.instances.FMAzureInstance` """ - instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) + instance = self.client._perform_tenant_json( + "POST", "/instances", body=self.data + ) return FMAzureInstance(self.client, instance) @@ -131,10 +155,11 @@ class FMInstance(object): A handle to interact with a DSS instance. Do not create this directly, use :meth:`FMClient.get_instance` or :meth: `FMClient.new_instance_creator` """ + def __init__(self, client, instance_data): - self.client = client + self.client = client self.instance_data = instance_data - self.id = instance_data['id'] + self.id = instance_data["id"] def reprovision(self): """ @@ -143,7 +168,9 @@ def reprovision(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the reprovision process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) + future = self.client._perform_tenant_json( + "GET", "/instances/%s/actions/reprovision" % self.id + ) return FMFuture.from_resp(self.client, future) def deprovision(self): @@ -153,7 +180,9 @@ def deprovision(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deprovision process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) + future = self.client._perform_tenant_json( + "GET", "/instances/%s/actions/deprovision" % self.id + ) return FMFuture.from_resp(self.client, future) def restart_dss(self): @@ -163,21 +192,29 @@ def restart_dss(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the restart process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("GET", "/instances/%s/actions/restart-dss" % self.id) + future = self.client._perform_tenant_json( + "GET", "/instances/%s/actions/restart-dss" % self.id + ) return FMFuture.from_resp(self.client, future) def save(self): """ Update the Instance. """ - self.client._perform_tenant_empty("PUT", "/instances/%s" % self.id, body=self.instance_data) - self.instance_data = self.client._perform_tenant_json("GET", "/instances/%s" % self.id) + self.client._perform_tenant_empty( + "PUT", "/instances/%s" % self.id, body=self.instance_data + ) + self.instance_data = self.client._perform_tenant_json( + "GET", "/instances/%s" % self.id + ) def get_status(self): """ Get the physical DSS instance's status """ - status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) + status = self.client._perform_tenant_json( + "GET", "/instances/%s/status" % self.id + ) return FMInstanceStatus(status) def delete(self): @@ -187,7 +224,9 @@ def delete(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("GET", "/instances/%s/actions/delete" % self.id) + future = self.client._perform_tenant_json( + "GET", "/instances/%s/actions/delete" % self.id + ) return FMFuture.from_resp(self.client, future) def set_automated_snapshots(self, enable, period, keep=0): @@ -198,9 +237,9 @@ def set_automated_snapshots(self, enable, period, keep=0): :param int period: The time period between 2 snapshot in hours :param int keep: Optional, the number of snapshot to keep. Use 0 to keep all snapshots. Defaults to 0. """ - self.instance_data['enableAutomatedSnapshot'] = enable - self.instance_data['automatedSnapshotPeriod'] = period - self.instance_data['automatedSnapshotRetention'] = keep + self.instance_data["enableAutomatedSnapshot"] = enable + self.instance_data["automatedSnapshotPeriod"] = period + self.instance_data["automatedSnapshotRetention"] = keep self.save() def set_custom_certificate(self, pem_data): @@ -211,7 +250,7 @@ def set_custom_certificate(self, pem_data): param: str pem_data: The SSL certificate """ - self.instance_data['sslCertificatePEM'] = pem_data + self.instance_data["sslCertificatePEM"] = pem_data self.save() @@ -223,8 +262,8 @@ def set_elastic_ip(self, enable, elasticip_allocation_id): :param boolan enable: Enable the elastic ip allocation :param str elaticip_allocation_id: AWS ElasticIP allocation ID """ - self.instance_data['awsAssignElasticIP'] = enable - self.instance_data['awsElasticIPAllocationId'] = elasticip_allocation_id + self.instance_data["awsAssignElasticIP"] = enable + self.instance_data["awsElasticIPAllocationId"] = elasticip_allocation_id self.save() @@ -236,8 +275,8 @@ def set_elastic_ip(self, enable, public_ip_id): :param boolan enable: Enable the elastic ip allocation :param str public_ip_id: Azure Public IP ID """ - self.instance_data['azureAssignElasticIP'] = enable - self.instance_data['azurePublicIPId'] = public_ip_id + self.instance_data["azureAssignElasticIP"] = enable + self.instance_data["azurePublicIPId"] = public_ip_id self.save() @@ -252,6 +291,7 @@ class FMInstanceStatus(dict): """A class holding read-only information about an Instance. This class should not be created directly. Instead, use :meth:`FMInstance.get_info` """ + def __init__(self, data): """Do not call this directly, use :meth:`FMInstance.get_status`""" super(FMInstanceStatus, self).__init__(data) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index 67c97dcd..ee6b1fb5 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -1,14 +1,17 @@ import json + + class FMCloudCredentials(object): """ A Tenant Cloud Credentials in the FM instance """ + def __init__(self, client, cloud_credentials): self.client = client self.cloud_credentials = cloud_credentials def set_cmk_key(self, cmk_key_id): - self.cloud_credentials['awsCMKId'] = cmk_key_id + self.cloud_credentials["awsCMKId"] = cmk_key_id self.save() def set_static_license(self, license_file=None, license_string=None): @@ -26,9 +29,11 @@ def set_static_license(self, license_file=None, license_string=None): elif license_string is not None: license = json.load(license_string) else: - raise ValueError("a valid license_file or license_string needs to be provided") - self.cloud_credentials['licenseMode'] = 'STATIC' - self.cloud_credentials['license'] = json.dumps(license, indent=2) + raise ValueError( + "a valid license_file or license_string needs to be provided" + ) + self.cloud_credentials["licenseMode"] = "STATIC" + self.cloud_credentials["license"] = json.dumps(license, indent=2) self.save() def set_automatically_updated_license(self, license_token): @@ -39,8 +44,8 @@ def set_automatically_updated_license(self, license_token): """ if license_token is None: raise ValueError("a valid license_token needs to be provided") - self.cloud_credentials['licenseMode'] = 'AUTO_UPDATE' - self.cloud_credentials['licenseToken'] = license_token + self.cloud_credentials["licenseMode"] = "AUTO_UPDATE" + self.cloud_credentials["licenseToken"] = license_token self.save() def set_authentication(self, authentication): @@ -55,8 +60,9 @@ def set_authentication(self, authentication): def save(self): """Saves back the settings to the project""" - self.client._perform_tenant_empty("PUT", "/cloud-credentials", - body = self.cloud_credentials) + self.client._perform_tenant_empty( + "PUT", "/cloud-credentials", body=self.cloud_credentials + ) class FMCloudAuthentication(dict): @@ -85,7 +91,9 @@ def aws_iam_role(role_arn): params: str role_arn: ARN of the IAM Role """ - return FMCloudAuthentication({"awsAuthenticationMode": "IAM_ROLE", "awsIAMRoleARN": role_arn}) + return FMCloudAuthentication( + {"awsAuthenticationMode": "IAM_ROLE", "awsIAMRoleARN": role_arn} + ) @staticmethod def aws_keypair(access_key_id, secret_access_key): @@ -95,7 +103,13 @@ def aws_keypair(access_key_id, secret_access_key): :param str access_key_id: AWS Access Key ID :param str secret_access_key: AWS Secret Access Key """ - return FMCloudAuthentication({"awsAuthenticationMode": "KEYPAIR", "awsAccessKeyId": access_key_id, "awsSecretAccessKey": secret_access_key}) + return FMCloudAuthentication( + { + "awsAuthenticationMode": "KEYPAIR", + "awsAccessKeyId": access_key_id, + "awsSecretAccessKey": secret_access_key, + } + ) @staticmethod def azure(subscription, tenant_id, environment, client_id): @@ -111,7 +125,7 @@ def azure(subscription, tenant_id, environment, client_id): "azureSubscription": subscription, "azureTenantId": tenant_id, "azureEnvironment": environment, - "azureFMAppClientId": client_id + "azureFMAppClientId": client_id, } return FMCloudAuthentication(data) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 0dbef6fc..b2ec0732 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -8,14 +8,36 @@ from .utils import DataikuException from .fm.tenant import FMCloudCredentials -from .fm.virtualnetworks import FMVirtualNetwork, FMAWSVirtualNetworkCreator, FMAzureVirtualNetworkCreator -from .fm.instances import FMInstance, FMInstanceEncryptionMode, FMAWSInstanceCreator, FMAzureInstanceCreator -from .fm.instancesettingstemplates import FMInstanceSettingsTemplate, FMAWSInstanceSettingsTemplateCreator, FMAzureInstanceSettingsTemplateCreator +from .fm.virtualnetworks import ( + FMVirtualNetwork, + FMAWSVirtualNetworkCreator, + FMAzureVirtualNetworkCreator, +) +from .fm.instances import ( + FMInstance, + FMInstanceEncryptionMode, + FMAWSInstanceCreator, + FMAzureInstanceCreator, +) +from .fm.instancesettingstemplates import ( + FMInstanceSettingsTemplate, + FMAWSInstanceSettingsTemplateCreator, + FMAzureInstanceSettingsTemplateCreator, +) + class FMClient(object): """Entry point for the FM API client""" - def __init__(self, host, api_key_id, api_key_secret, cloud, tenant_id="main", extra_headers=None): + def __init__( + self, + host, + api_key_id, + api_key_secret, + cloud, + tenant_id="main", + extra_headers=None, + ): """ Instantiate a new FM API client on the given host with the given API key. @@ -30,7 +52,7 @@ def __init__(self, host, api_key_id, api_key_secret, cloud, tenant_id="main", ex self.api_key_secret = api_key_secret self.host = host if cloud not in ["AWS", "Azure"]: - raise ValueError("cloud should be either \"AWS\" or \"Azure\"") + raise ValueError('cloud should be either "AWS" or "Azure"') self.cloud = cloud self.__tenant_id = tenant_id self._session = Session() @@ -43,7 +65,6 @@ def __init__(self, host, api_key_id, api_key_secret, cloud, tenant_id="main", ex if extra_headers is not None: self._session.headers.update(extra_headers) - ######################################################## # Tenant ######################################################## @@ -58,7 +79,6 @@ def get_cloud_credentials(self): creds = self._perform_tenant_json("GET", "/cloud-credentials") return FMCloudCredentials(self, creds) - ######################################################## # VirtualNetwork ######################################################## @@ -71,7 +91,7 @@ def list_virtual_networks(self): :rtype: list of :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ vns = self._perform_tenant_json("GET", "/virtual-networks") - return [ FMVirtualNetwork(self, x) for x in vns] + return [FMVirtualNetwork(self, x) for x in vns] def get_virtual_network(self, virtual_network_id): """ @@ -82,10 +102,11 @@ def get_virtual_network(self, virtual_network_id): :return: requested virtual network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ - vn = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) + vn = self._perform_tenant_json( + "GET", "/virtual-networks/%s" % virtual_network_id + ) return FMVirtualNetwork(self, vn) - def new_virtual_network_creator(self, label): """ Instantiate a new virtual network creator @@ -98,7 +119,6 @@ def new_virtual_network_creator(self, label): elif self.cloud == "Azure": return FMAzureVirtualNetworkCreator(self, label) - ######################################################## # Instance settings template ######################################################## @@ -111,7 +131,7 @@ def list_instance_templates(self): :rtype: list of :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` """ templates = self._perform_tenant_json("GET", "/instance-settings-templates") - return [ FMInstanceSettingsTemplate(self, x) for x in templates] + return [FMInstanceSettingsTemplate(self, x) for x in templates] def get_instance_template(self, template_id): """ @@ -122,7 +142,9 @@ def get_instance_template(self, template_id): :return: requested instance settings template :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ - template = self._perform_tenant_json("GET", "/instance-settings-templates/%s" % template_id) + template = self._perform_tenant_json( + "GET", "/instance-settings-templates/%s" % template_id + ) return FMInstanceSettingsTemplate(self, template) def new_instance_template_creator(self, label): @@ -137,7 +159,6 @@ def new_instance_template_creator(self, label): elif self.cloud == "Azure": return FMAzureInstanceSettingsTemplateCreator(self, label) - ######################################################## # Instance ######################################################## @@ -150,7 +171,7 @@ def list_instances(self): :rtype: list of :class:`dataikuapi.fm.instances.FMInstance` """ instances = self._perform_tenant_json("GET", "/instances") - return [ FMInstance(self, **x) for x in instances] + return [FMInstance(self, **x) for x in instances] def get_instance(self, instance_id): """ @@ -164,7 +185,9 @@ def get_instance(self, instance_id): instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return FMInstance(self, instance) - def new_instance_creator(self, label, instance_settings_template_id, virtual_network_id, image_id): + def new_instance_creator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ): """ Instantiate a new instance creator @@ -175,26 +198,41 @@ def new_instance_creator(self, label, instance_settings_template_id, virtual_net :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if self.cloud == "AWS": - return FMAWSInstanceCreator(self, label, instance_settings_template_id, virtual_network_id, image_id) + return FMAWSInstanceCreator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ) elif self.cloud == "Azure": - return FMAzureInstanceCreator(self, label, instance_settings_template_id, virtual_network_id, image_id) - + return FMAzureInstanceCreator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ) ######################################################## # Internal Request handling ######################################################## - def _perform_http(self, method, path, params=None, body=None, stream=False, files=None, raw_body=None): + def _perform_http( + self, + method, + path, + params=None, + body=None, + stream=False, + files=None, + raw_body=None, + ): if body is not None: body = json.dumps(body) if raw_body is not None: body = raw_body try: http_res = self._session.request( - method, "%s/api/public%s" % (self.host, path), - params=params, data=body, - files = files, - stream = stream) + method, + "%s/api/public%s" % (self.host, path), + params=params, + data=body, + files=files, + stream=stream, + ) http_res.raise_for_status() return http_res except exceptions.HTTPError: @@ -202,16 +240,60 @@ def _perform_http(self, method, path, params=None, body=None, stream=False, file ex = http_res.json() except ValueError: ex = {"message": http_res.text} - raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) - - def _perform_empty(self, method, path, params=None, body=None, files = None, raw_body=None): - self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body) - - def _perform_json(self, method, path, params=None, body=None,files=None, raw_body=None): - return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).json() - - def _perform_tenant_json(self, method, path, params=None, body=None,files=None, raw_body=None): - return self._perform_json(method, "/tenants/%s%s" % ( self.__tenant_id, path ), params=params, body=body, files=files, raw_body=raw_body) - - def _perform_tenant_empty(self, method, path, params=None, body=None, files = None, raw_body=None): - self._perform_empty(method, "/tenants/%s%s" % ( self.__tenant_id, path ), params=params, body=body, files=files, raw_body=raw_body) + raise DataikuException( + "%s: %s" + % ( + ex.get("errorType", "Unknown error"), + ex.get("message", "No message"), + ) + ) + + def _perform_empty( + self, method, path, params=None, body=None, files=None, raw_body=None + ): + self._perform_http( + method, + path, + params=params, + body=body, + files=files, + stream=False, + raw_body=raw_body, + ) + + def _perform_json( + self, method, path, params=None, body=None, files=None, raw_body=None + ): + return self._perform_http( + method, + path, + params=params, + body=body, + files=files, + stream=False, + raw_body=raw_body, + ).json() + + def _perform_tenant_json( + self, method, path, params=None, body=None, files=None, raw_body=None + ): + return self._perform_json( + method, + "/tenants/%s%s" % (self.__tenant_id, path), + params=params, + body=body, + files=files, + raw_body=raw_body, + ) + + def _perform_tenant_empty( + self, method, path, params=None, body=None, files=None, raw_body=None + ): + self._perform_empty( + method, + "/tenants/%s%s" % (self.__tenant_id, path), + params=params, + body=body, + files=files, + raw_body=raw_body, + ) From 26185d1a1e55de11db62cd3bf3c0a6cee7fbdc88 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 21 Sep 2021 20:26:27 +0200 Subject: [PATCH 33/50] FM: Split client by cloud --- dataikuapi/__init__.py | 2 +- dataikuapi/fm/virtualnetworks.py | 2 +- dataikuapi/fmclient.py | 182 +++++++++++++++++++++---------- 3 files changed, 125 insertions(+), 61 deletions(-) diff --git a/dataikuapi/__init__.py b/dataikuapi/__init__.py index 569b1017..5c4337bb 100644 --- a/dataikuapi/__init__.py +++ b/dataikuapi/__init__.py @@ -1,5 +1,5 @@ from .dssclient import DSSClient -from .fmclient import FMClient +from .fmclient import FMClientAWS, FMClientAzure from .apinode_client import APINodeClient from .apinode_admin_client import APINodeAdminClient diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index ca5a3ced..9b98173b 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -69,7 +69,7 @@ def create(self): class FMAzureVirtualNetworkCreator(FMVirtualNetworkCreator): - def with_virtual_network(self, azure_vn_id, azure_subnet_id): + def with_azure_virtual_network(self, azure_vn_id, azure_subnet_id): """ Setup the Azure Virtual Network and Subnet to with the VirtualNetwork diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index b2ec0732..404cd016 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -27,33 +27,25 @@ class FMClient(object): - """Entry point for the FM API client""" - def __init__( self, host, api_key_id, api_key_secret, - cloud, tenant_id="main", extra_headers=None, ): """ - Instantiate a new FM API client on the given host with the given API key. - - API keys can be managed in FM on the project page or in the global settings. - - The API key will define which operations are allowed for the client. - - :param str host: Full url of the FM - + Base class for the different FM Clients + Do not create this class, instead use :class:`dataikuapi.FMClientAWS` or :class:`dataikuapi.FMClientAzure` """ + if self.cloud == None: + raise NotImplementedError( + "Do not use FMClient directly, instead use FMClientAWS or FMClientAzure" + ) self.api_key_id = api_key_id self.api_key_secret = api_key_secret self.host = host - if cloud not in ["AWS", "Azure"]: - raise ValueError('cloud should be either "AWS" or "Azure"') - self.cloud = cloud self.__tenant_id = tenant_id self._session = Session() @@ -107,18 +99,6 @@ def get_virtual_network(self, virtual_network_id): ) return FMVirtualNetwork(self, vn) - def new_virtual_network_creator(self, label): - """ - Instantiate a new virtual network creator - - :param str label: The label of the - :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetworkCreator` - """ - if self.cloud == "AWS": - return FMAWSVirtualNetworkCreator(self, label) - elif self.cloud == "Azure": - return FMAzureVirtualNetworkCreator(self, label) - ######################################################## # Instance settings template ######################################################## @@ -147,18 +127,6 @@ def get_instance_template(self, template_id): ) return FMInstanceSettingsTemplate(self, template) - def new_instance_template_creator(self, label): - """ - Instantiate a new instance template creator - - :param str label: The label of the instance - :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplateCreator` - """ - if self.cloud == "AWS": - return FMAWSInstanceSettingsTemplateCreator(self, label) - elif self.cloud == "Azure": - return FMAzureInstanceSettingsTemplateCreator(self, label) - ######################################################## # Instance ######################################################## @@ -185,27 +153,6 @@ def get_instance(self, instance_id): instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return FMInstance(self, instance) - def new_instance_creator( - self, label, instance_settings_template_id, virtual_network_id, image_id - ): - """ - Instantiate a new instance creator - - :param str label: The label of the instance - :param str instance_settings_template: The instance settings template id this instance should be based on - :param str virtual_network: The virtual network where the instance should be spawned - :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) - :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` - """ - if self.cloud == "AWS": - return FMAWSInstanceCreator( - self, label, instance_settings_template_id, virtual_network_id, image_id - ) - elif self.cloud == "Azure": - return FMAzureInstanceCreator( - self, label, instance_settings_template_id, virtual_network_id, image_id - ) - ######################################################## # Internal Request handling ######################################################## @@ -297,3 +244,120 @@ def _perform_tenant_empty( files=files, raw_body=raw_body, ) + + +class FMClientAWS(FMClient): + def __init__( + self, + host, + api_key_id, + api_key_secret, + tenant_id="main", + extra_headers=None, + ): + """ + AWS Only - Instantiate a new FM API client on the given host with the given API key. + + API keys can be managed in FM on the project page or in the global settings. + + The API key will define which operations are allowed for the client. + + :param str host: Full url of the FM + + """ + self.cloud = "AWS" + super(FMClientAWS, self).__init__( + host, api_key_id, api_key_secret, tenant_id, extra_headers + ) + + def new_virtual_network_creator(self, label): + """ + Instantiate a new virtual network creator + + :param str label: The label of the + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAWSVirtualNetworkCreator` + """ + return FMAWSVirtualNetworkCreator(self, label) + + def new_instance_template_creator(self, label): + """ + Instantiate a new instance template creator + + :param str label: The label of the instance + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` + """ + return FMAWSInstanceSettingsTemplateCreator(self, label) + + def new_instance_creator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ): + """ + Instantiate a new instance creator + + :param str label: The label of the instance + :param str instance_settings_template: The instance settings template id this instance should be based on + :param str virtual_network: The virtual network where the instance should be spawned + :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) + :rtype: :class:`dataikuapi.fm.instances.FMAWSInstanceCreator` + """ + return FMAWSInstanceCreator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ) + + +class FMClientAzure(FMClient): + def __init__( + self, + host, + api_key_id, + api_key_secret, + tenant_id="main", + extra_headers=None, + ): + """ + Azure Only - Instantiate a new FM API client on the given host with the given API key. + + API keys can be managed in FM on the project page or in the global settings. + + The API key will define which operations are allowed for the client. + + :param str host: Full url of the FM + """ + self.cloud = "Azure" + super(FMClientAWS, self).__init__( + host, api_key_id, api_key_secret, tenant_id, extra_headers + ) + + def new_virtual_network_creator(self, label): + """ + Instantiate a new virtual network creator + + :param str label: The label of the + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAzureVirtualNetworkCreator` + """ + return FMAzureVirtualNetworkCreator(self, label) + + def new_instance_template_creator(self, label): + """ + Instantiate a new instance template creator + + :param str label: The label of the instance + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAzureInstanceSettingsTemplateCreator` + """ + return FMAzureInstanceSettingsTemplateCreator(self, label) + + def new_instance_creator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ): + """ + Instantiate a new instance creator + + :param str label: The label of the instance + :param str instance_settings_template: The instance settings template id this instance should be based on + :param str virtual_network: The virtual network where the instance should be spawned + :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) + :rtype: :class:`dataikuapi.fm.instances.FMAzureInstanceCreator` + """ + return FMAzureInstanceCreator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ) From 6182ba85ee9fc5adc0d127f6d148653140965591 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 21 Sep 2021 21:48:08 +0200 Subject: [PATCH 34/50] FM: Update Cloud Tags --- dataikuapi/fm/tenant.py | 58 +++++++++++++++++++++++++++++++++++++++++ dataikuapi/fmclient.py | 14 +++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index ee6b1fb5..5bf0499e 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -65,6 +65,64 @@ def save(self): ) +class FMCloudTags(object): + """ + A Tenant Cloud Tags in the FM instance + """ + + def __init__(self, client, tenant_id, cloud_tags): + self.client = client + self.tenant_id = tenant_id + self.cloud_tags = json.loads(cloud_tags["msg"]) + + def add_tag(self, key, value): + """ + Add a tag to the tenant + + + :param str key: Tag key + :param str value: Tag value + """ + if key in self.cloud_tags: + raise Exception("Key already exists") + self.cloud_tags[key] = value + + def update_tag(self, key, new_key=None, new_value=None): + """ + Update a tag key or value + + + :param str key: Key of the tag to update + :param str new_key: Optional, new key for the tag + :param str new_value: Optional, new value for the tag + """ + if key not in self.cloud_tags: + raise Exception("Key does not exists") + if new_value: + self.cloud_tags[key] = new_value + if new_key: + self.cloud_tags[new_key] = self.cloud_tags[key] + del self.cloud_tags[key] + + def delete_tag(self, key): + """ + Delete a tag + + + :param str key: Key of the tag to delete + """ + if key not in self.cloud_tags: + raise Exception("Key does not exists") + del self.cloud_tags[key] + + def save(self): + """Saves the tags on FM""" + + self.client._perform_empty( + "PUT", "/tenants/%s/cloud-tags" % (self.tenant_id), body=self.cloud_tags + ) + + class FMCloudAuthentication(dict): def __init__(self, data): """ diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 404cd016..744bba44 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -7,7 +7,7 @@ from enum import Enum from .utils import DataikuException -from .fm.tenant import FMCloudCredentials +from .fm.tenant import FMCloudCredentials, FMCloudTags from .fm.virtualnetworks import ( FMVirtualNetwork, FMAWSVirtualNetworkCreator, @@ -71,6 +71,18 @@ def get_cloud_credentials(self): creds = self._perform_tenant_json("GET", "/cloud-credentials") return FMCloudCredentials(self, creds) + def get_cloud_tags(self, tenant_id): + """ + Get Tenant's Cloud Tags + + :param string tenant_id + + :return: tenant's cloud tags + :rtype: :class:`dataikuapi.fm.tenant.FMCloudTags` + """ + tags = self._perform_json("GET", "/tenants/%s/cloud-tags" % tenant_id) + return FMCloudTags(self, tenant_id, tags) + ######################################################## # VirtualNetwork ######################################################## From b64b1715efd7a9bf62c7da0a6c7a963ef8f0e76e Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Wed, 22 Sep 2021 15:47:07 +0200 Subject: [PATCH 35/50] typo --- dataikuapi/fm/tenant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index 5bf0499e..d46b5ac5 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -27,7 +27,7 @@ def set_static_license(self, license_file=None, license_string=None): with open(license_file) as json_file: license = json.load(json_file) elif license_string is not None: - license = json.load(license_string) + license = json.loads(license_string) else: raise ValueError( "a valid license_file or license_string needs to be provided" From e7b1f5a74f2380162e04c7b5388430a11f2818b0 Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Wed, 22 Sep 2021 15:53:53 +0200 Subject: [PATCH 36/50] fix created object --- dataikuapi/fm/virtualnetworks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index 9b98173b..9720a067 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -65,7 +65,7 @@ def create(self): vn = self.client._perform_tenant_json( "POST", "/virtual-networks", body=self.data ) - return FMAWSVirtualNetwork(self, vn) + return FMAWSVirtualNetwork(self.client, vn) class FMAzureVirtualNetworkCreator(FMVirtualNetworkCreator): @@ -99,7 +99,7 @@ def create(self): vn = self.client._perform_tenant_json( "POST", "/virtual-networks", body=self.data ) - return FMAzureVirtualNetwork(self, vn) + return FMAzureVirtualNetwork(self.client, vn) class FMVirtualNetwork(object): From 0e9e58c84e3172925886ffa8cc593e845d6c0edc Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Wed, 22 Sep 2021 15:54:11 +0200 Subject: [PATCH 37/50] type returned objects to the appropriate subclass --- dataikuapi/fmclient.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 744bba44..2a6a3d6a 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -12,12 +12,16 @@ FMVirtualNetwork, FMAWSVirtualNetworkCreator, FMAzureVirtualNetworkCreator, + FMAWSVirtualNetwork, + FMAzureVirtualNetwork ) from .fm.instances import ( FMInstance, FMInstanceEncryptionMode, FMAWSInstanceCreator, FMAzureInstanceCreator, + FMAWSInstance, + FMAzureInstance ) from .fm.instancesettingstemplates import ( FMInstanceSettingsTemplate, @@ -87,6 +91,14 @@ def get_cloud_tags(self, tenant_id): # VirtualNetwork ######################################################## + def _make_virtual_network(self, vn): + if self.cloud == 'AWS': + return FMAWSVirtualNetwork(self, vn) + elif self.cloud == 'Azure': + return FMAzureVirtualNetwork(self, vn) + else: + raise Exception("Unknown cloud type %s" % self.cloud) + def list_virtual_networks(self): """ List all Virtual Networks @@ -95,7 +107,7 @@ def list_virtual_networks(self): :rtype: list of :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ vns = self._perform_tenant_json("GET", "/virtual-networks") - return [FMVirtualNetwork(self, x) for x in vns] + return [self._make_virtual_network(x) for x in vns] def get_virtual_network(self, virtual_network_id): """ @@ -109,7 +121,7 @@ def get_virtual_network(self, virtual_network_id): vn = self._perform_tenant_json( "GET", "/virtual-networks/%s" % virtual_network_id ) - return FMVirtualNetwork(self, vn) + return self._make_virtual_network(vn) ######################################################## # Instance settings template @@ -143,6 +155,14 @@ def get_instance_template(self, template_id): # Instance ######################################################## + def _make_instance(self, i): + if self.cloud == 'AWS': + return FMAWSInstance(self, i) + elif self.cloud == 'Azure': + return FMAzureInstance(self, i) + else: + raise Exception("Unknown cloud type %s" % self.cloud) + def list_instances(self): """ List all DSS Instances @@ -151,7 +171,7 @@ def list_instances(self): :rtype: list of :class:`dataikuapi.fm.instances.FMInstance` """ instances = self._perform_tenant_json("GET", "/instances") - return [FMInstance(self, **x) for x in instances] + return [self._make_instance(x) for x in instances] def get_instance(self, instance_id): """ @@ -163,7 +183,7 @@ def get_instance(self, instance_id): :rtype: :class:`dataikuapi.fm.instances.FMInstance` """ instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) - return FMInstance(self, instance) + return self._make_instance(instance) ######################################################## # Internal Request handling From a766032f7d864a51320bb50ae292a9c930684ab7 Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Wed, 22 Sep 2021 16:06:20 +0200 Subject: [PATCH 38/50] fix client passed to created object --- dataikuapi/fm/instancesettingstemplates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 4fe035c6..6d91d51a 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -26,7 +26,7 @@ def create(self): template = self.client._perform_tenant_json( "POST", "/instance-settings-templates", body=self.data ) - return FMInstanceSettingsTemplate(self, template) + return FMInstanceSettingsTemplate(self.client, template) def with_setup_actions(self, setup_actions): """ From e3947dd8403ef92dc07ce8400a259fe53a5d305f Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 23 Sep 2021 11:54:44 +0200 Subject: [PATCH 39/50] FM: Simplify CloudTags --- dataikuapi/fm/tenant.py | 69 +++++++++-------------------------------- dataikuapi/fmclient.py | 18 +++++------ 2 files changed, 23 insertions(+), 64 deletions(-) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index d46b5ac5..80e45d95 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -12,29 +12,27 @@ def __init__(self, client, cloud_credentials): def set_cmk_key(self, cmk_key_id): self.cloud_credentials["awsCMKId"] = cmk_key_id - self.save() + return self - def set_static_license(self, license_file=None, license_string=None): + def set_static_license(self, license_file_path=None, license_string=None): """ Set a default static license for the DSS instances - Requires either a license_file or a license_string - - :param str license_file: Optional, load the license from a json file + :param str license_file_path: Optional, load the license from a json file :param str license_string: Optional, load the license from a json string """ - if license_file is not None: - with open(license_file) as json_file: + if license_file_path is not None: + with open(license_file_path) as json_file: license = json.load(json_file) elif license_string is not None: license = json.loads(license_string) else: raise ValueError( - "a valid license_file or license_string needs to be provided" + "a valid license_file_path or license_string needs to be provided" ) self.cloud_credentials["licenseMode"] = "STATIC" self.cloud_credentials["license"] = json.dumps(license, indent=2) - self.save() + return self def set_automatically_updated_license(self, license_token): """ @@ -46,7 +44,7 @@ def set_automatically_updated_license(self, license_token): raise ValueError("a valid license_token needs to be provided") self.cloud_credentials["licenseMode"] = "AUTO_UPDATE" self.cloud_credentials["licenseToken"] = license_token - self.save() + return self def set_authentication(self, authentication): """ @@ -55,7 +53,7 @@ def set_authentication(self, authentication): :param object: a :class:`dataikuapi.fm.tenant.FMCloudAuthentication` """ self.cloud_credentials.update(authentication) - self.save() + return self def save(self): """Saves back the settings to the project""" @@ -70,57 +68,18 @@ class FMCloudTags(object): A Tenant Cloud Tags in the FM instance """ - def __init__(self, client, tenant_id, cloud_tags): + def __init__(self, client, cloud_tags): self.client = client - self.tenant_id = tenant_id self.cloud_tags = json.loads(cloud_tags["msg"]) - def add_tag(self, key, value): - """ - Add a tag to the tenant - - - :param str key: Tag key - :param str value: Tag value - """ - if key in self.cloud_tags: - raise Exception("Key already exists") - self.cloud_tags[key] = value - - def update_tag(self, key, new_key=None, new_value=None): - """ - Update a tag key or value - - - :param str key: Key of the tag to update - :param str new_key: Optional, new key for the tag - :param str new_value: Optional, new value for the tag - """ - if key not in self.cloud_tags: - raise Exception("Key does not exists") - if new_value: - self.cloud_tags[key] = new_value - if new_key: - self.cloud_tags[new_key] = self.cloud_tags[key] - del self.cloud_tags[key] - - def delete_tag(self, key): - """ - Delete a tag - - - :param str key: Key of the tag to delete - """ - if key not in self.cloud_tags: - raise Exception("Key does not exists") - del self.cloud_tags[key] + @property + def tags(self): + return self.cloud_tags def save(self): """Saves the tags on FM""" - self.client._perform_empty( - "PUT", "/tenants/%s/cloud-tags" % (self.tenant_id), body=self.cloud_tags - ) + self.client._perform_tenant_empty("PUT", "/cloud-tags", body=self.cloud_tags) class FMCloudAuthentication(dict): diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 2a6a3d6a..1b8bc2db 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -13,7 +13,7 @@ FMAWSVirtualNetworkCreator, FMAzureVirtualNetworkCreator, FMAWSVirtualNetwork, - FMAzureVirtualNetwork + FMAzureVirtualNetwork, ) from .fm.instances import ( FMInstance, @@ -21,7 +21,7 @@ FMAWSInstanceCreator, FMAzureInstanceCreator, FMAWSInstance, - FMAzureInstance + FMAzureInstance, ) from .fm.instancesettingstemplates import ( FMInstanceSettingsTemplate, @@ -75,7 +75,7 @@ def get_cloud_credentials(self): creds = self._perform_tenant_json("GET", "/cloud-credentials") return FMCloudCredentials(self, creds) - def get_cloud_tags(self, tenant_id): + def get_cloud_tags(self): """ Get Tenant's Cloud Tags @@ -84,17 +84,17 @@ def get_cloud_tags(self, tenant_id): :return: tenant's cloud tags :rtype: :class:`dataikuapi.fm.tenant.FMCloudTags` """ - tags = self._perform_json("GET", "/tenants/%s/cloud-tags" % tenant_id) - return FMCloudTags(self, tenant_id, tags) + tags = self._perform_tenant_json("GET", "/cloud-tags") + return FMCloudTags(self, tags) ######################################################## # VirtualNetwork ######################################################## def _make_virtual_network(self, vn): - if self.cloud == 'AWS': + if self.cloud == "AWS": return FMAWSVirtualNetwork(self, vn) - elif self.cloud == 'Azure': + elif self.cloud == "Azure": return FMAzureVirtualNetwork(self, vn) else: raise Exception("Unknown cloud type %s" % self.cloud) @@ -156,9 +156,9 @@ def get_instance_template(self, template_id): ######################################################## def _make_instance(self, i): - if self.cloud == 'AWS': + if self.cloud == "AWS": return FMAWSInstance(self, i) - elif self.cloud == 'Azure': + elif self.cloud == "Azure": return FMAzureInstance(self, i) else: raise Exception("Unknown cloud type %s" % self.cloud) From 9e2b9bb4fae557e4f7a15392a2ec73a96ead6c7d Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 23 Sep 2021 12:38:10 +0200 Subject: [PATCH 40/50] FM: Review feedback on VirtualNetworks --- dataikuapi/fm/virtualnetworks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index 9720a067..35aae048 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -45,11 +45,11 @@ def with_auto_create_security_groups(self): self.data["awsAutoCreateSecurityGroups"] = True return self - def with_aws_security_groups(self, aws_security_groups): + def with_aws_security_groups(self, *aws_security_groups): """ Use pre-created AWS Security Groups - :param list aws_security_groups: A list of up to 5 security group ids to assign to the instances created in this virtual network. + :param str *aws_security_groups: Up to 5 security group ids to assign to the instances created in this virtual network. """ self.data["awsAutoCreateSecurityGroups"] = False self.data["awsSecurityGroups"] = aws_security_groups @@ -149,7 +149,7 @@ def set_fleet_management( self.vn_data["managedNodesDirectory"] = enable self.vn_data["eventServerNodeLabel"] = event_server self.vn_data["nodesDirectoryDeployerMode"] = deployer_management - self.save() + return self def set_https_strategy(self, https_strategy): """ @@ -158,7 +158,7 @@ def set_https_strategy(self, https_strategy): :param object: a :class:`dataikuapi.fm.virtualnetworks.FMHTTPSStrategy` """ self.vn_data.update(https_strategy) - self.save() + return self class FMAWSVirtualNetwork(FMVirtualNetwork): @@ -183,7 +183,7 @@ def set_dns_strategy( else: self.vn_data["dnsStrategy"] = "NONE" - self.save() + return self class FMAzureVirtualNetwork(FMVirtualNetwork): @@ -201,7 +201,7 @@ def set_dns_strategy(self, assign_domain_name, azure_dns_zone_id=None): else: self.vn_data["dnsStrategy"] = "NONE" - self.save() + return self class FMHTTPSStrategy(dict): From a8cf85a2440decc067b93b2f85c46be851172203 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 23 Sep 2021 12:45:42 +0200 Subject: [PATCH 41/50] FM: Review feedback on InstanceSettingsTemplates --- dataikuapi/fm/instancesettingstemplates.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 6d91d51a..1ca123f0 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -38,8 +38,23 @@ def with_setup_actions(self, setup_actions): self.data["setupActions"] = setup_actions return self - def with_license(self, license): - self.data["license"] = license + def with_license(self, license_file_path=None, license_string=None): + """ + Override global license + + :param str license_file_path: Optional, load the license from a json file + :param str license_string: Optional, load the license from a json string + """ + if license_file_path is not None: + with open(license_file_path) as json_file: + license = json.load(json_file) + elif license_string is not None: + license = json.loads(license_string) + else: + raise ValueError( + "a valid license_file_path or license_string needs to be provided" + ) + self.data["license"] = json.dumps(license, indent=2) return self @@ -211,7 +226,7 @@ def add_setup_action(self, setup_action): :param object setup_action: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ self.ist_data["setupActions"].append(setup_action) - self.save() + return self class FMSetupAction(dict): From fb41a80a7fa4cce59014f724125caf88fb26bc5b Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 23 Sep 2021 12:47:02 +0200 Subject: [PATCH 42/50] FM: Review feedback on Instances --- dataikuapi/fm/instances.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 8e70c543..51301f21 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -240,7 +240,7 @@ def set_automated_snapshots(self, enable, period, keep=0): self.instance_data["enableAutomatedSnapshot"] = enable self.instance_data["automatedSnapshotPeriod"] = period self.instance_data["automatedSnapshotRetention"] = keep - self.save() + return self def set_custom_certificate(self, pem_data): """ @@ -251,7 +251,7 @@ def set_custom_certificate(self, pem_data): param: str pem_data: The SSL certificate """ self.instance_data["sslCertificatePEM"] = pem_data - self.save() + return self class FMAWSInstance(FMInstance): @@ -264,7 +264,7 @@ def set_elastic_ip(self, enable, elasticip_allocation_id): """ self.instance_data["awsAssignElasticIP"] = enable self.instance_data["awsElasticIPAllocationId"] = elasticip_allocation_id - self.save() + return self class FMAzureInstance(FMInstance): @@ -277,7 +277,7 @@ def set_elastic_ip(self, enable, public_ip_id): """ self.instance_data["azureAssignElasticIP"] = enable self.instance_data["azurePublicIPId"] = public_ip_id - self.save() + return self class FMInstanceEncryptionMode(Enum): From 59fb68857b30eab8b5ce742b4023304c11b0aae9 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 28 Sep 2021 17:46:45 +0200 Subject: [PATCH 43/50] Fix import Enum --- dataikuapi/fm/instances.py | 10 +++++++++- dataikuapi/fm/instancesettingstemplates.py | 10 +++++++++- dataikuapi/fmclient.py | 10 +++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 51301f21..42e1641a 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,6 +1,14 @@ -from enum import Enum from .future import FMFuture +import sys + +if sys.version_info > (3, 4): + from enum import Enum +else: + + class Enum(object): + pass + class FMInstanceCreator(object): def __init__( diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 1ca123f0..abf36407 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -1,7 +1,15 @@ -from enum import Enum import json from dataikuapi.fm.future import FMFuture +import sys + +if sys.version_info > (3, 4): + from enum import Enum +else: + + class Enum(object): + pass + class FMInstanceSettingsTemplateCreator(object): def __init__(self, client, label): diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 1b8bc2db..4ef7667f 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -4,7 +4,6 @@ from requests.auth import HTTPBasicAuth import os.path as osp -from enum import Enum from .utils import DataikuException from .fm.tenant import FMCloudCredentials, FMCloudTags @@ -29,6 +28,15 @@ FMAzureInstanceSettingsTemplateCreator, ) +import sys + +if sys.version_info > (3, 4): + from enum import Enum +else: + + class Enum(object): + pass + class FMClient(object): def __init__( From 603f628be4ff99172cdab36ab3610e90ef334f0e Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 28 Sep 2021 17:52:02 +0200 Subject: [PATCH 44/50] Add FM client to packages --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e1ff790..dca0923d 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ name='dataiku-api-client', version=VERSION, license="Apache Software License", - packages=["dataikuapi", "dataikuapi.dss", "dataikuapi.apinode_admin"], + packages=["dataikuapi", "dataikuapi.dss", "dataikuapi.apinode_admin", "dataikuapi.fm"], description="Python API client for Dataiku APIs", long_description=long_description, author="Dataiku", From 2abe499957733e2d21c9dbf3a154d420d96b3b24 Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Tue, 28 Sep 2021 21:37:53 +0200 Subject: [PATCH 45/50] public api for the list of images --- dataikuapi/fmclient.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 4ef7667f..662b8ef9 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -193,6 +193,14 @@ def get_instance(self, instance_id): instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return self._make_instance(instance) + def list_instance_images(self): + """ + List all available images to create new instances + + :return: list of images, as a pair of id and label + """ + return self._perform_tenant_json("GET", "/images") + ######################################################## # Internal Request handling ######################################################## From 6c901de6941ed182bc5ec65190520f557cd98ff0 Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Tue, 28 Sep 2021 21:38:14 +0200 Subject: [PATCH 46/50] let VN be created with the default values --- dataikuapi/fm/virtualnetworks.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index 35aae048..3419f420 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -10,6 +10,7 @@ def __init__(self, client, label): """ self.client = client self.data = {} + self.use_default_values = False self.data["label"] = label self.data["mode"] = "EXISTING_MONOTENANT" @@ -25,6 +26,13 @@ def with_internet_access_mode(self, internet_access_mode): self.data["internetAccessMode"] = internet_access_mode return self + def with_default_values(self): + """ + Setup the VPC and Subnet to with the default values: the vpc and subnet of the FM instance + """ + self.use_default_values = True + return self + class FMAWSVirtualNetworkCreator(FMVirtualNetworkCreator): def with_vpc(self, aws_vpc_id, aws_subnet_id): @@ -63,7 +71,7 @@ def create(self): :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAWSVirtualNetwork` """ vn = self.client._perform_tenant_json( - "POST", "/virtual-networks", body=self.data + "POST", "/virtual-networks", body=self.data, params={ 'useDefaultValues':self.use_default_values } ) return FMAWSVirtualNetwork(self.client, vn) @@ -97,7 +105,7 @@ def create(self): :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAzureVirtualNetwork` """ vn = self.client._perform_tenant_json( - "POST", "/virtual-networks", body=self.data + "POST", "/virtual-networks", body=self.data, params={ 'useDefaultValues':self.use_default_values } ) return FMAzureVirtualNetwork(self.client, vn) From 21e1abd082960b9d18b0c5423ce29ea143cff2c4 Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Tue, 28 Sep 2021 21:38:34 +0200 Subject: [PATCH 47/50] expose actions on instances --- dataikuapi/fm/instances.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 42e1641a..7e12e710 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -237,6 +237,44 @@ def delete(self): ) return FMFuture.from_resp(self.client, future) + def get_initial_password(self): + """ + Get the initial DSS admin password. + + Can only be called once + + :return: a password for the 'admin' user. + """ + return self.client._perform_tenant_json( + "GET", "/instances/%s/actions/get-initial-password" % self.id + ) + + def reset_user_password(self, username, password): + """ + Reset the password for a user on the DSS instance + + :param string username: login + :param string password: new password + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the password reset process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` + """ + future = self.client._perform_tenant_json( + "GET", "/instances/%s/actions/reset-user-password" % self.id, params={ 'userName':username, 'password':password } + ) + return FMFuture.from_resp(self.client, future) + + def replay_setup_actions(self): + """ + Replay the setup actions on the DSS instance + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the replay process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` + """ + future = self.client._perform_tenant_json( + "GET", "/instances/%s/actions/replay-setup-actions" % self.id + ) + return FMFuture.from_resp(self.client, future) + def set_automated_snapshots(self, enable, period, keep=0): """ Set the automated snapshots policy for this instance From 509c2fcdc3abd3cd0536d1612403fc5b88ad3eec Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Tue, 28 Sep 2021 21:38:44 +0200 Subject: [PATCH 48/50] expose snapshots --- dataikuapi/fm/instances.py | 87 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 7e12e710..0a05507f 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -300,6 +300,43 @@ def set_custom_certificate(self, pem_data): return self + ######################################################## + # Snapshots + ######################################################## + + def list_snapshots(self): + """ + List all snapshots of this instance + + :return: list of snapshots + :rtype: list of :class:`dataikuapi.fm.instances.FMSnapshot` + """ + snapshots = self.client._perform_tenant_json("GET", "/instances/%s/snapshots" % self.id) + return [FMSnapshot(self.client, self.id, x["id"], x) for x in snapshots] + + def get_snapshot(self, snapshot_id): + """ + Get a snapshot of this instance + + :param str snapshot_id: identifier of the snapshot + + :return: Snapshot + :rtype: :class:`dataikuapi.fm.instances.FMSnapshot` + """ + return FMSnapshot(self.client, self.id, snapshot_id) + + def snapshot(self, reason_for_snapshot=None): + """ + Create a snapshot of the DSS instance + + :return: Snapshot + :rtype: :class:`dataikuapi.fm.instances.FMSnapshot` + """ + snapshot = self.client._perform_tenant_json( + "POST", "/instances/%s/snapshots" % self.id, params={ "reasonForSnapshot":reason_for_snapshot } + ) + return FMSnapshot(self.client, self.id, snapshot["id"], snapshot) + class FMAWSInstance(FMInstance): def set_elastic_ip(self, enable, elasticip_allocation_id): """ @@ -341,3 +378,53 @@ class FMInstanceStatus(dict): def __init__(self, data): """Do not call this directly, use :meth:`FMInstance.get_status`""" super(FMInstanceStatus, self).__init__(data) + + +class FMSnapshot(object): + """ + A handle to interact with a snapshot of a DSS instance. + Do not create this directly, use :meth:`FMInstance.snapshot` + """ + + def __init__(self, client, instance_id, snapshot_id, snapshot_data=None): + self.client = client + self.instance_id = instance_id + self.snapshot_id = snapshot_id + self.snapshot_data = snapshot_data + + def get_info(self): + """ + Get the information about this snapshot + + :return: a dict + """ + if self.snapshot_data is None: + self.snapshot_data = self.client._perform_tenant_json( + "GET", "/instances/%s/snapshots/%s" % (self.instance_id, self.snapshot_id) + ) + return self.snapshot_data + + def reprovision(self): + """ + Reprovision the physical DSS instance from this snapshot + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the reprovision process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` + """ + future = self.client._perform_tenant_json( + "POST", "/instances/%s/snapshots/%s/reprovision" % (self.instance_id, self.snapshot_id) + ) + return FMFuture.from_resp(self.client, future) + + def delete(self): + """ + Delete the snapshot + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` + """ + future = self.client._perform_tenant_json( + "DELETE", "/instances/%s/snapshots/%s" % (self.instance_id, self.snapshot_id) + ) + return FMFuture.from_resp(self.client, future) + From 8646164f15120d2f702d2005489fb9f180737879 Mon Sep 17 00:00:00 2001 From: Valentin Thorey Date: Fri, 8 Oct 2021 17:38:26 +0200 Subject: [PATCH 49/50] Rename trainDiagnostics into mlDiagnostics --- dataikuapi/dss/ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index 2fb31ac1..ed9ffe45 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -1867,7 +1867,7 @@ def get_diagnostics(self): :returns: list of diagnostics :rtype: list of type `dataikuapi.dss.ml.DSSMLDiagnostic` """ - diagnostics = self.details.get("trainDiagnostics", {}) + diagnostics = self.details.get("mlDiagnostics", {}) return [DSSMLDiagnostic(d) for d in diagnostics.get("diagnostics", [])] def generate_documentation(self, folder_id=None, path=None): From 4e7d77e2e10b60e42c61856c4379726662a955a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20P=C3=A9net?= Date: Mon, 11 Oct 2021 13:56:52 +0200 Subject: [PATCH 50/50] Trading Run IDs for Evaluation IDs (#179) --- dataikuapi/dss/modelevaluationstore.py | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/dataikuapi/dss/modelevaluationstore.py b/dataikuapi/dss/modelevaluationstore.py index 4d2d2900..fb3c4947 100644 --- a/dataikuapi/dss/modelevaluationstore.py +++ b/dataikuapi/dss/modelevaluationstore.py @@ -124,18 +124,18 @@ def list_model_evaluations(self): :returns: The list of the model evaluations :rtype: list of :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation` """ - items = self.client._perform_json("GET", "/projects/%s/modelevaluationstores/%s/runs/" % (self.project_key, self.mes_id)) - return [DSSModelEvaluation(self, item["ref"]["runId"]) for item in items] + items = self.client._perform_json("GET", "/projects/%s/modelevaluationstores/%s/evaluations/" % (self.project_key, self.mes_id)) + return [DSSModelEvaluation(self, item["ref"]["evaluationId"]) for item in items] - def get_model_evaluation(self, run_id): + def get_model_evaluation(self, evaluation_id): """ Get a handle to interact with a specific model evaluation - :param string run_id: the id of the desired model evaluation + :param string evaluation_id: the id of the desired model evaluation :returns: A :class:`dataikuapi.dss.modelevaluationstore.DSSModelEvaluation` model evaluation handle """ - return DSSModelEvaluation(self, run_id) + return DSSModelEvaluation(self, evaluation_id) def get_latest_model_evaluation(self): """ @@ -146,11 +146,11 @@ def get_latest_model_evaluation(self): if the store is not empty, else None """ - latest_run_id = self.client._perform_text( - "GET", "/projects/%s/modelevaluationstores/%s/latestRunId" % (self.project_key, self.mes_id)) - if not latest_run_id: + latest_evaluation_id = self.client._perform_text( + "GET", "/projects/%s/modelevaluationstores/%s/latestEvaluationId" % (self.project_key, self.mes_id)) + if not latest_evaluation_id: return None - return DSSModelEvaluation(self, latest_run_id) + return DSSModelEvaluation(self, latest_evaluation_id) def delete_model_evaluations(self, evaluations): """ @@ -159,13 +159,13 @@ def delete_model_evaluations(self, evaluations): obj = [] for evaluation in evaluations: if isinstance(evaluation, DSSModelEvaluation): - obj.append(evaluation.run_id) + obj.append(evaluation.evaluation_id) elif isinstance(evaluation, dict): - obj.append(evaluation['run_id']) + obj.append(evaluation['evaluation_id']) else: obj.append(evaluation) self.client._perform_json( - "DELETE", "/projects/%s/modelevaluationstores/%s/runs/" % (self.project_key, self.mes_id, self.run_id), body=obj) + "DELETE", "/projects/%s/modelevaluationstores/%s/evaluations/" % (self.project_key, self.mes_id), body=obj) def build(self, job_type="NON_RECURSIVE_FORCED_BUILD", wait=True, no_fail=False): """ @@ -263,11 +263,11 @@ class DSSModelEvaluation: Do not create this class directly, instead use :meth:`dataikuapi.dss.DSSModelEvaluationStore.get_model_evaluation` """ - def __init__(self, model_evaluation_store, run_id): + def __init__(self, model_evaluation_store, evaluation_id): self.model_evaluation_store = model_evaluation_store self.client = model_evaluation_store.client # unpack some fields - self.run_id = run_id + self.evaluation_id = evaluation_id self.project_key = model_evaluation_store.project_key self.mes_id = model_evaluation_store.mes_id @@ -276,23 +276,23 @@ def get_full_info(self): Retrieve the model evaluation with its performance data """ data = self.client._perform_json( - "GET", "/projects/%s/modelevaluationstores/%s/runs/%s" % (self.project_key, self.mes_id, self.run_id)) + "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s" % (self.project_key, self.mes_id, self.evaluation_id)) return DSSModelEvaluationFullInfo(self, data) def get_full_id(self): - return "ME-{}-{}-{}".format(self.project_key, self.mes_id, self.run_id) + return "ME-{}-{}-{}".format(self.project_key, self.mes_id, self.evaluation_id) def delete(self): """ Remove this model evaluation """ - obj = [self.run_id] + obj = [self.evaluation_id] self.client._perform_json( - "DELETE", "/projects/%s/modelevaluationstores/%s/runs/" % (self.project_key, self.mes_id), body=obj) + "DELETE", "/projects/%s/modelevaluationstores/%s/evaluations/" % (self.project_key, self.mes_id), body=obj) @property def full_id(self): - return "ME-%s-%s-%s"%(self.project_key, self.mes_id, self.run_id) + return "ME-%s-%s-%s"%(self.project_key, self.mes_id, self.evaluation_id) def compute_data_drift(self, reference=None, data_drift_params=None, wait=True): """ @@ -310,7 +310,7 @@ def compute_data_drift(self, reference=None, data_drift_params=None, wait=True): reference = reference.full_id future_response = self.client._perform_json( - "POST", "/projects/%s/modelevaluationstores/%s/runs/%s/computeDataDrift" % (self.project_key, self.mes_id, self.run_id), + "POST", "/projects/%s/modelevaluationstores/%s/evaluations/%s/computeDataDrift" % (self.project_key, self.mes_id, self.evaluation_id), body={ "referenceId": reference, "dataDriftParams": data_drift_params @@ -325,7 +325,7 @@ def get_metrics(self): :return: the metrics, as a JSON object """ return self.client._perform_json( - "GET", "/projects/%s/modelevaluationstores/%s/runs/%s/metrics" % (self.project_key, self.mes_id, self.run_id)) + "GET", "/projects/%s/modelevaluationstores/%s/evaluations/%s/metrics" % (self.project_key, self.mes_id, self.evaluation_id)) def get_sample_df(self): """ @@ -337,12 +337,12 @@ def get_sample_df(self): buf = BytesIO() with self.client._perform_raw( "GET", - "/projects/%s/modelevaluationstores/%s/runs/%s/sample" % (self.project_key, self.mes_id, self.run_id) + "/projects/%s/modelevaluationstores/%s/evaluations/%s/sample" % (self.project_key, self.mes_id, self.evaluation_id) ).raw as f: buf.write(f.read()) schema_txt = self.client._perform_raw( "GET", - "/projects/%s/modelevaluationstores/%s/runs/%s/schema" % (self.project_key, self.mes_id, self.run_id) + "/projects/%s/modelevaluationstores/%s/evaluations/%s/schema" % (self.project_key, self.mes_id, self.evaluation_id) ).text schema = json.loads(schema_txt) import pandas as pd