From f2e4614abc6808eb1921e1d17774efed1f60d043 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 16 Apr 2021 12:06:57 +0200 Subject: [PATCH 01/38] Added FMClient and get_cloud_credentials --- dataikuapi/__init__.py | 3 +- dataikuapi/fm/tenant.py | 18 +++++++++ dataikuapi/fmclient.py | 81 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 dataikuapi/fm/tenant.py create mode 100644 dataikuapi/fmclient.py diff --git a/dataikuapi/__init__.py b/dataikuapi/__init__.py index 89b5119a..569b1017 100644 --- a/dataikuapi/__init__.py +++ b/dataikuapi/__init__.py @@ -1,8 +1,9 @@ from .dssclient import DSSClient +from .fmclient import FMClient from .apinode_client import APINodeClient from .apinode_admin_client import APINodeAdminClient from .dss.recipe import GroupingRecipeCreator, JoinRecipeCreator, StackRecipeCreator, WindowRecipeCreator, SyncRecipeCreator, SamplingRecipeCreator, SQLQueryRecipeCreator, CodeRecipeCreator, SplitRecipeCreator, SortRecipeCreator, TopNRecipeCreator, DistinctRecipeCreator, DownloadRecipeCreator, PredictionScoringRecipeCreator, ClusteringScoringRecipeCreator -from .dss.admin import DSSUserImpersonationRule, DSSGroupImpersonationRule \ No newline at end of file +from .dss.admin import DSSUserImpersonationRule, DSSGroupImpersonationRule diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py new file mode 100644 index 00000000..4fe5a194 --- /dev/null +++ b/dataikuapi/fm/tenant.py @@ -0,0 +1,18 @@ +class FMCloudCredentials(object): + """ + A Tenant Cloud Credentials in the FM instance + """ + def __init__(self, client, tenant_id, cloud_credentials): + self.client = client + self.tenant_id = tenant_id + self.cloud_credentials = cloud_credentials + + def set_cmk_key(self, cmk_key_id): + self.cloud_credentials['awsCMKId'] = cmk_key_id + + def save(self): + """Saves back the settings to the project""" + + self.client._perform_empty("PUT", "/tenants/%s/cloud-credentials" % (self.tenant_id), + body = self.cloud_credentials) + diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py new file mode 100644 index 00000000..b3275397 --- /dev/null +++ b/dataikuapi/fmclient.py @@ -0,0 +1,81 @@ +import json +from requests import Session +from requests import exceptions +from requests.auth import HTTPBasicAuth + +import os.path as osp +from .utils import DataikuException + +from .fm.tenant import FMCloudCredentials +from .fm.tenant import FMCloudTags + +class FMClient(object): + """Entry point for the FM API client""" + + def __init__(self, host, api_key_id, api_key_secret, extra_headers = None): + """ + Instantiate a new FM API client on the given host with the given API key. + + API keys can be managed in FM on the project page or in the global settings. + + The API key will define which operations are allowed for the client. + """ + self.api_key_id = api_key_id + self.api_key_secret = api_key_secret + self.host = host + self._session = Session() + + if self.api_key_id is not None and self.api_key_secret is not None: + self._session.auth = HTTPBasicAuth(self.api_key_id, self.api_key_secret) + else: + raise ValueError("API Key ID and API Key secret are required") + + if extra_headers is not None: + self._session.headers.update(extra_headers) + + ######################################################## + # Tenant + ######################################################## + + def get_cloud_credentials(self, tenant_id): + """ + Get Tenant's Cloud Credential + + :param string tenant_id + + :return: tenant's cloud credentials + :rtype: :class:`dataikuapi.fm.tenant.FMCloudCredentials` + """ + creds = self._perform_json("GET", "/tenants/%s/cloud-credentials" % tenant_id) + return FMCloudCredentials(self, tenant_id, creds) + + ######################################################## + # Internal Request handling + ######################################################## + + def _perform_http(self, method, path, params=None, body=None, stream=False, files=None, raw_body=None): + if body is not None: + body = json.dumps(body) + if raw_body is not None: + body = raw_body + + try: + http_res = self._session.request( + method, "%s/api/public%s" % (self.host, path), + params=params, data=body, + files = files, + stream = stream) + http_res.raise_for_status() + return http_res + except exceptions.HTTPError: + try: + ex = http_res.json() + except ValueError: + ex = {"message": http_res.text} + raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) + + def _perform_empty(self, method, path, params=None, body=None, files = None, raw_body=None): + self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body) + + def _perform_json(self, method, path, params=None, body=None,files=None, raw_body=None): + return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).json() From ed67e05abad064fbf2a1cd6d78392ddc9805d2b7 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 27 May 2021 18:17:24 +0200 Subject: [PATCH 02/38] Set tenant_id in FMClient --- dataikuapi/fm/tenant.py | 5 ++--- dataikuapi/fmclient.py | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index 4fe5a194..95f9aefa 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -2,9 +2,8 @@ class FMCloudCredentials(object): """ A Tenant Cloud Credentials in the FM instance """ - def __init__(self, client, tenant_id, cloud_credentials): + def __init__(self, client, cloud_credentials): self.client = client - self.tenant_id = tenant_id self.cloud_credentials = cloud_credentials def set_cmk_key(self, cmk_key_id): @@ -13,6 +12,6 @@ def set_cmk_key(self, cmk_key_id): def save(self): """Saves back the settings to the project""" - self.client._perform_empty("PUT", "/tenants/%s/cloud-credentials" % (self.tenant_id), + self.client._perform_tenant_empty("PUT", "/cloud-credentials", body = self.cloud_credentials) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index b3275397..41ce010a 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -7,12 +7,11 @@ from .utils import DataikuException from .fm.tenant import FMCloudCredentials -from .fm.tenant import FMCloudTags class FMClient(object): """Entry point for the FM API client""" - def __init__(self, host, api_key_id, api_key_secret, extra_headers = None): + def __init__(self, host, api_key_id, api_key_secret, tenant_id, extra_headers = None): """ Instantiate a new FM API client on the given host with the given API key. @@ -23,6 +22,7 @@ def __init__(self, host, api_key_id, api_key_secret, extra_headers = None): self.api_key_id = api_key_id self.api_key_secret = api_key_secret self.host = host + self.__tenant_id = tenant_id self._session = Session() if self.api_key_id is not None and self.api_key_secret is not None: @@ -37,17 +37,15 @@ def __init__(self, host, api_key_id, api_key_secret, extra_headers = None): # Tenant ######################################################## - def get_cloud_credentials(self, tenant_id): + def get_cloud_credentials(self): """ - Get Tenant's Cloud Credential + Get Cloud Credential - :param string tenant_id - - :return: tenant's cloud credentials + :return: Cloud credentials :rtype: :class:`dataikuapi.fm.tenant.FMCloudCredentials` """ - creds = self._perform_json("GET", "/tenants/%s/cloud-credentials" % tenant_id) - return FMCloudCredentials(self, tenant_id, creds) + creds = self._perform_tenant_json("GET", "/cloud-credentials") + return FMCloudCredentials(self, creds) ######################################################## # Internal Request handling @@ -79,3 +77,9 @@ def _perform_empty(self, method, path, params=None, body=None, files = None, raw def _perform_json(self, method, path, params=None, body=None,files=None, raw_body=None): return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).json() + + def _perform_tenant_json(self, method, path, params=None, body=None,files=None, raw_body=None): + return self._perform_json(method, "/tenants/%s%s" % ( self.__tenant_id, path ), params=params, body=body, files=files, raw_body=raw_body) + + def _perform_tenant_empty(self, method, path, params=None, body=None, files = None, raw_body=None): + self._perform_empty(method, "/tenants/%s%s" % ( self.__tenant_id, path ), params=params, body=body, files=files, raw_body=raw_body) From 69a1cdb7d6d40feaba536a4aaedf3ad3c8cf3430 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 27 May 2021 18:18:31 +0200 Subject: [PATCH 03/38] Added FMVirtualNetwork --- dataikuapi/fm/virtualnetworks.py | 5 +++++ dataikuapi/fmclient.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 dataikuapi/fm/virtualnetworks.py diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py new file mode 100644 index 00000000..455bd9b2 --- /dev/null +++ b/dataikuapi/fm/virtualnetworks.py @@ -0,0 +1,5 @@ +class FMVirtualNetwork(object): + def __init__(self, client, virtual_network_settings): + self.client = client + self.settings = virtual_network_settings + self.id = self.settings['id'] diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 41ce010a..a555b6aa 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -7,6 +7,7 @@ from .utils import DataikuException from .fm.tenant import FMCloudCredentials +from .fm.virtualnetworks import FMVirtualNetwork class FMClient(object): """Entry point for the FM API client""" @@ -47,6 +48,34 @@ def get_cloud_credentials(self): creds = self._perform_tenant_json("GET", "/cloud-credentials") return FMCloudCredentials(self, creds) + + ######################################################## + # VirtualNetwork + ######################################################## + + def get_virtual_networks(self): + """ + Get Virtual Networks + + :return: list of virtual networks + :rtype: list of :class:`dataikuapi.fm.tenant.FMVirtualNetwork` + """ + vns = self._perform_tenant_json("GET", "/virtual-networks") + return [ FMVirtualNetwork(self, x) for x in vns] + + def get_virtual_network(self, virtual_network_id): + """ + Get a Virtual Network + + :param str virtual_network_id + + :return: requested virtual network + :rtype: :class:`dataikuapi.fm.tenant.FMVirtualNetwork` + """ + template = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) + return FMVirtualNetwork(self, template) + + ######################################################## # Internal Request handling ######################################################## From 7518b291734c4b40e7c6ae15e8b43b22e491348a Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 27 May 2021 18:23:05 +0200 Subject: [PATCH 04/38] Added FMInstanceSettingsTemplate --- dataikuapi/fm/instancesettingstemplates.py | 6 +++++ dataikuapi/fmclient.py | 31 ++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 dataikuapi/fm/instancesettingstemplates.py diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py new file mode 100644 index 00000000..d9d640ee --- /dev/null +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -0,0 +1,6 @@ +class FMInstanceSettingsTemplate(object): + def __init__(self, client, ist_data): + self.client = client + self.id = ist_data['id'] + self.ist_data = ist_data + diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index a555b6aa..16a7c21c 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -8,6 +8,7 @@ from .fm.tenant import FMCloudCredentials from .fm.virtualnetworks import FMVirtualNetwork +from .fm.instancesettingstemplates import FMInstanceSettingsTemplate class FMClient(object): """Entry point for the FM API client""" @@ -34,6 +35,7 @@ def __init__(self, host, api_key_id, api_key_secret, tenant_id, extra_headers = if extra_headers is not None: self._session.headers.update(extra_headers) + ######################################################## # Tenant ######################################################## @@ -76,6 +78,35 @@ def get_virtual_network(self, virtual_network_id): return FMVirtualNetwork(self, template) + ######################################################## + # Instance settings template + ######################################################## + + def get_instance_templates(self): + """ + Get Instance Settings Templates + + :return: list of instance settings template + :rtype: list of :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` + """ + templates = self._perform_tenant_json("GET", "/instance-settings-templates") + return [ FMInstanceSettingsTemplate(self, x) for x in templates] + + def get_instance_template(self, template_id): + """ + Get an Instance + + :param str template_id + + :return: requested instance settings template + :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + """ + template = self._perform_tenant_json("GET", "/instance-settings-templates/%s" % template_id) + return FMInstanceSettingsTemplate(self, template) + + + + ######################################################## # Internal Request handling ######################################################## From ac1ed01cd59e0158946d712f3829a5b08bf1e785 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 27 May 2021 18:24:04 +0200 Subject: [PATCH 05/38] Added FMInstance --- dataikuapi/fm/instances.py | 11 +++++++ dataikuapi/fmclient.py | 62 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 dataikuapi/fm/instances.py diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py new file mode 100644 index 00000000..9d272a04 --- /dev/null +++ b/dataikuapi/fm/instances.py @@ -0,0 +1,11 @@ +from enum import Enum + +class FMInstance(object): + def __init__(self, client): + self.client = client + +class FMInstanceEncryptionMode(Enum): + NONE = "NONE" + DEFAULT_KEY = "DEFAULT_KEY" + TENANT = "TENANT" + CUSTOM = "CUSTOM" diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 16a7c21c..7a756151 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -8,6 +8,7 @@ from .fm.tenant import FMCloudCredentials from .fm.virtualnetworks import FMVirtualNetwork +from .fm.instances import FMInstance, FMInstanceEncryptionMode from .fm.instancesettingstemplates import FMInstanceSettingsTemplate class FMClient(object): @@ -105,6 +106,67 @@ def get_instance_template(self, template_id): return FMInstanceSettingsTemplate(self, template) + ######################################################## + # Instance + ######################################################## + + def get_instances(self): + """ + Get Instances + + :return: list of instances + :rtype: list of :class:`dataikuapi.fm.tenant.FMInstance` + """ + instances = self._perform_tenant_json("GET", "/instances") + return [ FMInstance(self, **x) for x in instances] + + def get_instance(self, instance_id): + """ + Get an Instance + + :param str instance_id + + :return: Instance + :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + """ + instance = self._perform_json("GET", "/instances/%s" % instance_id) + return FMInstance(self, **instance) + + def create_instance(self, instance_settings_template, virtual_network, label, + dss_node_type="design", image_id=None, + cloud_instance_type=None, data_volume_type=None, data_volume_size=None, + data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=FMInstanceEncryptionMode.NONE, + data_volume_encryption_key=None, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None, + cloud_tags=None, fm_tags=None): + """ + Create an Instance + + :param str instance_id + + :return: Instance + :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + """ + data = { + "virtualNetworkId": virtual_network.id, + "instanceSettingsTemplateId": instance_settings_template.id, + "label": label, + "dssNodeType": dss_node_type, + "imageId": image_id, + "cloudInstanceType": cloud_instance_type, + "dataVolumeType": data_volume_type, + "dataVolumeSizeGB": data_volume_size, + "dataVolumeSizeMaxGB": data_volume_size_max, + "dataVolumeIOPS": data_volume_IOPS, + "dataVolumeEncryption": str(data_volume_encryption), + "dataVolumeEncryptionKey": data_volume_encryption_key, + "awsRootVolumeSizeGB": aws_root_volume_size, + "awsRootVolumeType": aws_root_volume_type, + "awsRootVolumeIOPS": aws_root_volume_IOPS, + "cloudTags": cloud_tags, + "fmTags": fm_tags + } + instance = self._perform_tenant_json("POST", "/instances", body=data) + return FMInstance(self, instance) ######################################################## From a43f3f0765ec14b5f08782cf681d1e66a0742d3a Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 1 Jun 2021 17:19:14 +0200 Subject: [PATCH 06/38] fm public api: Add instance actions --- dataikuapi/fm/instances.py | 18 +++++++++++++++++- dataikuapi/fmclient.py | 6 +++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 9d272a04..69b9ee55 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,8 +1,24 @@ from enum import Enum class FMInstance(object): - def __init__(self, client): + def __init__(self, client, instance_data): self.client = client + self.instance_data = instance_data + print(instance_data) + self.id = instance_data['id'] + + def reprovision(self): + status = self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) + print(status) + + def deprovision(self): + status = self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) + print(status) + + def status(self): + status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) + return status + class FMInstanceEncryptionMode(Enum): NONE = "NONE" diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 7a756151..ed6b081e 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -129,8 +129,8 @@ def get_instance(self, instance_id): :return: Instance :rtype: :class:`dataikuapi.fm.tenant.FMInstance` """ - instance = self._perform_json("GET", "/instances/%s" % instance_id) - return FMInstance(self, **instance) + instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) + return FMInstance(self, instance) def create_instance(self, instance_settings_template, virtual_network, label, dss_node_type="design", image_id=None, @@ -157,7 +157,7 @@ def create_instance(self, instance_settings_template, virtual_network, label, "dataVolumeSizeGB": data_volume_size, "dataVolumeSizeMaxGB": data_volume_size_max, "dataVolumeIOPS": data_volume_IOPS, - "dataVolumeEncryption": str(data_volume_encryption), + "dataVolumeEncryption": data_volume_encryption.value, "dataVolumeEncryptionKey": data_volume_encryption_key, "awsRootVolumeSizeGB": aws_root_volume_size, "awsRootVolumeType": aws_root_volume_type, From 66ec5f38e595b28377f0068b6dec04901f4c12ae Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 2 Jun 2021 09:55:48 +0200 Subject: [PATCH 07/38] fm: Add FMInstanceStatus --- dataikuapi/fm/instances.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 69b9ee55..c63afcfe 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -15,9 +15,9 @@ def deprovision(self): status = self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) print(status) - def status(self): + def get_status(self): status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) - return status + return FMInstanceStatus(status) class FMInstanceEncryptionMode(Enum): @@ -25,3 +25,12 @@ class FMInstanceEncryptionMode(Enum): DEFAULT_KEY = "DEFAULT_KEY" TENANT = "TENANT" CUSTOM = "CUSTOM" + + +class FMInstanceStatus(dict): + """A class holding read-only information about an Instance. + This class should not be created directly. Instead, use :meth:`FMInstance.get_info` + """ + def __init__(self, data): + """Do not call this directly, use :meth:`FMInstance.get_status`""" + super(FMInstanceStatus, self).__init__(data) From cff9ea9379f8292746c817b81565676520b15637 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 2 Jun 2021 15:37:55 +0200 Subject: [PATCH 08/38] fm: Save and restart_dss --- dataikuapi/fm/instances.py | 32 +++++++++++++++++++++++++++----- dataikuapi/fmclient.py | 12 ++++++------ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index c63afcfe..af228243 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,19 +1,41 @@ from enum import Enum class FMInstance(object): + """ + A handle to interact with a DSS instance. + Do not create this directly, use :meth:`FMClient.get_instance` or :meth: `FMClient.create_instance` + """ def __init__(self, client, instance_data): self.client = client self.instance_data = instance_data - print(instance_data) self.id = instance_data['id'] def reprovision(self): - status = self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) - print(status) + """ + Reprovision the DSS physical instance + """ + self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) def deprovision(self): - status = self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) - print(status) + """ + Deprovision the DSS physical instance + """ + self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) + + def restart_dss(self): + """ + Restart the DSS running on the physical instance + """ + self.client._perform_tenant_json("GET", "/instances/%s/actions/restart-dss" % self.id) + + def save(self): + """ + Update the DSS Instance. + """ + self.client._perform_tenant_empty("PUT", "/instances/%s" % self.id, body=self.instance_data) + self.instance_data = self.client._perform_tenant_json("GET", "/instances/%s" % self.id) + + def get_status(self): status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index ed6b081e..f7981599 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -58,7 +58,7 @@ def get_cloud_credentials(self): def get_virtual_networks(self): """ - Get Virtual Networks + List all Virtual Networks :return: list of virtual networks :rtype: list of :class:`dataikuapi.fm.tenant.FMVirtualNetwork` @@ -85,7 +85,7 @@ def get_virtual_network(self, virtual_network_id): def get_instance_templates(self): """ - Get Instance Settings Templates + List all Instance Settings Templates :return: list of instance settings template :rtype: list of :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` @@ -95,7 +95,7 @@ def get_instance_templates(self): def get_instance_template(self, template_id): """ - Get an Instance + Get an Instance Template :param str template_id @@ -112,7 +112,7 @@ def get_instance_template(self, template_id): def get_instances(self): """ - Get Instances + List all DSS Instances :return: list of instances :rtype: list of :class:`dataikuapi.fm.tenant.FMInstance` @@ -122,7 +122,7 @@ def get_instances(self): def get_instance(self, instance_id): """ - Get an Instance + Get a DSS Instance :param str instance_id @@ -139,7 +139,7 @@ def create_instance(self, instance_settings_template, virtual_network, label, data_volume_encryption_key=None, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None, cloud_tags=None, fm_tags=None): """ - Create an Instance + Create a DSS Instance :param str instance_id From 22c6e5ae5578510de072b95d7d937e9b39c2a430 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 2 Jun 2021 21:47:46 +0200 Subject: [PATCH 09/38] fm: create instance template --- dataikuapi/fm/instances.py | 11 +-- dataikuapi/fm/instancesettingstemplates.py | 41 +++++++++++ dataikuapi/fmclient.py | 84 ++++++++++++++++++++-- 3 files changed, 126 insertions(+), 10 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index af228243..ab4ba2b0 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -12,13 +12,13 @@ def __init__(self, client, instance_data): def reprovision(self): """ - Reprovision the DSS physical instance + Reprovision the physical DSS instance """ self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) def deprovision(self): """ - Deprovision the DSS physical instance + Deprovision the physical DSS instance """ self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) @@ -30,14 +30,15 @@ def restart_dss(self): def save(self): """ - Update the DSS Instance. + Update the Instance. """ self.client._perform_tenant_empty("PUT", "/instances/%s" % self.id, body=self.instance_data) self.instance_data = self.client._perform_tenant_json("GET", "/instances/%s" % self.id) - - def get_status(self): + """ + Get the physical DSS instance's status + """ status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) return FMInstanceStatus(status) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index d9d640ee..6dd0782e 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -1,6 +1,47 @@ +from enum import Enum +import json + class FMInstanceSettingsTemplate(object): def __init__(self, client, ist_data): self.client = client self.id = ist_data['id'] self.ist_data = ist_data + def save(self): + """ + Update the Instance Settings Template. + """ + self.client._perform_tenant_empty("PUT", "/instance-settings-templates/%s" % self.id, body=self.ist_data) + self.ist_data = self.client._perform_tenant_json("GET", "/instance-settings-templates/%s" % self.id) + +class FMSetupAction(dict): + def __init__(self, setupActionType, params=None): + """ + param: object setupActionType: the type (`:class: FMSetupActionType`) of the SetupAction + param: str params: the parameters of the SetupAction in a json-encoded string + """ + data = { + "type": setupActionType.value, + } + if params: + data['params'] = params + + super(FMSetupAction, self).__init__(data) + +class FMSetupActionAddAuthorizedKey(FMSetupAction): + def __init__(self, ssh_key): + super(FMSetupActionAddAuthorizedKey, self).__init__(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key }) + +class FMSetupActionType(Enum): + RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" + INSTALL_SYSTEM_PACKAGES="INSTALL_SYSTEM_PACKAGES" + INSTALL_DSS_PLUGINS_FROM_STORE="INSTALL_DSS_PLUGINS_FROM_STORE" + SETUP_K8S_AND_SPARK="SETUP_K8S_AND_SPARK" + SETUP_RUNTIME_DATABASE="SETUP_RUNTIME_DATABASE" + SETUP_MUS_AUTOCREATE="SETUP_MUS_AUTOCREATE" + SETUP_UI_CUSTOMIZATION="SETUP_UI_CUSTOMIZATION" + SETUP_MEMORY_SETTINGS="SETUP_MEMORY_SETTINGS" + SETUP_GRAPHICS_EXPORT="SETUP_GRAPHICS_EXPORT" + ADD_AUTHORIZED_KEY="ADD_AUTHORIZED_KEY" + INSTALL_JDBC_DRIVER="INSTALL_JDBC_DRIVER" + SETUP_ADVANCED_SECURITY="SETUP_ADVANCED_SECURITY" diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index f7981599..247f0a66 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -100,12 +100,69 @@ def get_instance_template(self, template_id): :param str template_id :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + :rtype: :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` """ template = self._perform_tenant_json("GET", "/instance-settings-templates/%s" % template_id) return FMInstanceSettingsTemplate(self, template) + def create_instance_template(self, label, + setupActions=None, license=None, + awsKeyPairName=None, startupInstanceProfileArn=None, runtimeInstanceProfileArn=None, + restrictAwsMetadataServerAccess=True, dataikuAwsAPIAccessMode="NONE", dataikuAwsKeypairStorageMode=None, + dataikuAwsAccessKeyId=None, dataikuAwsSecretAccessKey=None, + dataikuAwsSecretAccessKeyAwsSecretName=None, awsSecretsManagerRegion=None, + azureSshKey=None, startupManagedIdentity=None, runtimeManagedIdentity=None): + """ + Create an Instance Template + + :param str label: The label of the Instance Settings Template + + :param list setupActions: Optional, a list of `:class: FMSetupAction` to be played on an instance + :param str license: Optional, overrides the license set in Cloud Setup + + :param str awsKeyPairName: Optional, AWS Only, the name of an AWS key pair to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. + :param str startupInstanceProfileArn: Optional, AWS Only, the ARN of the Instance profile assigned to the DSS instance at startup time + :param str runtimeInstanceProfileArn: Optional, AWS Only, the ARN of the Instance profile assigned to the DSS instance at runtime + :param boolean restrictAwsMetadataServerAccess: Optional, AWS Only, If true, restrict the access to the metadata server access. Defaults to true + :param str dataikuAwsAPIAccessMode: Optional, AWS Only, the access mode DSS is using to connect to the AWS API. If "NONE" DSS will use the Instance Profile, If "KEYPAIR", an AWS access key id and secret will be securely given to the dataiku account. + :param str dataikuAwsKeypairStorageMode: Optional, AWS Only, the storage mode of the AWS api key. Accepts "NONE", "INLINE_ENCRYPTED" or "AWS_SECRETS_MANAGER" + :param str dataikuAwsAccessKeyId: Optional, AWS Only, AWS Access Key ID. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" + :param str dataikuAwsSecretAccessKey: Optional, AWS Only, AWS Access Key Secret. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" and dataikuAwsKeypairStorageMode is "INLINE_ENCRYPTED" + :param str dataikuAwsSecretAccessKeyAwsSecretName: Optional, AWS Only, ASM secret name. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" and dataikuAwsKeypairStorageMode is "AWS_SECRET_MANAGER" + :param str awsSecretsManagerRegion: Optional, AWS Only + + :param str azureSshKey: Optional, Azure Only, the ssh public key to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. + :param str startupManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at startup time + :param str runtimeManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at runtime + + :return: requested instance settings template + :rtype: :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` + """ + + data = { + "label": label, + "setupActions": setupActions, + "license": license, + "awsKeyPairName": awsKeyPairName, + "startupInstanceProfileArn": startupInstanceProfileArn, + "runtimeInstanceProfileArn": runtimeInstanceProfileArn, + "restrictAwsMetadataServerAccess": restrictAwsMetadataServerAccess, + "dataikuAwsAPIAccessMode": "dataikuAwsAPIAccessMode", + "dataikuAwsKeypairStorageMode": dataikuAwsKeypairStorageMode, + "dataikuAwsAccessKeyId": dataikuAwsAccessKeyId, + "dataikuAwsSecretAccessKey": dataikuAwsSecretAccessKey, + "dataikuAwsSecretAccessKeyAwsSecretName": dataikuAwsSecretAccessKeyAwsSecretName, + "awsSecretsManagerRegion": awsSecretsManagerRegion, + "azureSshKey": azureSshKey, + "startupManagedIdentity": startupManagedIdentity, + "runtimeManagedIdentity": runtimeManagedIdentity + } + + template = self._perform_tenant_json("POST", "/instance-settings-templates", body=data) + return FMInstanceSettingsTemplate(self, template) + + ######################################################## # Instance ######################################################## @@ -132,8 +189,8 @@ def get_instance(self, instance_id): instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return FMInstance(self, instance) - def create_instance(self, instance_settings_template, virtual_network, label, - dss_node_type="design", image_id=None, + def create_instance(self, instance_settings_template, virtual_network, label, image_id, + dss_node_type="design", cloud_instance_type=None, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=FMInstanceEncryptionMode.NONE, data_volume_encryption_key=None, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None, @@ -141,7 +198,24 @@ def create_instance(self, instance_settings_template, virtual_network, label, """ Create a DSS Instance - :param str instance_id + :param str instance_settings_template: The instance settings template id this instance should be based on + :param str virtual_network: The virtual network where the instance should be spawned + :param str label: The label of the instance + :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) + + :param str dss_node_type: Optional , the type of the dss node to create. Defaults to "design" + :param str cloud_instance_type: Optional, Machine type + :param str data_volume_type: Optional, Data volume type + :param int data_volume_size: Optional, Data volume initial size + :param int data_volume_size_max: Optional, Data volume maximum size + :param int data_volume_IOPS: Optional, Data volume IOPS + :param object data_volume_encryption: Optional, a :class:`FMInstanceEncryptionMode` setting the encryption mode of the data volume + :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM + :param int aws_root_volume_size: Optional, the root volume size + :param str aws_root_volume_type: Optional, the root volume type + :param int aws_root_volume_IOPS: Optional, the root volume IOPS + :param dict cloud_tags: Optional, a key value dictionary of tags to be applied on the cloud resources + :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager :return: Instance :rtype: :class:`dataikuapi.fm.tenant.FMInstance` @@ -178,7 +252,7 @@ def _perform_http(self, method, path, params=None, body=None, stream=False, file body = json.dumps(body) if raw_body is not None: body = raw_body - + print(body) try: http_res = self._session.request( method, "%s/api/public%s" % (self.host, path), From 96fddb3572c2c49524b10968606ac190d6e24c6f Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 3 Jun 2021 12:49:30 +0200 Subject: [PATCH 10/38] FM: future --- dataikuapi/fm/future.py | 85 ++++++++++++++++++++++++++++++++++++++ dataikuapi/fm/instances.py | 19 +++++++-- dataikuapi/fmclient.py | 18 ++++---- 3 files changed, 110 insertions(+), 12 deletions(-) create mode 100644 dataikuapi/fm/future.py diff --git a/dataikuapi/fm/future.py b/dataikuapi/fm/future.py new file mode 100644 index 00000000..60d31b57 --- /dev/null +++ b/dataikuapi/fm/future.py @@ -0,0 +1,85 @@ +import sys, time + +class FMFuture(object): + """ + A future on the DSS instance + """ + def __init__(self, client, job_id, state=None, result_wrapper=lambda result: result): + self.client = client + self.job_id = job_id + self.state = state + self.state_is_peek = True + self.result_wrapper = result_wrapper + + @staticmethod + def from_resp(client, resp,result_wrapper=lambda result: result): + """Creates a DSSFuture from a parsed JSON response""" + return FMFuture(client, resp.get('jobId', None), state=resp, result_wrapper=result_wrapper) + + @classmethod + def get_result_wait_if_needed(cls, client, ret): + if 'jobId' in ret: + future = FMFuture(client, ret["jobId"], ret) + future.wait_for_result() + return future.get_result() + else: + return ret['result'] + + def abort(self): + """ + Abort the future + """ + return self.client._perform_tenant_empty("DELETE", "/futures/%s" % self.job_id) + + def get_state(self): + """ + Get the status of the future, and its result if it's ready + """ + self.state = self.client._perform_tenant_json( + "GET", "/futures/%s" % self.job_id, params={'peek' : False}) + self.state_is_peek = False + return self.state + + def peek_state(self): + """ + Get the status of the future, and its result if it's ready + """ + self.state = self.client._perform_tenant_json( + "GET", "/futures/%s" % self.job_id, params={'peek' : True}) + self.state_is_peek = True + return self.state + + def get_result(self): + """ + Get the future result if it's ready, raises an Exception otherwise + """ + if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: + self.get_state() + if self.state.get('hasResult', False): + return self.result_wrapper(self.state.get('result', None)) + else: + raise Exception("Result not ready") + + def has_result(self): + """ + Checks whether the future has a result ready + """ + if self.state is None or not self.state.get('hasResult', False): + self.get_state() + return self.state.get('hasResult', False) + + def wait_for_result(self): + """ + Wait and get the future result + """ + if self.state.get('hasResult', False): + return self.result_wrapper(self.state.get('result', None)) + if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: + self.get_state() + while not self.state.get('hasResult', False): + time.sleep(5) + self.get_state() + if self.state.get('hasResult', False): + return self.result_wrapper(self.state.get('result', None)) + else: + raise Exception("No result") diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index ab4ba2b0..41d41374 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,4 +1,5 @@ from enum import Enum +from .future import FMFuture class FMInstance(object): """ @@ -13,20 +14,32 @@ def __init__(self, client, instance_data): def reprovision(self): """ Reprovision the physical DSS instance + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the reprovision process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) + future = self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) + return FMFuture.from_resp(self.client, future) def deprovision(self): """ Deprovision the physical DSS instance + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deprovision process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) + future = self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) + return FMFuture.from_resp(self.client, future) def restart_dss(self): """ Restart the DSS running on the physical instance + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the restart process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - self.client._perform_tenant_json("GET", "/instances/%s/actions/restart-dss" % self.id) + future = self.client._perform_tenant_json("GET", "/instances/%s/actions/restart-dss" % self.id) + return FMFuture.from_resp(self.client, future) def save(self): """ diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 247f0a66..410c9794 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -61,7 +61,7 @@ def get_virtual_networks(self): List all Virtual Networks :return: list of virtual networks - :rtype: list of :class:`dataikuapi.fm.tenant.FMVirtualNetwork` + :rtype: list of :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ vns = self._perform_tenant_json("GET", "/virtual-networks") return [ FMVirtualNetwork(self, x) for x in vns] @@ -73,7 +73,7 @@ def get_virtual_network(self, virtual_network_id): :param str virtual_network_id :return: requested virtual network - :rtype: :class:`dataikuapi.fm.tenant.FMVirtualNetwork` + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ template = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) return FMVirtualNetwork(self, template) @@ -100,7 +100,7 @@ def get_instance_template(self, template_id): :param str template_id :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ template = self._perform_tenant_json("GET", "/instance-settings-templates/%s" % template_id) return FMInstanceSettingsTemplate(self, template) @@ -118,7 +118,7 @@ def create_instance_template(self, label, :param str label: The label of the Instance Settings Template - :param list setupActions: Optional, a list of `:class: FMSetupAction` to be played on an instance + :param list setupActions: Optional, a list of :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` to be played on an instance :param str license: Optional, overrides the license set in Cloud Setup :param str awsKeyPairName: Optional, AWS Only, the name of an AWS key pair to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. @@ -137,7 +137,7 @@ def create_instance_template(self, label, :param str runtimeManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at runtime :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ data = { @@ -172,7 +172,7 @@ def get_instances(self): List all DSS Instances :return: list of instances - :rtype: list of :class:`dataikuapi.fm.tenant.FMInstance` + :rtype: list of :class:`dataikuapi.fm.instances.FMInstance` """ instances = self._perform_tenant_json("GET", "/instances") return [ FMInstance(self, **x) for x in instances] @@ -184,7 +184,7 @@ def get_instance(self, instance_id): :param str instance_id :return: Instance - :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + :rtype: :class:`dataikuapi.fm.instances.FMInstance` """ instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return FMInstance(self, instance) @@ -209,7 +209,7 @@ def create_instance(self, instance_settings_template, virtual_network, label, im :param int data_volume_size: Optional, Data volume initial size :param int data_volume_size_max: Optional, Data volume maximum size :param int data_volume_IOPS: Optional, Data volume IOPS - :param object data_volume_encryption: Optional, a :class:`FMInstanceEncryptionMode` setting the encryption mode of the data volume + :param object data_volume_encryption: Optional, a :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode` setting the encryption mode of the data volume :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM :param int aws_root_volume_size: Optional, the root volume size :param str aws_root_volume_type: Optional, the root volume type @@ -218,7 +218,7 @@ def create_instance(self, instance_settings_template, virtual_network, label, im :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager :return: Instance - :rtype: :class:`dataikuapi.fm.tenant.FMInstance` + :rtype: :class:`dataikuapi.fm.instances.FMInstance` """ data = { "virtualNetworkId": virtual_network.id, From 32b2725842dc31ed0827a087cfae9a1be578d577 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 3 Jun 2021 13:27:02 +0200 Subject: [PATCH 11/38] fm: delete instance --- dataikuapi/fm/instances.py | 10 ++++++++++ dataikuapi/fmclient.py | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 41d41374..915ae874 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -55,6 +55,16 @@ def get_status(self): status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) return FMInstanceStatus(status) + def delete(self): + """ + Delete the DSS instance + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` + """ + future = self.client._perform_tenant_json("GET", "/instances/%s/actions/delete" % self.id) + return FMFuture.from_resp(self.client, future) + class FMInstanceEncryptionMode(Enum): NONE = "NONE" diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 410c9794..8a3af799 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -252,7 +252,6 @@ def _perform_http(self, method, path, params=None, body=None, stream=False, file body = json.dumps(body) if raw_body is not None: body = raw_body - print(body) try: http_res = self._session.request( method, "%s/api/public%s" % (self.host, path), From e4e00b597064b3dd3b3101f96f9fb166874a9b1d Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 3 Jun 2021 13:47:45 +0200 Subject: [PATCH 12/38] fm: Add delete instance settings template --- dataikuapi/fm/instancesettingstemplates.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 6dd0782e..3d2439cb 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -1,5 +1,6 @@ from enum import Enum import json +from dataikuapi.fm.future import FMFuture class FMInstanceSettingsTemplate(object): def __init__(self, client, ist_data): @@ -14,6 +15,16 @@ def save(self): self.client._perform_tenant_empty("PUT", "/instance-settings-templates/%s" % self.id, body=self.ist_data) self.ist_data = self.client._perform_tenant_json("GET", "/instance-settings-templates/%s" % self.id) + def delete(self): + """ + Delete the DSS Instance Settings Template. + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` + """ + future = self.client._perform_tenant_json("DELETE", "/instance-settings-templates/%s" % self.id) + return FMFuture.from_resp(self.client, future) + class FMSetupAction(dict): def __init__(self, setupActionType, params=None): """ From 3da4edd12eae4f0452835f61e66342d24a6d877c Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 3 Jun 2021 16:30:01 +0200 Subject: [PATCH 13/38] fm: Virtual network management --- dataikuapi/fm/virtualnetworks.py | 124 ++++++++++++++++++++++++++++++- dataikuapi/fmclient.py | 50 ++++++++++++- 2 files changed, 169 insertions(+), 5 deletions(-) diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index 455bd9b2..b1748d0f 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -1,5 +1,123 @@ +from dataikuapi.fm.future import FMFuture + class FMVirtualNetwork(object): - def __init__(self, client, virtual_network_settings): + def __init__(self, client, vn_data): self.client = client - self.settings = virtual_network_settings - self.id = self.settings['id'] + self.vn_data = vn_data + self.id = self.vn_data['id'] + + def save(self): + """ + Update the Virtual Network. + """ + self.client._perform_tenant_empty("PUT", "/virtual-networks/%s" % self.id, body=self.vn_data) + self.vn_data = self.client._perform_tenant_json("GET", "/virtual-networks/%s" % self.id) + + def delete(self): + """ + Delete the DSS Instance Settings Template. + + :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process + :rtype: :class:`~dataikuapi.fm.future.FMFuture` + """ + future = self.client._perform_tenant_json("DELETE", "/virtual-networks/%s" % self.id) + return FMFuture.from_resp(self.client, future) + + def set_fleet_management(self, enable, event_server=None, deployer_management="NO_MANAGED_DEPLOYER"): + """ + When enabled, all instances in this virtual network know each other and can centrally manage deployer and logs centralization + + :param boolean enable: Enable or not the Fleet Management + + :param str event_server: Optional, Node name of the node that should act as the centralized event server for logs concentration. Audit logs of all design, deployer and automation nodes will automatically be sent there. + :param str deployer_management: Optional, Accepts: + - "NO_MANAGED_DEPLOYER": Do not manage the the deployer. This is the default mode. + - "CENTRAL_DEPLOYER": Central deployer. Recommanded if you have more than one design node or may have more than one design node in the future. + - "EACH_DESIGN_NODE": Deployer from design. Recommanded if you have a single design node and want a simpler setup. + """ + + self.vn_data['managedNodesDirectory'] = enable + self.vn_data['eventServerNodeLabel'] = event_server + self.vn_data['nodesDirectoryDeployerMode'] = deployer_management + self.save() + + def set_dns_strategy(self, assign_domain_name, aws_private_ip_zone53_id=None, aws_public_ip_zone53_id=None, azure_dns_zone_id=None): + """ + Set the DNS strategy for this virtual network + + :param boolean assign_domain_name: If false, don't assign domain names, use ip_only + :param str aws_private_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for private ip + :param str aws_public_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for public ip + :param str azure_dns_zone_id: Optional, Azure Only, the ID of the Azure DNS zone to use + """ + + if assign_domain_name: + self.vn_data['dnsStrategy'] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" + self.vn_data['awsRoute53PrivateIPZoneId'] = aws_private_ip_zone53_id + self.vn_data['awsRoute53PublicIPZoneId'] = aws_public_ip_zone53_id + self.vn_data['azureDnsZoneId'] = azure_dns_zone_id + else : + self.vn_data['dnsStrategy'] = "NONE" + + self.save() + + def set_https_strategy(self, https_strategy): + """ + Set the HTTPS strategy for this virtual network + + :param object: a :class:`dataikuapi.fm.virtualnetworks.FMHTTPSStrategy` + """ + self.vn_data.update(https_strategy) + self.save() + +class FMHTTPSStrategy(dict): + def __init__(self, data, https_strategy, http_redirect=False): + """ + A class holding HTTPS Strategy for Virtual Network + + Do not create this directly, use: + - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.disable` to use HTTP only + - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.self_signed` to use self-signed certificates + - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.custom_cert` to use custom certificates + - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.lets_encrypt` to use Let's Encrypt + """ + super(FMHTTPSStrategy, self).__init__(data) + self['httpsStrategy'] = https_strategy + if http_redirect: + self['httpStrategy'] = "REDIRECT" + else: + self['httpStrategy'] = "DISABLE" + + @staticmethod + def disable(): + """ + Use HTTP only + """ + return FMHTTPSStrategy({}, "NONE", False) + + @staticmethod + def self_signed(http_redirect): + """ + Use self-signed certificates + + :param bool http_redirect: If true, HTTP is redirected to HTTPS. If false, HTTP is disabled. Defaults to false + """ + return FMHTTPSStrategy({}, "SELF_SIGNED", http_redirect) + + @staticmethod + def custom_cert(http_redirect): + """ + Use a custom certificate for each instance + + :param bool http_redirect: If true, HTTP is redirected to HTTPS. If false, HTTP is disabled. Defaults to false + """ + return FMHTTPSStrategy({}, "CUSTOM_CERTIFICATE", http_redirect) + + @staticmethod + def lets_encrypt(contact_mail): + """ + Use Let's Encrypt to generate https certificates + + :param str contact_mail: The contact email provided to Let's Encrypt + """ + return FMHTTPSStrategy({"contactMail": contact_mail}, "LETSENCRYPT", True) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 8a3af799..93e8f0dc 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -75,8 +75,54 @@ def get_virtual_network(self, virtual_network_id): :return: requested virtual network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ - template = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) - return FMVirtualNetwork(self, template) + vn = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) + return FMVirtualNetwork(self, vn) + + def create_virtual_network(self, + label, + awsVpcId=None, + awsSubnetId=None, + awsAutoCreateSecurityGroups=False, + awsSecurityGroups=None, + azureVnId=None, + azureSubnetId=None, + azureAutoUpdateSecurityGroups=None, + internetAccessMode = "YES"): + """ + Create a Virtual Network + + :param str label: The label of the Virtual Network + + :param str awsVpcId: AWS Only, ID of the VPC to use + :param str awsSubnetId: AWS Only, ID of the subnet to use + :param boolean awsAutoCreateSecurityGroups: Optional, AWS Only, If false, do not create security groups automatically. Defaults to false + :param list awsSecurityGroups: Optional, AWS Only, A list of up to 5 security group ids to assign to the instances created in this virtual network. Ignored if awsAutoCreateSecurityGroups is true + + :param str azureVnId: Azure Only, ID of the Azure Virtual Network to use + :param str azureSubnetId: Azure Only, ID of the subnet to use + :param boolean azureAutoUpdateSecurityGroups: Azure Only, Auto update the subnet security group + + :param str internetAccessMode: Optional, The internet access mode of the instances created in this virtual network. Accepts "YES", "NO", "EGRESS_ONLY". Defaults to "YES" + + :return: requested instance settings template + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + """ + + data = { + "label": label, + "awsVpcId": awsVpcId, + "awsSubnetId": awsSubnetId, + "awsAutoCreateSecurityGroups": awsAutoCreateSecurityGroups, + "awsSecurityGroups": awsSecurityGroups, + "azureVnId": azureVnId, + "azureSubnetId": azureSubnetId, + "azureAutoUpdateSecurityGroups": azureAutoUpdateSecurityGroups, + "internetAccessMode": internetAccessMode, + "mode": "EXISTING_MONOTENANT" + } + + vn = self._perform_tenant_json("POST", "/virtual-networks", body=data) + return FMVirtualNetwork(self, vn) ######################################################## From 349cbbdf69268249c5bdf730aec37b767c882391 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 3 Jun 2021 17:53:21 +0200 Subject: [PATCH 14/38] fm: Instance settings update --- dataikuapi/fm/instances.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 915ae874..b6f95cb2 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -65,6 +65,43 @@ def delete(self): future = self.client._perform_tenant_json("GET", "/instances/%s/actions/delete" % self.id) return FMFuture.from_resp(self.client, future) + def set_automated_snapshots(self, enable, period, keep=0): + """ + Set the automated snapshots policy for this instance + + :param boolean enable: Enable the automated snapshots + :param int period: The time period between 2 snapshot in hours + :param int keep: Optional, the number of snapshot to keep. Use 0 to keep all snapshots. Defaults to 0. + """ + self.instance_data['enableAutomatedSnapshot'] = enable + self.instance_data['automatedSnapshotPeriod'] = period + self.instance_data['automatedSnapshotRetention'] = keep + self.save() + + def set_elastic_ip(self, enable, elasticip_allocation_id): + """ + Set a public elastic ip for this instance + + :param boolan enable: Enable the elastic ip allocation + :param str elaticip_allocation_id: The AWS ElasticIP allocation ID or the Azure Public IP ID + """ + self.instance_data['awsAssignElasticIP'] = enable + self.instance_data['awsElasticIPAllocationId'] = elasticip_allocation_id + self.instance_data['azureAssignElasticIP'] = enable + self.instance_data['azurePublicIPId'] = elasticip_allocation_id + self.save() + + def set_custom_certificate(self, pem_data): + """ + Set the custom certificate for this instance + + Only needed when Virtual Network HTTPS Strategy is set to Custom Certificate + + param: str pem_data: The SSL certificate + """ + self.instance_data['sslCertificatePEM'] = pem_data + self.save() + class FMInstanceEncryptionMode(Enum): NONE = "NONE" From d7ceed6379bd39e4d0005fcf5da1b84b1ce03a76 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 1 Sep 2021 14:28:49 +0200 Subject: [PATCH 15/38] Add __init__.py in fm module --- dataikuapi/fm/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 dataikuapi/fm/__init__.py diff --git a/dataikuapi/fm/__init__.py b/dataikuapi/fm/__init__.py new file mode 100644 index 00000000..e69de29b From a971a96148260e021c0584fcd388898af33c4349 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 1 Sep 2021 16:15:46 +0200 Subject: [PATCH 16/38] FM Tenant: Update license --- dataikuapi/fm/tenant.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index 95f9aefa..4cafd8e6 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -1,3 +1,4 @@ +import json class FMCloudCredentials(object): """ A Tenant Cloud Credentials in the FM instance @@ -9,6 +10,36 @@ def __init__(self, client, cloud_credentials): def set_cmk_key(self, cmk_key_id): self.cloud_credentials['awsCMKId'] = cmk_key_id + def set_static_license(self, license_file=None, license_string=None): + """ + Set a default static license for the DSS instances + + Requires either a license_file or a license_string + + :param str license_file: Optional, load the license from a json file + :param str license_string: Optional, load the license from a json string + """ + if license_file is not None: + with open(license_file) as json_file: + license = json.load(json_file) + elif license_string is not None: + license = json.load(license_string) + else: + raise ValueError("a valid license_file or license_string needs to be provided") + self.cloud_credentials['licenseMode'] = 'STATIC' + self.cloud_credentials['license'] = json.dumps(license, indent=2) + + def set_automatically_updated_license(self, license_token): + """ + Set an automatically updated license for the DSS instances + + :param str license_token: License token + """ + if license_token is None: + raise ValueError("a valid license_token needs to be provided") + self.cloud_credentials['licenseMode'] = 'AUTO_UPDATE' + self.cloud_credentials['licenseToken'] = license_token + def save(self): """Saves back the settings to the project""" From 8706647b68950877ea552be29e685aa3ac89f8c2 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 1 Sep 2021 17:35:17 +0200 Subject: [PATCH 17/38] FM: Add helper to set cloud credentials --- dataikuapi/fm/tenant.py | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index 4cafd8e6..67c97dcd 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -9,6 +9,7 @@ def __init__(self, client, cloud_credentials): def set_cmk_key(self, cmk_key_id): self.cloud_credentials['awsCMKId'] = cmk_key_id + self.save() def set_static_license(self, license_file=None, license_string=None): """ @@ -28,6 +29,7 @@ def set_static_license(self, license_file=None, license_string=None): raise ValueError("a valid license_file or license_string needs to be provided") self.cloud_credentials['licenseMode'] = 'STATIC' self.cloud_credentials['license'] = json.dumps(license, indent=2) + self.save() def set_automatically_updated_license(self, license_token): """ @@ -39,6 +41,16 @@ def set_automatically_updated_license(self, license_token): raise ValueError("a valid license_token needs to be provided") self.cloud_credentials['licenseMode'] = 'AUTO_UPDATE' self.cloud_credentials['licenseToken'] = license_token + self.save() + + def set_authentication(self, authentication): + """ + Set the authentication for the tenant + + :param object: a :class:`dataikuapi.fm.tenant.FMCloudAuthentication` + """ + self.cloud_credentials.update(authentication) + self.save() def save(self): """Saves back the settings to the project""" @@ -46,3 +58,60 @@ def save(self): self.client._perform_tenant_empty("PUT", "/cloud-credentials", body = self.cloud_credentials) + +class FMCloudAuthentication(dict): + def __init__(self, data): + """ + A class holding the Cloud Authentication information + + Do not create this directly, use: + - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_same_as_fm` to use the same authentication as Fleet Manager + - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_iam_role` to use a custom IAM Role + - :meth:`dataikuapi.fm.tenant.FMCloudAuthentication.aws_keypair` to use a AWS Access key ID and AWS Secret Access Key pair + """ + super(FMCloudAuthentication, self).__init__(data) + + @staticmethod + def aws_same_as_fm(): + """ + AWS Only: Use the same authentication as Fleet Manager + """ + return FMCloudAuthentication({"awsAuthenticationMode": "SAME_AS_FM"}) + + @staticmethod + def aws_iam_role(role_arn): + """ + AWS Only: Use an IAM Role + + params: str role_arn: ARN of the IAM Role + """ + return FMCloudAuthentication({"awsAuthenticationMode": "IAM_ROLE", "awsIAMRoleARN": role_arn}) + + @staticmethod + def aws_keypair(access_key_id, secret_access_key): + """ + AWS Only: Use an AWS Access Key + + :param str access_key_id: AWS Access Key ID + :param str secret_access_key: AWS Secret Access Key + """ + return FMCloudAuthentication({"awsAuthenticationMode": "KEYPAIR", "awsAccessKeyId": access_key_id, "awsSecretAccessKey": secret_access_key}) + + @staticmethod + def azure(subscription, tenant_id, environment, client_id): + """ + Azure Only + + :param str subscription: Azure Subscription + :param str tenant_id: Azure Tenant Id + :param str environment: Azure Environment + :param str client_id: Azure Client Id + """ + data = { + "azureSubscription": subscription, + "azureTenantId": tenant_id, + "azureEnvironment": environment, + "azureFMAppClientId": client_id + } + + return FMCloudAuthentication(data) From fdf3be86ffeb8ffbe94539c0ef21b5f4397cd90e Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 1 Sep 2021 17:40:12 +0200 Subject: [PATCH 18/38] FM: Review comments --- dataikuapi/fmclient.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 93e8f0dc..92cf7a01 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -14,7 +14,7 @@ class FMClient(object): """Entry point for the FM API client""" - def __init__(self, host, api_key_id, api_key_secret, tenant_id, extra_headers = None): + def __init__(self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None): """ Instantiate a new FM API client on the given host with the given API key. @@ -43,7 +43,7 @@ def __init__(self, host, api_key_id, api_key_secret, tenant_id, extra_headers = def get_cloud_credentials(self): """ - Get Cloud Credential + Get Cloud Credentials :return: Cloud credentials :rtype: :class:`dataikuapi.fm.tenant.FMCloudCredentials` @@ -56,7 +56,7 @@ def get_cloud_credentials(self): # VirtualNetwork ######################################################## - def get_virtual_networks(self): + def list_virtual_networks(self): """ List all Virtual Networks @@ -129,7 +129,7 @@ def create_virtual_network(self, # Instance settings template ######################################################## - def get_instance_templates(self): + def list_instance_templates(self): """ List all Instance Settings Templates @@ -213,7 +213,7 @@ def create_instance_template(self, label, # Instance ######################################################## - def get_instances(self): + def list_instances(self): """ List all DSS Instances From df98c0ac96403cce56e30ab8789d7258a3aa862c Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 3 Sep 2021 16:44:57 +0200 Subject: [PATCH 19/38] Add run_ansible_task & install_system_packages --- dataikuapi/fm/instancesettingstemplates.py | 50 +++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 3d2439cb..9df21882 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -25,11 +25,24 @@ def delete(self): future = self.client._perform_tenant_json("DELETE", "/instance-settings-templates/%s" % self.id) return FMFuture.from_resp(self.client, future) + def add_setup_action(self, setup_action): + """ + Add a setup_action + + :param object setup_action: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` + """ + self.ist_data['setupActions'].append(setup_action) + self.save() + + class FMSetupAction(dict): def __init__(self, setupActionType, params=None): """ - param: object setupActionType: the type (`:class: FMSetupActionType`) of the SetupAction - param: str params: the parameters of the SetupAction in a json-encoded string + A class representing a SetupAction + + Do not create this directly, use: + - :meth:`dataikuapi.fm.instancesettingstemplates.FMSetupAction.add_authorized_key` + """ data = { "type": setupActionType.value, @@ -39,9 +52,31 @@ def __init__(self, setupActionType, params=None): super(FMSetupAction, self).__init__(data) -class FMSetupActionAddAuthorizedKey(FMSetupAction): - def __init__(self, ssh_key): - super(FMSetupActionAddAuthorizedKey, self).__init__(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key }) + @staticmethod + def add_authorized_key(ssh_key): + """ + Return a ADD_AUTHORIZED_KEY SetupAction + """ + return FMSetupAction(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key }) + + @staticmethod + def run_ansible_task(stage, yaml_string): + """ + Return a RUN_ANSIBLE_TASK SetupAction + + :params object stage: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` + :params str yaml_string: a yaml encoded string defining the ansibles tasks to run + """ + return FMSetupAction(FMSetupActionType.RUN_ANSIBLE_TASKS, {"stage": stage.value, "ansibleTasks": yaml_string }) + + @staticmethod + def install_system_packages(packages): + """ + Return an INSTALL_SYSTEM_PACKAGES SetupAction + + :params list packages: List of packages to install + """ + return FMSetupAction(FMSetupActionType.INSTALL_SYSTEM_PACKAGES, {"packages": packages }) class FMSetupActionType(Enum): RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" @@ -56,3 +91,8 @@ class FMSetupActionType(Enum): ADD_AUTHORIZED_KEY="ADD_AUTHORIZED_KEY" INSTALL_JDBC_DRIVER="INSTALL_JDBC_DRIVER" SETUP_ADVANCED_SECURITY="SETUP_ADVANCED_SECURITY" + +class FMSetupActionStage(Enum): + after_dss_startup="after_dss_startup" + after_install="after_install" + before_install="before_install" From 0b4340f800d1ff6a427726d72a9c6cc368cb68ad Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 3 Sep 2021 16:59:37 +0200 Subject: [PATCH 20/38] FM: Setup Advanced Security SetupAction --- dataikuapi/fm/instancesettingstemplates.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 9df21882..95b9da21 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -64,8 +64,8 @@ def run_ansible_task(stage, yaml_string): """ Return a RUN_ANSIBLE_TASK SetupAction - :params object stage: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` - :params str yaml_string: a yaml encoded string defining the ansibles tasks to run + :param object stage: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` + :param str yaml_string: a yaml encoded string defining the ansibles tasks to run """ return FMSetupAction(FMSetupActionType.RUN_ANSIBLE_TASKS, {"stage": stage.value, "ansibleTasks": yaml_string }) @@ -74,10 +74,20 @@ def install_system_packages(packages): """ Return an INSTALL_SYSTEM_PACKAGES SetupAction - :params list packages: List of packages to install + :param list packages: List of packages to install """ return FMSetupAction(FMSetupActionType.INSTALL_SYSTEM_PACKAGES, {"packages": packages }) + @staticmethod + def setup_advanced_security(basic_headers = True, hsts = False): + """ + Return an SETUP_ADVANCED_SECURITY SetupAction + + :param boolean basic_headers: Optional, Prevent browsers to render Web content served by DSS to be embedded into a frame, iframe, embed or object tag. Defaults to True + :param boolean hsts: Optional, Enforce HTTP Strict Transport Security. Defaults to False + """ + return FMSetupAction(FMSetupActionType.SETUP_ADVANCED_SECURITY, {"basic_headers": basic_headers, "hsts": hsts}) + class FMSetupActionType(Enum): RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" INSTALL_SYSTEM_PACKAGES="INSTALL_SYSTEM_PACKAGES" From 2f476ff89b49ac5ece6f32b1c0139e8e577eb3a2 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 3 Sep 2021 17:25:52 +0200 Subject: [PATCH 21/38] FM: Install JDBC Driver SetupAction --- dataikuapi/fm/instancesettingstemplates.py | 32 +++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 95b9da21..bb6bde6a 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -55,14 +55,14 @@ def __init__(self, setupActionType, params=None): @staticmethod def add_authorized_key(ssh_key): """ - Return a ADD_AUTHORIZED_KEY SetupAction + Return a ADD_AUTHORIZED_KEY FMSetupAction """ return FMSetupAction(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key }) @staticmethod def run_ansible_task(stage, yaml_string): """ - Return a RUN_ANSIBLE_TASK SetupAction + Return a RUN_ANSIBLE_TASK FMSetupAction :param object stage: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` :param str yaml_string: a yaml encoded string defining the ansibles tasks to run @@ -72,7 +72,7 @@ def run_ansible_task(stage, yaml_string): @staticmethod def install_system_packages(packages): """ - Return an INSTALL_SYSTEM_PACKAGES SetupAction + Return an INSTALL_SYSTEM_PACKAGES FMSetupAction :param list packages: List of packages to install """ @@ -81,13 +81,28 @@ def install_system_packages(packages): @staticmethod def setup_advanced_security(basic_headers = True, hsts = False): """ - Return an SETUP_ADVANCED_SECURITY SetupAction + Return an SETUP_ADVANCED_SECURITY FMSetupAction :param boolean basic_headers: Optional, Prevent browsers to render Web content served by DSS to be embedded into a frame, iframe, embed or object tag. Defaults to True :param boolean hsts: Optional, Enforce HTTP Strict Transport Security. Defaults to False """ return FMSetupAction(FMSetupActionType.SETUP_ADVANCED_SECURITY, {"basic_headers": basic_headers, "hsts": hsts}) + @staticmethod + def install_jdbc_driver(database_type, url, paths_in_archive=None, http_headers=None, http_username=None, http_password=None, datadir_subdirectory=None): + """ + Return a INSTALL_JDBC_DRIVER FMSetupAction + + :param object database_type: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionAddJDBCDriverDatabaseType` + :param str url: The full address to the driver. Supports http(s)://, s3://, abs:// or file:// endpoints + :param list paths_in_archive: Optional, must be used when the driver is shipped as a tarball or a ZIP file. Add here all the paths to find the JAR files in the driver archive. Paths are relative to the top of the archive. Wildcards are supported. + :param dict http_headers: Optional, If you download the driver from a HTTP(S) endpoint, add here the headers you want to add to the query. This setting is ignored for any other type of download. + :param str http_username: Optional, If the HTTP(S) endpoint expect a Basic Authentication, add here the username. To explicitely specify which Assigned Identity use if the machine have several, set the client_id here. To authenticate with a SAS Token on Azure Blob Storage (not recommended), use "token" as the value here. + :param str http_password: Optional, If the HTTP(S) endpoint expect a Basic Authentication, add here the password. To authenticate with a SAS Token on Azure Blob Storage (not recommended), store the token in this field. + :param str datadir_subdirectory: Optional, Some drivers are shipped with a high number of JAR files along with them. In that case, you might want to install them under an additional level in the DSS data directory. Set the name of this subdirectory here. Not required for most drivers. + """ + return FMSetupAction(FMSetupActionType.INSTALL_JDBC_DRIVER, {"url": url, "dbType": database_type.value, "pathsInArchive": paths_in_archive, "headers": http_headers, "username": http_username, "password": http_password, "subpathInDatadir": datadir_subdirectory}) + class FMSetupActionType(Enum): RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" INSTALL_SYSTEM_PACKAGES="INSTALL_SYSTEM_PACKAGES" @@ -106,3 +121,12 @@ class FMSetupActionStage(Enum): after_dss_startup="after_dss_startup" after_install="after_install" before_install="before_install" + +class FMSetupActionAddJDBCDriverDatabaseType(Enum): + mysql="mysql" + mssql="mssql" + oracle="oracle" + mariadb="mariadb" + snowflake="snowflake" + athena="athena" + bigquery="bigquery" From bd41121adec6d811ed9cf0a66a8a343cbf92872c Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 3 Sep 2021 17:40:45 +0200 Subject: [PATCH 22/38] FM: setup_k8s_and_spark SetupAction --- dataikuapi/fm/instancesettingstemplates.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index bb6bde6a..5eaa4a49 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -103,6 +103,13 @@ def install_jdbc_driver(database_type, url, paths_in_archive=None, http_headers= """ return FMSetupAction(FMSetupActionType.INSTALL_JDBC_DRIVER, {"url": url, "dbType": database_type.value, "pathsInArchive": paths_in_archive, "headers": http_headers, "username": http_username, "password": http_password, "subpathInDatadir": datadir_subdirectory}) + @staticmethod + def setup_k8s_and_spark(): + """ + Return a SETUP_K8S_AND_SPARK FMSetupAction + """ + return FMSetupAction(FMSetupActionType.SETUP_K8S_AND_SPARK) + class FMSetupActionType(Enum): RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" INSTALL_SYSTEM_PACKAGES="INSTALL_SYSTEM_PACKAGES" From 3ea7e8135881fba79674359c055631a9d185d2ae Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 7 Sep 2021 17:23:13 +0200 Subject: [PATCH 23/38] FM: FMInstanceCreator --- dataikuapi/fm/instances.py | 98 ++++++++++++++++++++++++++++++++++++++ dataikuapi/fmclient.py | 11 ++++- 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index b6f95cb2..3a4e7f7b 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,6 +1,104 @@ from enum import Enum from .future import FMFuture +class FMInstanceCreator(object): + def __init__(self, client, label, instance_settings_template_id, virtual_network_id, image_id): + """ + Helper to create a DSS Instance + + :param str instance_settings_template: The instance settings template id this instance should be based on + :param str virtual_network: The virtual network where the instance should be spawned + :param str label: The label of the instance + :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) + """ + self.client = client + self.data = {} + self.data["label"] = label + self.data["instanceSettingsTemplateId"] = instance_settings_template_id + self.data["virtualNetworkId"] = virtual_network_id + self.data["imageId"] = image_id + + # Set the default value for dssNodeType + self.data["dssNodeType"] = "design" + + def create(self): + """ + Create the DSS instance + + :return: Instance + :rtype: :class:`dataikuapi.fm.instances.FMInstance` + """ + instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) + return FMInstance(self.client, instance) + + def set_dss_node_type(self, dss_node_type): + """ + Set the DSS Node type of the instance to create + + :param str dss_node_type: Optional , the type of the dss node to create. Supports "design", "automation ordeployer". Defaults to "design" + """ + if dss_node_type not in ["design", "automation", "deployer"]: + raise ValueError("Only \"design\", \"automation\" or \"deployer\" dss_node_type are supported") + self.data["dssNodeType"] = dss_node_type + + def set_cloud_instance_type(self, cloud_instance_type): + """ + Set the machine type for the DSS Instance + """ + self.data["cloudInstanceType"] = cloud_instance_type + + def set_data_volume_options(self, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=None, data_volume_encryption_key=None): + """ + Set the options of the data volume to use with the DSS Instance + + :param str data_volume_type: Optional, Data volume type + :param int data_volume_size: Optional, Data volume initial size + :param int data_volume_size_max: Optional, Data volume maximum size + :param int data_volume_IOPS: Optional, Data volume IOPS + :param object data_volume_encryption: Optional, a :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode` setting the encryption mode of the data volume + :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM + """ + if type(data_volume_encryption) is not FMInstanceEncryptionMode: + raise TypeError("data_volume encryption needs to be of type FMInstanceEncryptionMode") + + self.data["dataVolumeType"] = data_volume_type + self.data["dataVolumeSizeGB"] = data_volume_size + self.data["dataVolumeSizeMaxGB"] = data_volume_size_max + self.data["dataVolumeIOPS"] = data_volume_IOPS + self.data["dataVolumeEncryption"] = data_volume_encryption.value + self.data["dataVolumeEncryptionKey"] = data_volume_encryption_key + + def set_aws_root_volume_options(self, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None): + """ + AWS Only: Set the options of the root volume of the DSS Instance + + :param int aws_root_volume_size: Optional, the root volume size + :param str aws_root_volume_type: Optional, the root volume type + :param int aws_root_volume_IOPS: Optional, the root volume IOPS + """ + if self.client.cloud != "AWS": + raise BaseException("set_aws_root_volume_options is only usable on AWS tenants") + self.data["awsRootVolumeSizeGB"] = aws_root_volume_size + self.data["awsRootVolumeType"] = aws_root_volume_type + self.data["awsRootVolumeIOPS"] = aws_root_volume_IOPS + + def set_cloud_tags(self, cloud_tags): + """ + Set the tags to be applied to the cloud resources created for this DSS instance + + :param dict cloud_tags: a key value dictionary of tags to be applied on the cloud resources + """ + self.data["cloudTags"] = cloud_tags + + def set_fm_tags(self, fm_tags): + """ + A list of tags to add on the DSS Instance in Fleet Manager + + :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager + """ + self.data["fmTags"] = fm_tags + + class FMInstance(object): """ A handle to interact with a DSS instance. diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 92cf7a01..87d1210b 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -2,8 +2,9 @@ from requests import Session from requests import exceptions from requests.auth import HTTPBasicAuth - import os.path as osp + +from enum import Enum from .utils import DataikuException from .fm.tenant import FMCloudCredentials @@ -14,17 +15,23 @@ class FMClient(object): """Entry point for the FM API client""" - def __init__(self, host, api_key_id, api_key_secret, tenant_id="main", extra_headers=None): + def __init__(self, host, api_key_id, api_key_secret, cloud, tenant_id="main", extra_headers=None): """ Instantiate a new FM API client on the given host with the given API key. API keys can be managed in FM on the project page or in the global settings. The API key will define which operations are allowed for the client. + + :param str host: Full url of the FM + """ self.api_key_id = api_key_id self.api_key_secret = api_key_secret self.host = host + if cloud not in ["AWS", "Azure"]: + raise ValueError("cloud should be either \"AWS\" or \"Azure\"") + self.cloud = cloud self.__tenant_id = tenant_id self._session = Session() From 00f698c377b4c503459c7236115439f8c9456556 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Wed, 8 Sep 2021 14:57:18 +0200 Subject: [PATCH 24/38] FM: Split Instance and InstanceCreator per cloud --- dataikuapi/fm/instances.py | 112 +++++++++++++++++++++++++------------ dataikuapi/fmclient.py | 54 +++--------------- 2 files changed, 84 insertions(+), 82 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 3a4e7f7b..fec296d1 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -6,9 +6,10 @@ def __init__(self, client, label, instance_settings_template_id, virtual_network """ Helper to create a DSS Instance + :param object client: :class:`dataikuapi.fm.fmclient` + :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned - :param str label: The label of the instance :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) """ self.client = client @@ -21,33 +22,29 @@ def __init__(self, client, label, instance_settings_template_id, virtual_network # Set the default value for dssNodeType self.data["dssNodeType"] = "design" - def create(self): - """ - Create the DSS instance - - :return: Instance - :rtype: :class:`dataikuapi.fm.instances.FMInstance` - """ - instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) - return FMInstance(self.client, instance) - - def set_dss_node_type(self, dss_node_type): + def with_dss_node_type(self, dss_node_type): """ Set the DSS Node type of the instance to create :param str dss_node_type: Optional , the type of the dss node to create. Supports "design", "automation ordeployer". Defaults to "design" + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if dss_node_type not in ["design", "automation", "deployer"]: raise ValueError("Only \"design\", \"automation\" or \"deployer\" dss_node_type are supported") self.data["dssNodeType"] = dss_node_type + return self - def set_cloud_instance_type(self, cloud_instance_type): + def with_cloud_instance_type(self, cloud_instance_type): """ Set the machine type for the DSS Instance + + :param str cloud_instance_type + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ self.data["cloudInstanceType"] = cloud_instance_type + return self - def set_data_volume_options(self, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=None, data_volume_encryption_key=None): + def with_data_volume_options(self, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=None, data_volume_encryption_key=None): """ Set the options of the data volume to use with the DSS Instance @@ -57,6 +54,7 @@ def set_data_volume_options(self, data_volume_type=None, data_volume_size=None, :param int data_volume_IOPS: Optional, Data volume IOPS :param object data_volume_encryption: Optional, a :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode` setting the encryption mode of the data volume :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if type(data_volume_encryption) is not FMInstanceEncryptionMode: raise TypeError("data_volume encryption needs to be of type FMInstanceEncryptionMode") @@ -67,42 +65,71 @@ def set_data_volume_options(self, data_volume_type=None, data_volume_size=None, self.data["dataVolumeIOPS"] = data_volume_IOPS self.data["dataVolumeEncryption"] = data_volume_encryption.value self.data["dataVolumeEncryptionKey"] = data_volume_encryption_key + return self + + def with_cloud_tags(self, cloud_tags): + """ + Set the tags to be applied to the cloud resources created for this DSS instance - def set_aws_root_volume_options(self, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None): + :param dict cloud_tags: a key value dictionary of tags to be applied on the cloud resources + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ - AWS Only: Set the options of the root volume of the DSS Instance + self.data["cloudTags"] = cloud_tags + return self + + def with_fm_tags(self, fm_tags): + """ + A list of tags to add on the DSS Instance in Fleet Manager + + :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` + """ + self.data["fmTags"] = fm_tags + return self + + +class FMAWSInstanceCreator(FMInstanceCreator): + def with_aws_root_volume_options(self, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None): + """ + Set the options of the root volume of the DSS Instance :param int aws_root_volume_size: Optional, the root volume size :param str aws_root_volume_type: Optional, the root volume type :param int aws_root_volume_IOPS: Optional, the root volume IOPS + :rtype: :class:`dataikuapi.fm.instances.FMAWSInstanceCreator` """ - if self.client.cloud != "AWS": - raise BaseException("set_aws_root_volume_options is only usable on AWS tenants") self.data["awsRootVolumeSizeGB"] = aws_root_volume_size self.data["awsRootVolumeType"] = aws_root_volume_type self.data["awsRootVolumeIOPS"] = aws_root_volume_IOPS + return self - def set_cloud_tags(self, cloud_tags): + def create(self): """ - Set the tags to be applied to the cloud resources created for this DSS instance + Create the DSS instance - :param dict cloud_tags: a key value dictionary of tags to be applied on the cloud resources + :return: Created DSS Instance + :rtype: :class:`dataikuapi.fm.instances.FMAWSInstance` """ - self.data["cloudTags"] = cloud_tags + instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) + return FMAWSInstance(self.client, instance) - def set_fm_tags(self, fm_tags): + +class FMAzureInstanceCreator(FMInstanceCreator): + def create(self): """ - A list of tags to add on the DSS Instance in Fleet Manager + Create the DSS instance - :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager + :return: Created DSS Instance + :rtype: :class:`dataikuapi.fm.instances.FMAzureInstance` """ - self.data["fmTags"] = fm_tags + instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) + return FMAzureInstance(self.client, instance) class FMInstance(object): """ A handle to interact with a DSS instance. - Do not create this directly, use :meth:`FMClient.get_instance` or :meth: `FMClient.create_instance` + Do not create this directly, use :meth:`FMClient.get_instance` or :meth: `FMClient.new_instance_creator` """ def __init__(self, client, instance_data): self.client = client @@ -176,28 +203,41 @@ def set_automated_snapshots(self, enable, period, keep=0): self.instance_data['automatedSnapshotRetention'] = keep self.save() + def set_custom_certificate(self, pem_data): + """ + Set the custom certificate for this instance + + Only needed when Virtual Network HTTPS Strategy is set to Custom Certificate + + param: str pem_data: The SSL certificate + """ + self.instance_data['sslCertificatePEM'] = pem_data + self.save() + + +class FMAWSInstance(FMInstance): def set_elastic_ip(self, enable, elasticip_allocation_id): """ Set a public elastic ip for this instance :param boolan enable: Enable the elastic ip allocation - :param str elaticip_allocation_id: The AWS ElasticIP allocation ID or the Azure Public IP ID + :param str elaticip_allocation_id: AWS ElasticIP allocation ID """ self.instance_data['awsAssignElasticIP'] = enable self.instance_data['awsElasticIPAllocationId'] = elasticip_allocation_id - self.instance_data['azureAssignElasticIP'] = enable - self.instance_data['azurePublicIPId'] = elasticip_allocation_id self.save() - def set_custom_certificate(self, pem_data): - """ - Set the custom certificate for this instance - Only needed when Virtual Network HTTPS Strategy is set to Custom Certificate +class FMAzureInstance(FMInstance): + def set_elastic_ip(self, enable, public_ip_id): + """ + Set a public elastic ip for this instance - param: str pem_data: The SSL certificate + :param boolan enable: Enable the elastic ip allocation + :param str public_ip_id: Azure Public IP ID """ - self.instance_data['sslCertificatePEM'] = pem_data + self.instance_data['azureAssignElasticIP'] = enable + self.instance_data['azurePublicIPId'] = public_ip_id self.save() diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 87d1210b..123688a0 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -11,6 +11,7 @@ from .fm.virtualnetworks import FMVirtualNetwork from .fm.instances import FMInstance, FMInstanceEncryptionMode from .fm.instancesettingstemplates import FMInstanceSettingsTemplate +from dataikuapi.fm.instances import FMAWSInstanceCreator, FMAzureInstanceCreator class FMClient(object): """Entry point for the FM API client""" @@ -242,58 +243,19 @@ def get_instance(self, instance_id): instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return FMInstance(self, instance) - def create_instance(self, instance_settings_template, virtual_network, label, image_id, - dss_node_type="design", - cloud_instance_type=None, data_volume_type=None, data_volume_size=None, - data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=FMInstanceEncryptionMode.NONE, - data_volume_encryption_key=None, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None, - cloud_tags=None, fm_tags=None): + def new_instance_creator(self, label, instance_settings_template_id, virtual_network_id, image_id): """ - Create a DSS Instance + Instantiate a new instance creator + :param str label: The label of the instance :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned - :param str label: The label of the instance :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) - - :param str dss_node_type: Optional , the type of the dss node to create. Defaults to "design" - :param str cloud_instance_type: Optional, Machine type - :param str data_volume_type: Optional, Data volume type - :param int data_volume_size: Optional, Data volume initial size - :param int data_volume_size_max: Optional, Data volume maximum size - :param int data_volume_IOPS: Optional, Data volume IOPS - :param object data_volume_encryption: Optional, a :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode` setting the encryption mode of the data volume - :param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM - :param int aws_root_volume_size: Optional, the root volume size - :param str aws_root_volume_type: Optional, the root volume type - :param int aws_root_volume_IOPS: Optional, the root volume IOPS - :param dict cloud_tags: Optional, a key value dictionary of tags to be applied on the cloud resources - :param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager - - :return: Instance - :rtype: :class:`dataikuapi.fm.instances.FMInstance` """ - data = { - "virtualNetworkId": virtual_network.id, - "instanceSettingsTemplateId": instance_settings_template.id, - "label": label, - "dssNodeType": dss_node_type, - "imageId": image_id, - "cloudInstanceType": cloud_instance_type, - "dataVolumeType": data_volume_type, - "dataVolumeSizeGB": data_volume_size, - "dataVolumeSizeMaxGB": data_volume_size_max, - "dataVolumeIOPS": data_volume_IOPS, - "dataVolumeEncryption": data_volume_encryption.value, - "dataVolumeEncryptionKey": data_volume_encryption_key, - "awsRootVolumeSizeGB": aws_root_volume_size, - "awsRootVolumeType": aws_root_volume_type, - "awsRootVolumeIOPS": aws_root_volume_IOPS, - "cloudTags": cloud_tags, - "fmTags": fm_tags - } - instance = self._perform_tenant_json("POST", "/instances", body=data) - return FMInstance(self, instance) + if self.cloud == "AWS": + return FMAWSInstanceCreator(self, label, instance_settings_template_id, virtual_network_id, image_id) + elif self.cloud == "Azure": + return FMAzureInstanceCreator(self, label, instance_settings_template_id, virtual_network_id, image_id) ######################################################## From 1de09279ffc1545fa4424349d1bf06cd81adae0b Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Fri, 17 Sep 2021 17:51:18 +0200 Subject: [PATCH 25/38] FM InstanceSettingsTemplateCreator --- dataikuapi/fm/instancesettingstemplates.py | 145 +++++++++++++++++++++ dataikuapi/fmclient.py | 67 ++-------- 2 files changed, 156 insertions(+), 56 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 5eaa4a49..437da3c9 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -2,6 +2,151 @@ import json from dataikuapi.fm.future import FMFuture +class FMInstanceSettingsTemplateCreator(object): + def __init__(self, client, label): + """ + Create an Instance Template + + :param str azureSshKey: Optional, Azure Only, the ssh public key to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. + :param str startupManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at startup time + :param str runtimeManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at runtime + + :return: requested instance settings template + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + """ + + self.data = {} + self.data["label"] = label + self.client = client + + def create(self): + template = self.client._perform_tenant_json("POST", "/instance-settings-templates", body=self.data) + return FMInstanceSettingsTemplate(self, template) + + def with_setup_actions(self, setup_actions): + """ + Add setup actions + + :param list setup_actions: List of :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` to be played on an instance + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplateCreator` + """ + self.data["setupActions"] = setup_actions + return self + + def with_license(self, license): + self.data["license"] = license + return self + + +class FMAWSInstanceSettingsTemplateCreator(FMInstanceSettingsTemplateCreator): + def with_aws_keypair(self, aws_keypair_name): + """ + Add an AWS Keypair to the DSS instance. + Needed to get SSH access to the DSS instance, using the centos user. + + :param str aws_keypair_name: Name of an AWS key pair to add to the instance. + """ + self.data["awsKeyPairName"] = aws_keypair_name + return self + + def with_startup_instance_profile(self, startup_instance_profile_arn): + """ + Add a Instance Profile to be assign to the DSS instance on startup + + :param str startup_instance_profile_arn: ARN of the Instance profile assigned to the DSS instance at startup time + """ + self.data["startupInstanceProfileArn"] = startup_instance_profile_arn + return self + + def with_runtime_instance_profile(self, runtime_instance_profile_arn): + """ + Add a Instance Profile to be assign to the DSS instance when running + + :param str runtime_instance_profile_arn: ARN of the Instance profile assigned to the DSS instance during runtime + """ + self.data["runtimeInstanceProfileArn"] = runtime_instance_profile_arn + return self + + def with_restrict_aws_metadata_server_access(self, restrict_aws_metadata_server_access = True): + """ + Restrict AWS metadata server access on the DSS instance. + + :param boolean restrict_aws_metadata_server_access: Optional, If true, restrict the access to the metadata server access. Defaults to true + """ + self.data["restrictAwsMetadataServerAccess"] = restrict_aws_metadata_server_access + return self + + def with_default_aws_api_access_mode(self): + """ + The DSS Instance will use the Runtime Instance Profile to access AWS API. + """ + self.data["dataikuAwsAPIAccessMode"] = "NONE" + return self + + def with_keypair_aws_api_access_mode(self, aws_access_key_id, aws_keypair_storage_mode="NONE", aws_secret_access_key=None, aws_secret_access_key_aws_secret_name=None, aws_secrets_manager_region=None): + """ + DSS Instance will use an Access Key to authenticate against the AWS API. + + :param str aws_access_key_id: AWS Access Key ID. + :param str aws_keypair_storage_mode: Optional, the storage mode of the AWS api key. Accepts "NONE", "INLINE_ENCRYPTED" or "AWS_SECRETS_MANAGER". Defaults to "NONE" + :param str aws_secret_access_key: Optional, AWS Access Key Secret. Only needed if keypair_storage_mode is "INLINE_ENCRYPTED" + :param str aws_secret_access_key_aws_secret_name: Optional, ASM secret name. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" + :param str aws_secrets_manager_region: Optional, Secret Manager region to use. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" + """ + if aws_keypair_storage_mode not in ["NONE", "INLINE_ENCRYPTED", "AWS_SECRETS_MANAGER"]: + raise ValueError("aws_keypair_storage_mode should be either \"NONE\", \"INLINE_ENCRYPTED\" or \"AWS_SECRET_MANAGER\"") + + self.data["dataikuAwsAPIAccessMode"] = "KEYPAIR" + self.data["dataikuAwsKeypairStorageMode"] = aws_keypair_storage_mode + + if aws_keypair_storage_mode == "NONE": + return self + + self.data["dataikuAwsAccessKeyId"] = aws_access_key_id + + if aws_keypair_storage_mode == "INLINE_ENCRYPTED": + if aws_secret_access_key == None: + raise ValueError("When aws_keypair_storage_mode is \"INLINE_ENCRYPTED\", aws_secret_access_key should be provided") + self.data["dataikuAwsSecretAccessKey"] = aws_secret_access_key + elif aws_keypair_storage_mode == "AWS_SECRETS_MANAGER": + if aws_secret_access_key_aws_secret_name == None: + raise ValueError("When aws_keypair_storage_mode is \"AWS_SECRETS_MANAGER\", aws_secret_access_key_aws_secret_name should be provided") + self.data["dataikuAwsSecretAccessKeyAwsSecretName"] = aws_secret_access_key_aws_secret_name + self.data["awsSecretsManagerRegion"] = aws_secrets_manager_region + + return self + + +class FMAzureInstanceSettingsTemplateCreator(FMInstanceSettingsTemplateCreator): + def with_ssh_key(self, ssh_public_key): + """ + Add an SSH public key to the DSS Instance. + Needed to access it through SSH, using the centos user. + + :param str ssh_public_key: The content of the public key to add to the instance. + """ + self.data["azureSshKey"] = ssh_public_key + return self + + def with_startup_managed_identity(self, startup_managed_identity): + """ + Add a managed identity to be assign to the DSS instance on startup + + :param str startup_managed_identity: Managed Identity ID + """ + self.data["startupManagedIdentity"] = startup_managed_identity + return self + + def with_runtime_managed_identity(self, runtime_managed_identity): + """ + Add a managed identity to be assign to the DSS instance when running + + :param str runtime_managed_identity: Managed Identity ID + """ + self.data["runtimeManagedIdentity"] = runtime_managed_identity + return self + + class FMInstanceSettingsTemplate(object): def __init__(self, client, ist_data): self.client = client diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 123688a0..2857c406 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -9,9 +9,8 @@ from .fm.tenant import FMCloudCredentials from .fm.virtualnetworks import FMVirtualNetwork -from .fm.instances import FMInstance, FMInstanceEncryptionMode -from .fm.instancesettingstemplates import FMInstanceSettingsTemplate -from dataikuapi.fm.instances import FMAWSInstanceCreator, FMAzureInstanceCreator +from .fm.instances import FMInstance, FMInstanceEncryptionMode, FMAWSInstanceCreator, FMAzureInstanceCreator +from .fm.instancesettingstemplates import FMInstanceSettingsTemplate, FMAWSInstanceSettingsTemplateCreator, FMAzureInstanceSettingsTemplateCreator class FMClient(object): """Entry point for the FM API client""" @@ -159,62 +158,17 @@ def get_instance_template(self, template_id): template = self._perform_tenant_json("GET", "/instance-settings-templates/%s" % template_id) return FMInstanceSettingsTemplate(self, template) - - def create_instance_template(self, label, - setupActions=None, license=None, - awsKeyPairName=None, startupInstanceProfileArn=None, runtimeInstanceProfileArn=None, - restrictAwsMetadataServerAccess=True, dataikuAwsAPIAccessMode="NONE", dataikuAwsKeypairStorageMode=None, - dataikuAwsAccessKeyId=None, dataikuAwsSecretAccessKey=None, - dataikuAwsSecretAccessKeyAwsSecretName=None, awsSecretsManagerRegion=None, - azureSshKey=None, startupManagedIdentity=None, runtimeManagedIdentity=None): + def new_instance_template_creator(self, label): """ - Create an Instance Template - - :param str label: The label of the Instance Settings Template - - :param list setupActions: Optional, a list of :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` to be played on an instance - :param str license: Optional, overrides the license set in Cloud Setup - - :param str awsKeyPairName: Optional, AWS Only, the name of an AWS key pair to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. - :param str startupInstanceProfileArn: Optional, AWS Only, the ARN of the Instance profile assigned to the DSS instance at startup time - :param str runtimeInstanceProfileArn: Optional, AWS Only, the ARN of the Instance profile assigned to the DSS instance at runtime - :param boolean restrictAwsMetadataServerAccess: Optional, AWS Only, If true, restrict the access to the metadata server access. Defaults to true - :param str dataikuAwsAPIAccessMode: Optional, AWS Only, the access mode DSS is using to connect to the AWS API. If "NONE" DSS will use the Instance Profile, If "KEYPAIR", an AWS access key id and secret will be securely given to the dataiku account. - :param str dataikuAwsKeypairStorageMode: Optional, AWS Only, the storage mode of the AWS api key. Accepts "NONE", "INLINE_ENCRYPTED" or "AWS_SECRETS_MANAGER" - :param str dataikuAwsAccessKeyId: Optional, AWS Only, AWS Access Key ID. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" - :param str dataikuAwsSecretAccessKey: Optional, AWS Only, AWS Access Key Secret. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" and dataikuAwsKeypairStorageMode is "INLINE_ENCRYPTED" - :param str dataikuAwsSecretAccessKeyAwsSecretName: Optional, AWS Only, ASM secret name. Only needed if dataikuAwsAPIAccessMode is "KEYPAIR" and dataikuAwsKeypairStorageMode is "AWS_SECRET_MANAGER" - :param str awsSecretsManagerRegion: Optional, AWS Only + Instantiate a new instance template creator - :param str azureSshKey: Optional, Azure Only, the ssh public key to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. - :param str startupManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at startup time - :param str runtimeManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at runtime - - :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + :param str label: The label of the instance + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplateCreator` """ - - data = { - "label": label, - "setupActions": setupActions, - "license": license, - "awsKeyPairName": awsKeyPairName, - "startupInstanceProfileArn": startupInstanceProfileArn, - "runtimeInstanceProfileArn": runtimeInstanceProfileArn, - "restrictAwsMetadataServerAccess": restrictAwsMetadataServerAccess, - "dataikuAwsAPIAccessMode": "dataikuAwsAPIAccessMode", - "dataikuAwsKeypairStorageMode": dataikuAwsKeypairStorageMode, - "dataikuAwsAccessKeyId": dataikuAwsAccessKeyId, - "dataikuAwsSecretAccessKey": dataikuAwsSecretAccessKey, - "dataikuAwsSecretAccessKeyAwsSecretName": dataikuAwsSecretAccessKeyAwsSecretName, - "awsSecretsManagerRegion": awsSecretsManagerRegion, - "azureSshKey": azureSshKey, - "startupManagedIdentity": startupManagedIdentity, - "runtimeManagedIdentity": runtimeManagedIdentity - } - - template = self._perform_tenant_json("POST", "/instance-settings-templates", body=data) - return FMInstanceSettingsTemplate(self, template) + if self.cloud == "AWS": + return FMAWSInstanceSettingsTemplateCreator(self, label) + elif self.cloud == "Azure": + return FMAzureInstanceSettingsTemplateCreator(self, label) ######################################################## @@ -251,6 +205,7 @@ def new_instance_creator(self, label, instance_settings_template_id, virtual_net :param str instance_settings_template: The instance settings template id this instance should be based on :param str virtual_network: The virtual network where the instance should be spawned :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) + :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if self.cloud == "AWS": return FMAWSInstanceCreator(self, label, instance_settings_template_id, virtual_network_id, image_id) From 234341e8c92c441626fb03a154e83784da8af0e5 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 21 Sep 2021 18:17:22 +0200 Subject: [PATCH 26/38] FM: InstanceSettingsTemplate format + doc --- dataikuapi/fm/instancesettingstemplates.py | 168 ++++++++++++++------- 1 file changed, 116 insertions(+), 52 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 437da3c9..4fe035c6 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -2,17 +2,13 @@ import json from dataikuapi.fm.future import FMFuture + class FMInstanceSettingsTemplateCreator(object): def __init__(self, client, label): """ - Create an Instance Template - - :param str azureSshKey: Optional, Azure Only, the ssh public key to add to the instance. Needed to get SSH access to the DSS instance, using the centos user. - :param str startupManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at startup time - :param str runtimeManagedIdentity: Optional, Azure Only, the managed identity assigned to the DSS instance at runtime + A builder class to create an Instance Settings Template - :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + :param str label: The label of the Virtual Network """ self.data = {} @@ -20,7 +16,16 @@ def __init__(self, client, label): self.client = client def create(self): - template = self.client._perform_tenant_json("POST", "/instance-settings-templates", body=self.data) + """ + Create the Instance Settings Template + + :return: Created InstanceSettingsTemplate + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + """ + + template = self.client._perform_tenant_json( + "POST", "/instance-settings-templates", body=self.data + ) return FMInstanceSettingsTemplate(self, template) def with_setup_actions(self, setup_actions): @@ -67,13 +72,17 @@ def with_runtime_instance_profile(self, runtime_instance_profile_arn): self.data["runtimeInstanceProfileArn"] = runtime_instance_profile_arn return self - def with_restrict_aws_metadata_server_access(self, restrict_aws_metadata_server_access = True): + def with_restrict_aws_metadata_server_access( + self, restrict_aws_metadata_server_access=True + ): """ Restrict AWS metadata server access on the DSS instance. :param boolean restrict_aws_metadata_server_access: Optional, If true, restrict the access to the metadata server access. Defaults to true """ - self.data["restrictAwsMetadataServerAccess"] = restrict_aws_metadata_server_access + self.data[ + "restrictAwsMetadataServerAccess" + ] = restrict_aws_metadata_server_access return self def with_default_aws_api_access_mode(self): @@ -83,7 +92,14 @@ def with_default_aws_api_access_mode(self): self.data["dataikuAwsAPIAccessMode"] = "NONE" return self - def with_keypair_aws_api_access_mode(self, aws_access_key_id, aws_keypair_storage_mode="NONE", aws_secret_access_key=None, aws_secret_access_key_aws_secret_name=None, aws_secrets_manager_region=None): + def with_keypair_aws_api_access_mode( + self, + aws_access_key_id, + aws_keypair_storage_mode="NONE", + aws_secret_access_key=None, + aws_secret_access_key_aws_secret_name=None, + aws_secrets_manager_region=None, + ): """ DSS Instance will use an Access Key to authenticate against the AWS API. @@ -93,8 +109,14 @@ def with_keypair_aws_api_access_mode(self, aws_access_key_id, aws_keypair_storag :param str aws_secret_access_key_aws_secret_name: Optional, ASM secret name. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" :param str aws_secrets_manager_region: Optional, Secret Manager region to use. Only needed if aws_keypair_storage_mode is "AWS_SECRET_MANAGER" """ - if aws_keypair_storage_mode not in ["NONE", "INLINE_ENCRYPTED", "AWS_SECRETS_MANAGER"]: - raise ValueError("aws_keypair_storage_mode should be either \"NONE\", \"INLINE_ENCRYPTED\" or \"AWS_SECRET_MANAGER\"") + if aws_keypair_storage_mode not in [ + "NONE", + "INLINE_ENCRYPTED", + "AWS_SECRETS_MANAGER", + ]: + raise ValueError( + 'aws_keypair_storage_mode should be either "NONE", "INLINE_ENCRYPTED" or "AWS_SECRET_MANAGER"' + ) self.data["dataikuAwsAPIAccessMode"] = "KEYPAIR" self.data["dataikuAwsKeypairStorageMode"] = aws_keypair_storage_mode @@ -106,12 +128,18 @@ def with_keypair_aws_api_access_mode(self, aws_access_key_id, aws_keypair_storag if aws_keypair_storage_mode == "INLINE_ENCRYPTED": if aws_secret_access_key == None: - raise ValueError("When aws_keypair_storage_mode is \"INLINE_ENCRYPTED\", aws_secret_access_key should be provided") + raise ValueError( + 'When aws_keypair_storage_mode is "INLINE_ENCRYPTED", aws_secret_access_key should be provided' + ) self.data["dataikuAwsSecretAccessKey"] = aws_secret_access_key elif aws_keypair_storage_mode == "AWS_SECRETS_MANAGER": if aws_secret_access_key_aws_secret_name == None: - raise ValueError("When aws_keypair_storage_mode is \"AWS_SECRETS_MANAGER\", aws_secret_access_key_aws_secret_name should be provided") - self.data["dataikuAwsSecretAccessKeyAwsSecretName"] = aws_secret_access_key_aws_secret_name + raise ValueError( + 'When aws_keypair_storage_mode is "AWS_SECRETS_MANAGER", aws_secret_access_key_aws_secret_name should be provided' + ) + self.data[ + "dataikuAwsSecretAccessKeyAwsSecretName" + ] = aws_secret_access_key_aws_secret_name self.data["awsSecretsManagerRegion"] = aws_secrets_manager_region return self @@ -149,16 +177,20 @@ def with_runtime_managed_identity(self, runtime_managed_identity): class FMInstanceSettingsTemplate(object): def __init__(self, client, ist_data): - self.client = client - self.id = ist_data['id'] + self.client = client + self.id = ist_data["id"] self.ist_data = ist_data def save(self): """ Update the Instance Settings Template. """ - self.client._perform_tenant_empty("PUT", "/instance-settings-templates/%s" % self.id, body=self.ist_data) - self.ist_data = self.client._perform_tenant_json("GET", "/instance-settings-templates/%s" % self.id) + self.client._perform_tenant_empty( + "PUT", "/instance-settings-templates/%s" % self.id, body=self.ist_data + ) + self.ist_data = self.client._perform_tenant_json( + "GET", "/instance-settings-templates/%s" % self.id + ) def delete(self): """ @@ -167,7 +199,9 @@ def delete(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("DELETE", "/instance-settings-templates/%s" % self.id) + future = self.client._perform_tenant_json( + "DELETE", "/instance-settings-templates/%s" % self.id + ) return FMFuture.from_resp(self.client, future) def add_setup_action(self, setup_action): @@ -176,7 +210,7 @@ def add_setup_action(self, setup_action): :param object setup_action: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ - self.ist_data['setupActions'].append(setup_action) + self.ist_data["setupActions"].append(setup_action) self.save() @@ -193,7 +227,7 @@ def __init__(self, setupActionType, params=None): "type": setupActionType.value, } if params: - data['params'] = params + data["params"] = params super(FMSetupAction, self).__init__(data) @@ -202,7 +236,7 @@ def add_authorized_key(ssh_key): """ Return a ADD_AUTHORIZED_KEY FMSetupAction """ - return FMSetupAction(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key }) + return FMSetupAction(FMSetupActionType.ADD_AUTHORIZED_KEY, {"sshKey": ssh_key}) @staticmethod def run_ansible_task(stage, yaml_string): @@ -212,7 +246,10 @@ def run_ansible_task(stage, yaml_string): :param object stage: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupActionStage` :param str yaml_string: a yaml encoded string defining the ansibles tasks to run """ - return FMSetupAction(FMSetupActionType.RUN_ANSIBLE_TASKS, {"stage": stage.value, "ansibleTasks": yaml_string }) + return FMSetupAction( + FMSetupActionType.RUN_ANSIBLE_TASKS, + {"stage": stage.value, "ansibleTasks": yaml_string}, + ) @staticmethod def install_system_packages(packages): @@ -221,20 +258,33 @@ def install_system_packages(packages): :param list packages: List of packages to install """ - return FMSetupAction(FMSetupActionType.INSTALL_SYSTEM_PACKAGES, {"packages": packages }) + return FMSetupAction( + FMSetupActionType.INSTALL_SYSTEM_PACKAGES, {"packages": packages} + ) @staticmethod - def setup_advanced_security(basic_headers = True, hsts = False): + def setup_advanced_security(basic_headers=True, hsts=False): """ Return an SETUP_ADVANCED_SECURITY FMSetupAction :param boolean basic_headers: Optional, Prevent browsers to render Web content served by DSS to be embedded into a frame, iframe, embed or object tag. Defaults to True :param boolean hsts: Optional, Enforce HTTP Strict Transport Security. Defaults to False """ - return FMSetupAction(FMSetupActionType.SETUP_ADVANCED_SECURITY, {"basic_headers": basic_headers, "hsts": hsts}) + return FMSetupAction( + FMSetupActionType.SETUP_ADVANCED_SECURITY, + {"basic_headers": basic_headers, "hsts": hsts}, + ) @staticmethod - def install_jdbc_driver(database_type, url, paths_in_archive=None, http_headers=None, http_username=None, http_password=None, datadir_subdirectory=None): + def install_jdbc_driver( + database_type, + url, + paths_in_archive=None, + http_headers=None, + http_username=None, + http_password=None, + datadir_subdirectory=None, + ): """ Return a INSTALL_JDBC_DRIVER FMSetupAction @@ -246,7 +296,18 @@ def install_jdbc_driver(database_type, url, paths_in_archive=None, http_headers= :param str http_password: Optional, If the HTTP(S) endpoint expect a Basic Authentication, add here the password. To authenticate with a SAS Token on Azure Blob Storage (not recommended), store the token in this field. :param str datadir_subdirectory: Optional, Some drivers are shipped with a high number of JAR files along with them. In that case, you might want to install them under an additional level in the DSS data directory. Set the name of this subdirectory here. Not required for most drivers. """ - return FMSetupAction(FMSetupActionType.INSTALL_JDBC_DRIVER, {"url": url, "dbType": database_type.value, "pathsInArchive": paths_in_archive, "headers": http_headers, "username": http_username, "password": http_password, "subpathInDatadir": datadir_subdirectory}) + return FMSetupAction( + FMSetupActionType.INSTALL_JDBC_DRIVER, + { + "url": url, + "dbType": database_type.value, + "pathsInArchive": paths_in_archive, + "headers": http_headers, + "username": http_username, + "password": http_password, + "subpathInDatadir": datadir_subdirectory, + }, + ) @staticmethod def setup_k8s_and_spark(): @@ -255,30 +316,33 @@ def setup_k8s_and_spark(): """ return FMSetupAction(FMSetupActionType.SETUP_K8S_AND_SPARK) + class FMSetupActionType(Enum): - RUN_ANSIBLE_TASKS="RUN_ANSIBLE_TASKS" - INSTALL_SYSTEM_PACKAGES="INSTALL_SYSTEM_PACKAGES" - INSTALL_DSS_PLUGINS_FROM_STORE="INSTALL_DSS_PLUGINS_FROM_STORE" - SETUP_K8S_AND_SPARK="SETUP_K8S_AND_SPARK" - SETUP_RUNTIME_DATABASE="SETUP_RUNTIME_DATABASE" - SETUP_MUS_AUTOCREATE="SETUP_MUS_AUTOCREATE" - SETUP_UI_CUSTOMIZATION="SETUP_UI_CUSTOMIZATION" - SETUP_MEMORY_SETTINGS="SETUP_MEMORY_SETTINGS" - SETUP_GRAPHICS_EXPORT="SETUP_GRAPHICS_EXPORT" - ADD_AUTHORIZED_KEY="ADD_AUTHORIZED_KEY" - INSTALL_JDBC_DRIVER="INSTALL_JDBC_DRIVER" - SETUP_ADVANCED_SECURITY="SETUP_ADVANCED_SECURITY" + RUN_ANSIBLE_TASKS = "RUN_ANSIBLE_TASKS" + INSTALL_SYSTEM_PACKAGES = "INSTALL_SYSTEM_PACKAGES" + INSTALL_DSS_PLUGINS_FROM_STORE = "INSTALL_DSS_PLUGINS_FROM_STORE" + SETUP_K8S_AND_SPARK = "SETUP_K8S_AND_SPARK" + SETUP_RUNTIME_DATABASE = "SETUP_RUNTIME_DATABASE" + SETUP_MUS_AUTOCREATE = "SETUP_MUS_AUTOCREATE" + SETUP_UI_CUSTOMIZATION = "SETUP_UI_CUSTOMIZATION" + SETUP_MEMORY_SETTINGS = "SETUP_MEMORY_SETTINGS" + SETUP_GRAPHICS_EXPORT = "SETUP_GRAPHICS_EXPORT" + ADD_AUTHORIZED_KEY = "ADD_AUTHORIZED_KEY" + INSTALL_JDBC_DRIVER = "INSTALL_JDBC_DRIVER" + SETUP_ADVANCED_SECURITY = "SETUP_ADVANCED_SECURITY" + class FMSetupActionStage(Enum): - after_dss_startup="after_dss_startup" - after_install="after_install" - before_install="before_install" + after_dss_startup = "after_dss_startup" + after_install = "after_install" + before_install = "before_install" + class FMSetupActionAddJDBCDriverDatabaseType(Enum): - mysql="mysql" - mssql="mssql" - oracle="oracle" - mariadb="mariadb" - snowflake="snowflake" - athena="athena" - bigquery="bigquery" + mysql = "mysql" + mssql = "mssql" + oracle = "oracle" + mariadb = "mariadb" + snowflake = "snowflake" + athena = "athena" + bigquery = "bigquery" From 1cc61b75407522c82fcfe6bfc0d1ab9199394f17 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 21 Sep 2021 18:19:11 +0200 Subject: [PATCH 27/38] fm: FMVirtualNetworkCreator --- dataikuapi/fm/virtualnetworks.py | 182 +++++++++++++++++++++++++++---- dataikuapi/fmclient.py | 53 ++------- 2 files changed, 168 insertions(+), 67 deletions(-) diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index b1748d0f..ca5a3ced 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -1,17 +1,123 @@ from dataikuapi.fm.future import FMFuture + +class FMVirtualNetworkCreator(object): + def __init__(self, client, label): + """ + A builder class to create a Virtual Network + + :param str label: The label of the Virtual Network + """ + self.client = client + self.data = {} + self.data["label"] = label + self.data["mode"] = "EXISTING_MONOTENANT" + + def with_internet_access_mode(self, internet_access_mode): + """ + :param str internet_access_mode: The internet access mode of the instances created in this virtual network. Accepts "YES", "NO", "EGRESS_ONLY". Defaults to "YES" + """ + if internet_access_mode not in ["YES", "NO", "EGRESS_ONLY"]: + raise ValueError( + 'internet_access_mode should be either "YES", "NO", or "EGRESS_ONLY"' + ) + + self.data["internetAccessMode"] = internet_access_mode + return self + + +class FMAWSVirtualNetworkCreator(FMVirtualNetworkCreator): + def with_vpc(self, aws_vpc_id, aws_subnet_id): + """ + Setup the VPC and Subnet to with the VirtualNetwork + + :param str aws_vpc_id: ID of the VPC to use + :param str aws_subnet_id: ID of the subnet to use + """ + self.data["awsVpcId"] = aws_vpc_id + self.data["awsSubnetId"] = aws_subnet_id + return self + + def with_auto_create_security_groups(self): + """ + Automatically create the AWS Security Groups when creating this VirtualNetwork + """ + self.data["awsAutoCreateSecurityGroups"] = True + return self + + def with_aws_security_groups(self, aws_security_groups): + """ + Use pre-created AWS Security Groups + + :param list aws_security_groups: A list of up to 5 security group ids to assign to the instances created in this virtual network. + """ + self.data["awsAutoCreateSecurityGroups"] = False + self.data["awsSecurityGroups"] = aws_security_groups + return self + + def create(self): + """ + Create the VirtualNetwork + + :return: Created VirtualNetwork + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAWSVirtualNetwork` + """ + vn = self.client._perform_tenant_json( + "POST", "/virtual-networks", body=self.data + ) + return FMAWSVirtualNetwork(self, vn) + + +class FMAzureVirtualNetworkCreator(FMVirtualNetworkCreator): + def with_virtual_network(self, azure_vn_id, azure_subnet_id): + """ + Setup the Azure Virtual Network and Subnet to with the VirtualNetwork + + :param str azure_vn_id: Resource ID of the Azure Virtual Network to use + :param str azure_subnet_id: Resource ID of the subnet to use + """ + self.data["azureVnId"] = azure_vn_id + self.data["azureSubnetId"] = azure_subnet_id + return self + + def with_auto_update_security_groups(self, auto_update_security_groups=True): + """ + Auto update the security groups of the Azure Virtual Network + + :param boolean auto_update_security_groups: Optional, Auto update the subnet security group. Defaults to True + """ + self.data["azureAutoUpdateSecurityGroups"] = auto_update_security_groups + return self + + def create(self): + """ + Create the VirtualNetwork + + :return: Created VirtualNetwork + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAzureVirtualNetwork` + """ + vn = self.client._perform_tenant_json( + "POST", "/virtual-networks", body=self.data + ) + return FMAzureVirtualNetwork(self, vn) + + class FMVirtualNetwork(object): def __init__(self, client, vn_data): - self.client = client + self.client = client self.vn_data = vn_data - self.id = self.vn_data['id'] + self.id = self.vn_data["id"] def save(self): """ Update the Virtual Network. """ - self.client._perform_tenant_empty("PUT", "/virtual-networks/%s" % self.id, body=self.vn_data) - self.vn_data = self.client._perform_tenant_json("GET", "/virtual-networks/%s" % self.id) + self.client._perform_tenant_empty( + "PUT", "/virtual-networks/%s" % self.id, body=self.vn_data + ) + self.vn_data = self.client._perform_tenant_json( + "GET", "/virtual-networks/%s" % self.id + ) def delete(self): """ @@ -20,10 +126,14 @@ def delete(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("DELETE", "/virtual-networks/%s" % self.id) + future = self.client._perform_tenant_json( + "DELETE", "/virtual-networks/%s" % self.id + ) return FMFuture.from_resp(self.client, future) - def set_fleet_management(self, enable, event_server=None, deployer_management="NO_MANAGED_DEPLOYER"): + def set_fleet_management( + self, enable, event_server=None, deployer_management="NO_MANAGED_DEPLOYER" + ): """ When enabled, all instances in this virtual network know each other and can centrally manage deployer and logs centralization @@ -36,40 +146,64 @@ def set_fleet_management(self, enable, event_server=None, deployer_management="N - "EACH_DESIGN_NODE": Deployer from design. Recommanded if you have a single design node and want a simpler setup. """ - self.vn_data['managedNodesDirectory'] = enable - self.vn_data['eventServerNodeLabel'] = event_server - self.vn_data['nodesDirectoryDeployerMode'] = deployer_management + self.vn_data["managedNodesDirectory"] = enable + self.vn_data["eventServerNodeLabel"] = event_server + self.vn_data["nodesDirectoryDeployerMode"] = deployer_management + self.save() + + def set_https_strategy(self, https_strategy): + """ + Set the HTTPS strategy for this virtual network + + :param object: a :class:`dataikuapi.fm.virtualnetworks.FMHTTPSStrategy` + """ + self.vn_data.update(https_strategy) self.save() - def set_dns_strategy(self, assign_domain_name, aws_private_ip_zone53_id=None, aws_public_ip_zone53_id=None, azure_dns_zone_id=None): + +class FMAWSVirtualNetwork(FMVirtualNetwork): + def set_dns_strategy( + self, + assign_domain_name, + aws_private_ip_zone53_id=None, + aws_public_ip_zone53_id=None, + ): """ Set the DNS strategy for this virtual network :param boolean assign_domain_name: If false, don't assign domain names, use ip_only :param str aws_private_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for private ip :param str aws_public_ip_zone53_id: Optional, AWS Only, the ID of the AWS Route53 Zone to use for public ip - :param str azure_dns_zone_id: Optional, Azure Only, the ID of the Azure DNS zone to use """ if assign_domain_name: - self.vn_data['dnsStrategy'] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" - self.vn_data['awsRoute53PrivateIPZoneId'] = aws_private_ip_zone53_id - self.vn_data['awsRoute53PublicIPZoneId'] = aws_public_ip_zone53_id - self.vn_data['azureDnsZoneId'] = azure_dns_zone_id - else : - self.vn_data['dnsStrategy'] = "NONE" + self.vn_data["dnsStrategy"] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" + self.vn_data["awsRoute53PrivateIPZoneId"] = aws_private_ip_zone53_id + self.vn_data["awsRoute53PublicIPZoneId"] = aws_public_ip_zone53_id + else: + self.vn_data["dnsStrategy"] = "NONE" self.save() - def set_https_strategy(self, https_strategy): + +class FMAzureVirtualNetwork(FMVirtualNetwork): + def set_dns_strategy(self, assign_domain_name, azure_dns_zone_id=None): """ - Set the HTTPS strategy for this virtual network + Set the DNS strategy for this virtual network - :param object: a :class:`dataikuapi.fm.virtualnetworks.FMHTTPSStrategy` + :param boolean assign_domain_name: If false, don't assign domain names, use ip_only + :param str azure_dns_zone_id: Optional, Azure Only, the ID of the Azure DNS zone to use """ - self.vn_data.update(https_strategy) + + if assign_domain_name: + self.vn_data["dnsStrategy"] = "VN_SPECIFIC_CLOUD_DNS_SERVICE" + self.vn_data["azureDnsZoneId"] = azure_dns_zone_id + else: + self.vn_data["dnsStrategy"] = "NONE" + self.save() + class FMHTTPSStrategy(dict): def __init__(self, data, https_strategy, http_redirect=False): """ @@ -82,11 +216,11 @@ def __init__(self, data, https_strategy, http_redirect=False): - :meth:`dataikuapi.fm.virtualnetwork.FMHTTPSStrategy.lets_encrypt` to use Let's Encrypt """ super(FMHTTPSStrategy, self).__init__(data) - self['httpsStrategy'] = https_strategy + self["httpsStrategy"] = https_strategy if http_redirect: - self['httpStrategy'] = "REDIRECT" + self["httpStrategy"] = "REDIRECT" else: - self['httpStrategy'] = "DISABLE" + self["httpStrategy"] = "DISABLE" @staticmethod def disable(): diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 2857c406..0dbef6fc 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -8,7 +8,7 @@ from .utils import DataikuException from .fm.tenant import FMCloudCredentials -from .fm.virtualnetworks import FMVirtualNetwork +from .fm.virtualnetworks import FMVirtualNetwork, FMAWSVirtualNetworkCreator, FMAzureVirtualNetworkCreator from .fm.instances import FMInstance, FMInstanceEncryptionMode, FMAWSInstanceCreator, FMAzureInstanceCreator from .fm.instancesettingstemplates import FMInstanceSettingsTemplate, FMAWSInstanceSettingsTemplateCreator, FMAzureInstanceSettingsTemplateCreator @@ -85,51 +85,18 @@ def get_virtual_network(self, virtual_network_id): vn = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) return FMVirtualNetwork(self, vn) - def create_virtual_network(self, - label, - awsVpcId=None, - awsSubnetId=None, - awsAutoCreateSecurityGroups=False, - awsSecurityGroups=None, - azureVnId=None, - azureSubnetId=None, - azureAutoUpdateSecurityGroups=None, - internetAccessMode = "YES"): - """ - Create a Virtual Network - - :param str label: The label of the Virtual Network - - :param str awsVpcId: AWS Only, ID of the VPC to use - :param str awsSubnetId: AWS Only, ID of the subnet to use - :param boolean awsAutoCreateSecurityGroups: Optional, AWS Only, If false, do not create security groups automatically. Defaults to false - :param list awsSecurityGroups: Optional, AWS Only, A list of up to 5 security group ids to assign to the instances created in this virtual network. Ignored if awsAutoCreateSecurityGroups is true - :param str azureVnId: Azure Only, ID of the Azure Virtual Network to use - :param str azureSubnetId: Azure Only, ID of the subnet to use - :param boolean azureAutoUpdateSecurityGroups: Azure Only, Auto update the subnet security group - - :param str internetAccessMode: Optional, The internet access mode of the instances created in this virtual network. Accepts "YES", "NO", "EGRESS_ONLY". Defaults to "YES" - - :return: requested instance settings template - :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` + def new_virtual_network_creator(self, label): """ + Instantiate a new virtual network creator - data = { - "label": label, - "awsVpcId": awsVpcId, - "awsSubnetId": awsSubnetId, - "awsAutoCreateSecurityGroups": awsAutoCreateSecurityGroups, - "awsSecurityGroups": awsSecurityGroups, - "azureVnId": azureVnId, - "azureSubnetId": azureSubnetId, - "azureAutoUpdateSecurityGroups": azureAutoUpdateSecurityGroups, - "internetAccessMode": internetAccessMode, - "mode": "EXISTING_MONOTENANT" - } - - vn = self._perform_tenant_json("POST", "/virtual-networks", body=data) - return FMVirtualNetwork(self, vn) + :param str label: The label of the + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetworkCreator` + """ + if self.cloud == "AWS": + return FMAWSVirtualNetworkCreator(self, label) + elif self.cloud == "Azure": + return FMAzureVirtualNetworkCreator(self, label) ######################################################## From 6a5ee567548a7af12facb0a8d93173b6887fa62e Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 21 Sep 2021 18:21:25 +0200 Subject: [PATCH 28/38] FM: format --- dataikuapi/fm/future.py | 62 +++++++++------ dataikuapi/fm/instances.py | 88 +++++++++++++++------ dataikuapi/fm/tenant.py | 36 ++++++--- dataikuapi/fmclient.py | 156 ++++++++++++++++++++++++++++--------- 4 files changed, 247 insertions(+), 95 deletions(-) diff --git a/dataikuapi/fm/future.py b/dataikuapi/fm/future.py index 60d31b57..c52b9a04 100644 --- a/dataikuapi/fm/future.py +++ b/dataikuapi/fm/future.py @@ -1,29 +1,35 @@ import sys, time + class FMFuture(object): """ A future on the DSS instance """ - def __init__(self, client, job_id, state=None, result_wrapper=lambda result: result): - self.client = client - self.job_id = job_id - self.state = state - self.state_is_peek = True - self.result_wrapper = result_wrapper + + def __init__( + self, client, job_id, state=None, result_wrapper=lambda result: result + ): + self.client = client + self.job_id = job_id + self.state = state + self.state_is_peek = True + self.result_wrapper = result_wrapper @staticmethod - def from_resp(client, resp,result_wrapper=lambda result: result): + def from_resp(client, resp, result_wrapper=lambda result: result): """Creates a DSSFuture from a parsed JSON response""" - return FMFuture(client, resp.get('jobId', None), state=resp, result_wrapper=result_wrapper) + return FMFuture( + client, resp.get("jobId", None), state=resp, result_wrapper=result_wrapper + ) @classmethod def get_result_wait_if_needed(cls, client, ret): - if 'jobId' in ret: + if "jobId" in ret: future = FMFuture(client, ret["jobId"], ret) future.wait_for_result() return future.get_result() else: - return ret['result'] + return ret["result"] def abort(self): """ @@ -36,7 +42,8 @@ def get_state(self): Get the status of the future, and its result if it's ready """ self.state = self.client._perform_tenant_json( - "GET", "/futures/%s" % self.job_id, params={'peek' : False}) + "GET", "/futures/%s" % self.job_id, params={"peek": False} + ) self.state_is_peek = False return self.state @@ -45,7 +52,8 @@ def peek_state(self): Get the status of the future, and its result if it's ready """ self.state = self.client._perform_tenant_json( - "GET", "/futures/%s" % self.job_id, params={'peek' : True}) + "GET", "/futures/%s" % self.job_id, params={"peek": True} + ) self.state_is_peek = True return self.state @@ -53,10 +61,14 @@ def get_result(self): """ Get the future result if it's ready, raises an Exception otherwise """ - if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: + if ( + self.state is None + or not self.state.get("hasResult", False) + or self.state_is_peek + ): self.get_state() - if self.state.get('hasResult', False): - return self.result_wrapper(self.state.get('result', None)) + if self.state.get("hasResult", False): + return self.result_wrapper(self.state.get("result", None)) else: raise Exception("Result not ready") @@ -64,22 +76,26 @@ def has_result(self): """ Checks whether the future has a result ready """ - if self.state is None or not self.state.get('hasResult', False): + if self.state is None or not self.state.get("hasResult", False): self.get_state() - return self.state.get('hasResult', False) + return self.state.get("hasResult", False) def wait_for_result(self): """ Wait and get the future result """ - if self.state.get('hasResult', False): - return self.result_wrapper(self.state.get('result', None)) - if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: + if self.state.get("hasResult", False): + return self.result_wrapper(self.state.get("result", None)) + if ( + self.state is None + or not self.state.get("hasResult", False) + or self.state_is_peek + ): self.get_state() - while not self.state.get('hasResult', False): + while not self.state.get("hasResult", False): time.sleep(5) self.get_state() - if self.state.get('hasResult', False): - return self.result_wrapper(self.state.get('result', None)) + if self.state.get("hasResult", False): + return self.result_wrapper(self.state.get("result", None)) else: raise Exception("No result") diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index fec296d1..8e70c543 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -1,8 +1,11 @@ from enum import Enum from .future import FMFuture + class FMInstanceCreator(object): - def __init__(self, client, label, instance_settings_template_id, virtual_network_id, image_id): + def __init__( + self, client, label, instance_settings_template_id, virtual_network_id, image_id + ): """ Helper to create a DSS Instance @@ -30,7 +33,9 @@ def with_dss_node_type(self, dss_node_type): :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if dss_node_type not in ["design", "automation", "deployer"]: - raise ValueError("Only \"design\", \"automation\" or \"deployer\" dss_node_type are supported") + raise ValueError( + 'Only "design", "automation" or "deployer" dss_node_type are supported' + ) self.data["dssNodeType"] = dss_node_type return self @@ -44,7 +49,15 @@ def with_cloud_instance_type(self, cloud_instance_type): self.data["cloudInstanceType"] = cloud_instance_type return self - def with_data_volume_options(self, data_volume_type=None, data_volume_size=None, data_volume_size_max=None, data_volume_IOPS=None, data_volume_encryption=None, data_volume_encryption_key=None): + def with_data_volume_options( + self, + data_volume_type=None, + data_volume_size=None, + data_volume_size_max=None, + data_volume_IOPS=None, + data_volume_encryption=None, + data_volume_encryption_key=None, + ): """ Set the options of the data volume to use with the DSS Instance @@ -57,7 +70,9 @@ def with_data_volume_options(self, data_volume_type=None, data_volume_size=None, :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if type(data_volume_encryption) is not FMInstanceEncryptionMode: - raise TypeError("data_volume encryption needs to be of type FMInstanceEncryptionMode") + raise TypeError( + "data_volume encryption needs to be of type FMInstanceEncryptionMode" + ) self.data["dataVolumeType"] = data_volume_type self.data["dataVolumeSizeGB"] = data_volume_size @@ -89,7 +104,12 @@ def with_fm_tags(self, fm_tags): class FMAWSInstanceCreator(FMInstanceCreator): - def with_aws_root_volume_options(self, aws_root_volume_size=None, aws_root_volume_type=None, aws_root_volume_IOPS=None): + def with_aws_root_volume_options( + self, + aws_root_volume_size=None, + aws_root_volume_type=None, + aws_root_volume_IOPS=None, + ): """ Set the options of the root volume of the DSS Instance @@ -110,7 +130,9 @@ def create(self): :return: Created DSS Instance :rtype: :class:`dataikuapi.fm.instances.FMAWSInstance` """ - instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) + instance = self.client._perform_tenant_json( + "POST", "/instances", body=self.data + ) return FMAWSInstance(self.client, instance) @@ -122,7 +144,9 @@ def create(self): :return: Created DSS Instance :rtype: :class:`dataikuapi.fm.instances.FMAzureInstance` """ - instance = self.client._perform_tenant_json("POST", "/instances", body=self.data) + instance = self.client._perform_tenant_json( + "POST", "/instances", body=self.data + ) return FMAzureInstance(self.client, instance) @@ -131,10 +155,11 @@ class FMInstance(object): A handle to interact with a DSS instance. Do not create this directly, use :meth:`FMClient.get_instance` or :meth: `FMClient.new_instance_creator` """ + def __init__(self, client, instance_data): - self.client = client + self.client = client self.instance_data = instance_data - self.id = instance_data['id'] + self.id = instance_data["id"] def reprovision(self): """ @@ -143,7 +168,9 @@ def reprovision(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the reprovision process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("GET", "/instances/%s/actions/reprovision" % self.id) + future = self.client._perform_tenant_json( + "GET", "/instances/%s/actions/reprovision" % self.id + ) return FMFuture.from_resp(self.client, future) def deprovision(self): @@ -153,7 +180,9 @@ def deprovision(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deprovision process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("GET", "/instances/%s/actions/deprovision" % self.id) + future = self.client._perform_tenant_json( + "GET", "/instances/%s/actions/deprovision" % self.id + ) return FMFuture.from_resp(self.client, future) def restart_dss(self): @@ -163,21 +192,29 @@ def restart_dss(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the restart process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("GET", "/instances/%s/actions/restart-dss" % self.id) + future = self.client._perform_tenant_json( + "GET", "/instances/%s/actions/restart-dss" % self.id + ) return FMFuture.from_resp(self.client, future) def save(self): """ Update the Instance. """ - self.client._perform_tenant_empty("PUT", "/instances/%s" % self.id, body=self.instance_data) - self.instance_data = self.client._perform_tenant_json("GET", "/instances/%s" % self.id) + self.client._perform_tenant_empty( + "PUT", "/instances/%s" % self.id, body=self.instance_data + ) + self.instance_data = self.client._perform_tenant_json( + "GET", "/instances/%s" % self.id + ) def get_status(self): """ Get the physical DSS instance's status """ - status = self.client._perform_tenant_json("GET", "/instances/%s/status" % self.id) + status = self.client._perform_tenant_json( + "GET", "/instances/%s/status" % self.id + ) return FMInstanceStatus(status) def delete(self): @@ -187,7 +224,9 @@ def delete(self): :return: A :class:`~dataikuapi.fm.future.FMFuture` representing the deletion process :rtype: :class:`~dataikuapi.fm.future.FMFuture` """ - future = self.client._perform_tenant_json("GET", "/instances/%s/actions/delete" % self.id) + future = self.client._perform_tenant_json( + "GET", "/instances/%s/actions/delete" % self.id + ) return FMFuture.from_resp(self.client, future) def set_automated_snapshots(self, enable, period, keep=0): @@ -198,9 +237,9 @@ def set_automated_snapshots(self, enable, period, keep=0): :param int period: The time period between 2 snapshot in hours :param int keep: Optional, the number of snapshot to keep. Use 0 to keep all snapshots. Defaults to 0. """ - self.instance_data['enableAutomatedSnapshot'] = enable - self.instance_data['automatedSnapshotPeriod'] = period - self.instance_data['automatedSnapshotRetention'] = keep + self.instance_data["enableAutomatedSnapshot"] = enable + self.instance_data["automatedSnapshotPeriod"] = period + self.instance_data["automatedSnapshotRetention"] = keep self.save() def set_custom_certificate(self, pem_data): @@ -211,7 +250,7 @@ def set_custom_certificate(self, pem_data): param: str pem_data: The SSL certificate """ - self.instance_data['sslCertificatePEM'] = pem_data + self.instance_data["sslCertificatePEM"] = pem_data self.save() @@ -223,8 +262,8 @@ def set_elastic_ip(self, enable, elasticip_allocation_id): :param boolan enable: Enable the elastic ip allocation :param str elaticip_allocation_id: AWS ElasticIP allocation ID """ - self.instance_data['awsAssignElasticIP'] = enable - self.instance_data['awsElasticIPAllocationId'] = elasticip_allocation_id + self.instance_data["awsAssignElasticIP"] = enable + self.instance_data["awsElasticIPAllocationId"] = elasticip_allocation_id self.save() @@ -236,8 +275,8 @@ def set_elastic_ip(self, enable, public_ip_id): :param boolan enable: Enable the elastic ip allocation :param str public_ip_id: Azure Public IP ID """ - self.instance_data['azureAssignElasticIP'] = enable - self.instance_data['azurePublicIPId'] = public_ip_id + self.instance_data["azureAssignElasticIP"] = enable + self.instance_data["azurePublicIPId"] = public_ip_id self.save() @@ -252,6 +291,7 @@ class FMInstanceStatus(dict): """A class holding read-only information about an Instance. This class should not be created directly. Instead, use :meth:`FMInstance.get_info` """ + def __init__(self, data): """Do not call this directly, use :meth:`FMInstance.get_status`""" super(FMInstanceStatus, self).__init__(data) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index 67c97dcd..ee6b1fb5 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -1,14 +1,17 @@ import json + + class FMCloudCredentials(object): """ A Tenant Cloud Credentials in the FM instance """ + def __init__(self, client, cloud_credentials): self.client = client self.cloud_credentials = cloud_credentials def set_cmk_key(self, cmk_key_id): - self.cloud_credentials['awsCMKId'] = cmk_key_id + self.cloud_credentials["awsCMKId"] = cmk_key_id self.save() def set_static_license(self, license_file=None, license_string=None): @@ -26,9 +29,11 @@ def set_static_license(self, license_file=None, license_string=None): elif license_string is not None: license = json.load(license_string) else: - raise ValueError("a valid license_file or license_string needs to be provided") - self.cloud_credentials['licenseMode'] = 'STATIC' - self.cloud_credentials['license'] = json.dumps(license, indent=2) + raise ValueError( + "a valid license_file or license_string needs to be provided" + ) + self.cloud_credentials["licenseMode"] = "STATIC" + self.cloud_credentials["license"] = json.dumps(license, indent=2) self.save() def set_automatically_updated_license(self, license_token): @@ -39,8 +44,8 @@ def set_automatically_updated_license(self, license_token): """ if license_token is None: raise ValueError("a valid license_token needs to be provided") - self.cloud_credentials['licenseMode'] = 'AUTO_UPDATE' - self.cloud_credentials['licenseToken'] = license_token + self.cloud_credentials["licenseMode"] = "AUTO_UPDATE" + self.cloud_credentials["licenseToken"] = license_token self.save() def set_authentication(self, authentication): @@ -55,8 +60,9 @@ def set_authentication(self, authentication): def save(self): """Saves back the settings to the project""" - self.client._perform_tenant_empty("PUT", "/cloud-credentials", - body = self.cloud_credentials) + self.client._perform_tenant_empty( + "PUT", "/cloud-credentials", body=self.cloud_credentials + ) class FMCloudAuthentication(dict): @@ -85,7 +91,9 @@ def aws_iam_role(role_arn): params: str role_arn: ARN of the IAM Role """ - return FMCloudAuthentication({"awsAuthenticationMode": "IAM_ROLE", "awsIAMRoleARN": role_arn}) + return FMCloudAuthentication( + {"awsAuthenticationMode": "IAM_ROLE", "awsIAMRoleARN": role_arn} + ) @staticmethod def aws_keypair(access_key_id, secret_access_key): @@ -95,7 +103,13 @@ def aws_keypair(access_key_id, secret_access_key): :param str access_key_id: AWS Access Key ID :param str secret_access_key: AWS Secret Access Key """ - return FMCloudAuthentication({"awsAuthenticationMode": "KEYPAIR", "awsAccessKeyId": access_key_id, "awsSecretAccessKey": secret_access_key}) + return FMCloudAuthentication( + { + "awsAuthenticationMode": "KEYPAIR", + "awsAccessKeyId": access_key_id, + "awsSecretAccessKey": secret_access_key, + } + ) @staticmethod def azure(subscription, tenant_id, environment, client_id): @@ -111,7 +125,7 @@ def azure(subscription, tenant_id, environment, client_id): "azureSubscription": subscription, "azureTenantId": tenant_id, "azureEnvironment": environment, - "azureFMAppClientId": client_id + "azureFMAppClientId": client_id, } return FMCloudAuthentication(data) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 0dbef6fc..b2ec0732 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -8,14 +8,36 @@ from .utils import DataikuException from .fm.tenant import FMCloudCredentials -from .fm.virtualnetworks import FMVirtualNetwork, FMAWSVirtualNetworkCreator, FMAzureVirtualNetworkCreator -from .fm.instances import FMInstance, FMInstanceEncryptionMode, FMAWSInstanceCreator, FMAzureInstanceCreator -from .fm.instancesettingstemplates import FMInstanceSettingsTemplate, FMAWSInstanceSettingsTemplateCreator, FMAzureInstanceSettingsTemplateCreator +from .fm.virtualnetworks import ( + FMVirtualNetwork, + FMAWSVirtualNetworkCreator, + FMAzureVirtualNetworkCreator, +) +from .fm.instances import ( + FMInstance, + FMInstanceEncryptionMode, + FMAWSInstanceCreator, + FMAzureInstanceCreator, +) +from .fm.instancesettingstemplates import ( + FMInstanceSettingsTemplate, + FMAWSInstanceSettingsTemplateCreator, + FMAzureInstanceSettingsTemplateCreator, +) + class FMClient(object): """Entry point for the FM API client""" - def __init__(self, host, api_key_id, api_key_secret, cloud, tenant_id="main", extra_headers=None): + def __init__( + self, + host, + api_key_id, + api_key_secret, + cloud, + tenant_id="main", + extra_headers=None, + ): """ Instantiate a new FM API client on the given host with the given API key. @@ -30,7 +52,7 @@ def __init__(self, host, api_key_id, api_key_secret, cloud, tenant_id="main", ex self.api_key_secret = api_key_secret self.host = host if cloud not in ["AWS", "Azure"]: - raise ValueError("cloud should be either \"AWS\" or \"Azure\"") + raise ValueError('cloud should be either "AWS" or "Azure"') self.cloud = cloud self.__tenant_id = tenant_id self._session = Session() @@ -43,7 +65,6 @@ def __init__(self, host, api_key_id, api_key_secret, cloud, tenant_id="main", ex if extra_headers is not None: self._session.headers.update(extra_headers) - ######################################################## # Tenant ######################################################## @@ -58,7 +79,6 @@ def get_cloud_credentials(self): creds = self._perform_tenant_json("GET", "/cloud-credentials") return FMCloudCredentials(self, creds) - ######################################################## # VirtualNetwork ######################################################## @@ -71,7 +91,7 @@ def list_virtual_networks(self): :rtype: list of :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ vns = self._perform_tenant_json("GET", "/virtual-networks") - return [ FMVirtualNetwork(self, x) for x in vns] + return [FMVirtualNetwork(self, x) for x in vns] def get_virtual_network(self, virtual_network_id): """ @@ -82,10 +102,11 @@ def get_virtual_network(self, virtual_network_id): :return: requested virtual network :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ - vn = self._perform_tenant_json("GET", "/virtual-networks/%s" % virtual_network_id) + vn = self._perform_tenant_json( + "GET", "/virtual-networks/%s" % virtual_network_id + ) return FMVirtualNetwork(self, vn) - def new_virtual_network_creator(self, label): """ Instantiate a new virtual network creator @@ -98,7 +119,6 @@ def new_virtual_network_creator(self, label): elif self.cloud == "Azure": return FMAzureVirtualNetworkCreator(self, label) - ######################################################## # Instance settings template ######################################################## @@ -111,7 +131,7 @@ def list_instance_templates(self): :rtype: list of :class:`dataikuapi.fm.tenant.FMInstanceSettingsTemplate` """ templates = self._perform_tenant_json("GET", "/instance-settings-templates") - return [ FMInstanceSettingsTemplate(self, x) for x in templates] + return [FMInstanceSettingsTemplate(self, x) for x in templates] def get_instance_template(self, template_id): """ @@ -122,7 +142,9 @@ def get_instance_template(self, template_id): :return: requested instance settings template :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplate` """ - template = self._perform_tenant_json("GET", "/instance-settings-templates/%s" % template_id) + template = self._perform_tenant_json( + "GET", "/instance-settings-templates/%s" % template_id + ) return FMInstanceSettingsTemplate(self, template) def new_instance_template_creator(self, label): @@ -137,7 +159,6 @@ def new_instance_template_creator(self, label): elif self.cloud == "Azure": return FMAzureInstanceSettingsTemplateCreator(self, label) - ######################################################## # Instance ######################################################## @@ -150,7 +171,7 @@ def list_instances(self): :rtype: list of :class:`dataikuapi.fm.instances.FMInstance` """ instances = self._perform_tenant_json("GET", "/instances") - return [ FMInstance(self, **x) for x in instances] + return [FMInstance(self, **x) for x in instances] def get_instance(self, instance_id): """ @@ -164,7 +185,9 @@ def get_instance(self, instance_id): instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return FMInstance(self, instance) - def new_instance_creator(self, label, instance_settings_template_id, virtual_network_id, image_id): + def new_instance_creator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ): """ Instantiate a new instance creator @@ -175,26 +198,41 @@ def new_instance_creator(self, label, instance_settings_template_id, virtual_net :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` """ if self.cloud == "AWS": - return FMAWSInstanceCreator(self, label, instance_settings_template_id, virtual_network_id, image_id) + return FMAWSInstanceCreator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ) elif self.cloud == "Azure": - return FMAzureInstanceCreator(self, label, instance_settings_template_id, virtual_network_id, image_id) - + return FMAzureInstanceCreator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ) ######################################################## # Internal Request handling ######################################################## - def _perform_http(self, method, path, params=None, body=None, stream=False, files=None, raw_body=None): + def _perform_http( + self, + method, + path, + params=None, + body=None, + stream=False, + files=None, + raw_body=None, + ): if body is not None: body = json.dumps(body) if raw_body is not None: body = raw_body try: http_res = self._session.request( - method, "%s/api/public%s" % (self.host, path), - params=params, data=body, - files = files, - stream = stream) + method, + "%s/api/public%s" % (self.host, path), + params=params, + data=body, + files=files, + stream=stream, + ) http_res.raise_for_status() return http_res except exceptions.HTTPError: @@ -202,16 +240,60 @@ def _perform_http(self, method, path, params=None, body=None, stream=False, file ex = http_res.json() except ValueError: ex = {"message": http_res.text} - raise DataikuException("%s: %s" % (ex.get("errorType", "Unknown error"), ex.get("message", "No message"))) - - def _perform_empty(self, method, path, params=None, body=None, files = None, raw_body=None): - self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body) - - def _perform_json(self, method, path, params=None, body=None,files=None, raw_body=None): - return self._perform_http(method, path, params=params, body=body, files=files, stream=False, raw_body=raw_body).json() - - def _perform_tenant_json(self, method, path, params=None, body=None,files=None, raw_body=None): - return self._perform_json(method, "/tenants/%s%s" % ( self.__tenant_id, path ), params=params, body=body, files=files, raw_body=raw_body) - - def _perform_tenant_empty(self, method, path, params=None, body=None, files = None, raw_body=None): - self._perform_empty(method, "/tenants/%s%s" % ( self.__tenant_id, path ), params=params, body=body, files=files, raw_body=raw_body) + raise DataikuException( + "%s: %s" + % ( + ex.get("errorType", "Unknown error"), + ex.get("message", "No message"), + ) + ) + + def _perform_empty( + self, method, path, params=None, body=None, files=None, raw_body=None + ): + self._perform_http( + method, + path, + params=params, + body=body, + files=files, + stream=False, + raw_body=raw_body, + ) + + def _perform_json( + self, method, path, params=None, body=None, files=None, raw_body=None + ): + return self._perform_http( + method, + path, + params=params, + body=body, + files=files, + stream=False, + raw_body=raw_body, + ).json() + + def _perform_tenant_json( + self, method, path, params=None, body=None, files=None, raw_body=None + ): + return self._perform_json( + method, + "/tenants/%s%s" % (self.__tenant_id, path), + params=params, + body=body, + files=files, + raw_body=raw_body, + ) + + def _perform_tenant_empty( + self, method, path, params=None, body=None, files=None, raw_body=None + ): + self._perform_empty( + method, + "/tenants/%s%s" % (self.__tenant_id, path), + params=params, + body=body, + files=files, + raw_body=raw_body, + ) From 52f58fbcd41a245aa3bffcdf512165e4f151e9d2 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 21 Sep 2021 20:26:27 +0200 Subject: [PATCH 29/38] FM: Split client by cloud --- dataikuapi/__init__.py | 2 +- dataikuapi/fm/virtualnetworks.py | 2 +- dataikuapi/fmclient.py | 182 +++++++++++++++++++++---------- 3 files changed, 125 insertions(+), 61 deletions(-) diff --git a/dataikuapi/__init__.py b/dataikuapi/__init__.py index 569b1017..5c4337bb 100644 --- a/dataikuapi/__init__.py +++ b/dataikuapi/__init__.py @@ -1,5 +1,5 @@ from .dssclient import DSSClient -from .fmclient import FMClient +from .fmclient import FMClientAWS, FMClientAzure from .apinode_client import APINodeClient from .apinode_admin_client import APINodeAdminClient diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index ca5a3ced..9b98173b 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -69,7 +69,7 @@ def create(self): class FMAzureVirtualNetworkCreator(FMVirtualNetworkCreator): - def with_virtual_network(self, azure_vn_id, azure_subnet_id): + def with_azure_virtual_network(self, azure_vn_id, azure_subnet_id): """ Setup the Azure Virtual Network and Subnet to with the VirtualNetwork diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index b2ec0732..404cd016 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -27,33 +27,25 @@ class FMClient(object): - """Entry point for the FM API client""" - def __init__( self, host, api_key_id, api_key_secret, - cloud, tenant_id="main", extra_headers=None, ): """ - Instantiate a new FM API client on the given host with the given API key. - - API keys can be managed in FM on the project page or in the global settings. - - The API key will define which operations are allowed for the client. - - :param str host: Full url of the FM - + Base class for the different FM Clients + Do not create this class, instead use :class:`dataikuapi.FMClientAWS` or :class:`dataikuapi.FMClientAzure` """ + if self.cloud == None: + raise NotImplementedError( + "Do not use FMClient directly, instead use FMClientAWS or FMClientAzure" + ) self.api_key_id = api_key_id self.api_key_secret = api_key_secret self.host = host - if cloud not in ["AWS", "Azure"]: - raise ValueError('cloud should be either "AWS" or "Azure"') - self.cloud = cloud self.__tenant_id = tenant_id self._session = Session() @@ -107,18 +99,6 @@ def get_virtual_network(self, virtual_network_id): ) return FMVirtualNetwork(self, vn) - def new_virtual_network_creator(self, label): - """ - Instantiate a new virtual network creator - - :param str label: The label of the - :rtype: :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetworkCreator` - """ - if self.cloud == "AWS": - return FMAWSVirtualNetworkCreator(self, label) - elif self.cloud == "Azure": - return FMAzureVirtualNetworkCreator(self, label) - ######################################################## # Instance settings template ######################################################## @@ -147,18 +127,6 @@ def get_instance_template(self, template_id): ) return FMInstanceSettingsTemplate(self, template) - def new_instance_template_creator(self, label): - """ - Instantiate a new instance template creator - - :param str label: The label of the instance - :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMInstanceSettingsTemplateCreator` - """ - if self.cloud == "AWS": - return FMAWSInstanceSettingsTemplateCreator(self, label) - elif self.cloud == "Azure": - return FMAzureInstanceSettingsTemplateCreator(self, label) - ######################################################## # Instance ######################################################## @@ -185,27 +153,6 @@ def get_instance(self, instance_id): instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) return FMInstance(self, instance) - def new_instance_creator( - self, label, instance_settings_template_id, virtual_network_id, image_id - ): - """ - Instantiate a new instance creator - - :param str label: The label of the instance - :param str instance_settings_template: The instance settings template id this instance should be based on - :param str virtual_network: The virtual network where the instance should be spawned - :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) - :rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator` - """ - if self.cloud == "AWS": - return FMAWSInstanceCreator( - self, label, instance_settings_template_id, virtual_network_id, image_id - ) - elif self.cloud == "Azure": - return FMAzureInstanceCreator( - self, label, instance_settings_template_id, virtual_network_id, image_id - ) - ######################################################## # Internal Request handling ######################################################## @@ -297,3 +244,120 @@ def _perform_tenant_empty( files=files, raw_body=raw_body, ) + + +class FMClientAWS(FMClient): + def __init__( + self, + host, + api_key_id, + api_key_secret, + tenant_id="main", + extra_headers=None, + ): + """ + AWS Only - Instantiate a new FM API client on the given host with the given API key. + + API keys can be managed in FM on the project page or in the global settings. + + The API key will define which operations are allowed for the client. + + :param str host: Full url of the FM + + """ + self.cloud = "AWS" + super(FMClientAWS, self).__init__( + host, api_key_id, api_key_secret, tenant_id, extra_headers + ) + + def new_virtual_network_creator(self, label): + """ + Instantiate a new virtual network creator + + :param str label: The label of the + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAWSVirtualNetworkCreator` + """ + return FMAWSVirtualNetworkCreator(self, label) + + def new_instance_template_creator(self, label): + """ + Instantiate a new instance template creator + + :param str label: The label of the instance + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAWSInstanceSettingsTemplateCreator` + """ + return FMAWSInstanceSettingsTemplateCreator(self, label) + + def new_instance_creator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ): + """ + Instantiate a new instance creator + + :param str label: The label of the instance + :param str instance_settings_template: The instance settings template id this instance should be based on + :param str virtual_network: The virtual network where the instance should be spawned + :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) + :rtype: :class:`dataikuapi.fm.instances.FMAWSInstanceCreator` + """ + return FMAWSInstanceCreator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ) + + +class FMClientAzure(FMClient): + def __init__( + self, + host, + api_key_id, + api_key_secret, + tenant_id="main", + extra_headers=None, + ): + """ + Azure Only - Instantiate a new FM API client on the given host with the given API key. + + API keys can be managed in FM on the project page or in the global settings. + + The API key will define which operations are allowed for the client. + + :param str host: Full url of the FM + """ + self.cloud = "Azure" + super(FMClientAWS, self).__init__( + host, api_key_id, api_key_secret, tenant_id, extra_headers + ) + + def new_virtual_network_creator(self, label): + """ + Instantiate a new virtual network creator + + :param str label: The label of the + :rtype: :class:`dataikuapi.fm.virtualnetworks.FMAzureVirtualNetworkCreator` + """ + return FMAzureVirtualNetworkCreator(self, label) + + def new_instance_template_creator(self, label): + """ + Instantiate a new instance template creator + + :param str label: The label of the instance + :rtype: :class:`dataikuapi.fm.instancesettingstemplates.FMAzureInstanceSettingsTemplateCreator` + """ + return FMAzureInstanceSettingsTemplateCreator(self, label) + + def new_instance_creator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ): + """ + Instantiate a new instance creator + + :param str label: The label of the instance + :param str instance_settings_template: The instance settings template id this instance should be based on + :param str virtual_network: The virtual network where the instance should be spawned + :param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default) + :rtype: :class:`dataikuapi.fm.instances.FMAzureInstanceCreator` + """ + return FMAzureInstanceCreator( + self, label, instance_settings_template_id, virtual_network_id, image_id + ) From f4728015353ba88f475d676ba660545e5f3a1ad8 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Tue, 21 Sep 2021 21:48:08 +0200 Subject: [PATCH 30/38] FM: Update Cloud Tags --- dataikuapi/fm/tenant.py | 58 +++++++++++++++++++++++++++++++++++++++++ dataikuapi/fmclient.py | 14 +++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index ee6b1fb5..5bf0499e 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -65,6 +65,64 @@ def save(self): ) +class FMCloudTags(object): + """ + A Tenant Cloud Tags in the FM instance + """ + + def __init__(self, client, tenant_id, cloud_tags): + self.client = client + self.tenant_id = tenant_id + self.cloud_tags = json.loads(cloud_tags["msg"]) + + def add_tag(self, key, value): + """ + Add a tag to the tenant + + + :param str key: Tag key + :param str value: Tag value + """ + if key in self.cloud_tags: + raise Exception("Key already exists") + self.cloud_tags[key] = value + + def update_tag(self, key, new_key=None, new_value=None): + """ + Update a tag key or value + + + :param str key: Key of the tag to update + :param str new_key: Optional, new key for the tag + :param str new_value: Optional, new value for the tag + """ + if key not in self.cloud_tags: + raise Exception("Key does not exists") + if new_value: + self.cloud_tags[key] = new_value + if new_key: + self.cloud_tags[new_key] = self.cloud_tags[key] + del self.cloud_tags[key] + + def delete_tag(self, key): + """ + Delete a tag + + + :param str key: Key of the tag to delete + """ + if key not in self.cloud_tags: + raise Exception("Key does not exists") + del self.cloud_tags[key] + + def save(self): + """Saves the tags on FM""" + + self.client._perform_empty( + "PUT", "/tenants/%s/cloud-tags" % (self.tenant_id), body=self.cloud_tags + ) + + class FMCloudAuthentication(dict): def __init__(self, data): """ diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 404cd016..744bba44 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -7,7 +7,7 @@ from enum import Enum from .utils import DataikuException -from .fm.tenant import FMCloudCredentials +from .fm.tenant import FMCloudCredentials, FMCloudTags from .fm.virtualnetworks import ( FMVirtualNetwork, FMAWSVirtualNetworkCreator, @@ -71,6 +71,18 @@ def get_cloud_credentials(self): creds = self._perform_tenant_json("GET", "/cloud-credentials") return FMCloudCredentials(self, creds) + def get_cloud_tags(self, tenant_id): + """ + Get Tenant's Cloud Tags + + :param string tenant_id + + :return: tenant's cloud tags + :rtype: :class:`dataikuapi.fm.tenant.FMCloudTags` + """ + tags = self._perform_json("GET", "/tenants/%s/cloud-tags" % tenant_id) + return FMCloudTags(self, tenant_id, tags) + ######################################################## # VirtualNetwork ######################################################## From 137f05660a5e52c84a9043933e61e0224bc99b49 Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Wed, 22 Sep 2021 15:47:07 +0200 Subject: [PATCH 31/38] typo --- dataikuapi/fm/tenant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index 5bf0499e..d46b5ac5 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -27,7 +27,7 @@ def set_static_license(self, license_file=None, license_string=None): with open(license_file) as json_file: license = json.load(json_file) elif license_string is not None: - license = json.load(license_string) + license = json.loads(license_string) else: raise ValueError( "a valid license_file or license_string needs to be provided" From 3bf7ae4d1d3cb6a620ef1de664cfad321d5c6fc0 Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Wed, 22 Sep 2021 15:53:53 +0200 Subject: [PATCH 32/38] fix created object --- dataikuapi/fm/virtualnetworks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index 9b98173b..9720a067 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -65,7 +65,7 @@ def create(self): vn = self.client._perform_tenant_json( "POST", "/virtual-networks", body=self.data ) - return FMAWSVirtualNetwork(self, vn) + return FMAWSVirtualNetwork(self.client, vn) class FMAzureVirtualNetworkCreator(FMVirtualNetworkCreator): @@ -99,7 +99,7 @@ def create(self): vn = self.client._perform_tenant_json( "POST", "/virtual-networks", body=self.data ) - return FMAzureVirtualNetwork(self, vn) + return FMAzureVirtualNetwork(self.client, vn) class FMVirtualNetwork(object): From 87bc084ce11a7a5758b83f25e9a9da1076cd06c8 Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Wed, 22 Sep 2021 15:54:11 +0200 Subject: [PATCH 33/38] type returned objects to the appropriate subclass --- dataikuapi/fmclient.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 744bba44..2a6a3d6a 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -12,12 +12,16 @@ FMVirtualNetwork, FMAWSVirtualNetworkCreator, FMAzureVirtualNetworkCreator, + FMAWSVirtualNetwork, + FMAzureVirtualNetwork ) from .fm.instances import ( FMInstance, FMInstanceEncryptionMode, FMAWSInstanceCreator, FMAzureInstanceCreator, + FMAWSInstance, + FMAzureInstance ) from .fm.instancesettingstemplates import ( FMInstanceSettingsTemplate, @@ -87,6 +91,14 @@ def get_cloud_tags(self, tenant_id): # VirtualNetwork ######################################################## + def _make_virtual_network(self, vn): + if self.cloud == 'AWS': + return FMAWSVirtualNetwork(self, vn) + elif self.cloud == 'Azure': + return FMAzureVirtualNetwork(self, vn) + else: + raise Exception("Unknown cloud type %s" % self.cloud) + def list_virtual_networks(self): """ List all Virtual Networks @@ -95,7 +107,7 @@ def list_virtual_networks(self): :rtype: list of :class:`dataikuapi.fm.virtualnetworks.FMVirtualNetwork` """ vns = self._perform_tenant_json("GET", "/virtual-networks") - return [FMVirtualNetwork(self, x) for x in vns] + return [self._make_virtual_network(x) for x in vns] def get_virtual_network(self, virtual_network_id): """ @@ -109,7 +121,7 @@ def get_virtual_network(self, virtual_network_id): vn = self._perform_tenant_json( "GET", "/virtual-networks/%s" % virtual_network_id ) - return FMVirtualNetwork(self, vn) + return self._make_virtual_network(vn) ######################################################## # Instance settings template @@ -143,6 +155,14 @@ def get_instance_template(self, template_id): # Instance ######################################################## + def _make_instance(self, i): + if self.cloud == 'AWS': + return FMAWSInstance(self, i) + elif self.cloud == 'Azure': + return FMAzureInstance(self, i) + else: + raise Exception("Unknown cloud type %s" % self.cloud) + def list_instances(self): """ List all DSS Instances @@ -151,7 +171,7 @@ def list_instances(self): :rtype: list of :class:`dataikuapi.fm.instances.FMInstance` """ instances = self._perform_tenant_json("GET", "/instances") - return [FMInstance(self, **x) for x in instances] + return [self._make_instance(x) for x in instances] def get_instance(self, instance_id): """ @@ -163,7 +183,7 @@ def get_instance(self, instance_id): :rtype: :class:`dataikuapi.fm.instances.FMInstance` """ instance = self._perform_tenant_json("GET", "/instances/%s" % instance_id) - return FMInstance(self, instance) + return self._make_instance(instance) ######################################################## # Internal Request handling From 8dfa95345271b9e84b2740545e0bc919b1277464 Mon Sep 17 00:00:00 2001 From: Frederic Chataigner Date: Wed, 22 Sep 2021 16:06:20 +0200 Subject: [PATCH 34/38] fix client passed to created object --- dataikuapi/fm/instancesettingstemplates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 4fe035c6..6d91d51a 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -26,7 +26,7 @@ def create(self): template = self.client._perform_tenant_json( "POST", "/instance-settings-templates", body=self.data ) - return FMInstanceSettingsTemplate(self, template) + return FMInstanceSettingsTemplate(self.client, template) def with_setup_actions(self, setup_actions): """ From c9c631c5466e98f1e060c37475b799d6d5d54eef Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 23 Sep 2021 11:54:44 +0200 Subject: [PATCH 35/38] FM: Simplify CloudTags --- dataikuapi/fm/tenant.py | 69 +++++++++-------------------------------- dataikuapi/fmclient.py | 18 +++++------ 2 files changed, 23 insertions(+), 64 deletions(-) diff --git a/dataikuapi/fm/tenant.py b/dataikuapi/fm/tenant.py index d46b5ac5..80e45d95 100644 --- a/dataikuapi/fm/tenant.py +++ b/dataikuapi/fm/tenant.py @@ -12,29 +12,27 @@ def __init__(self, client, cloud_credentials): def set_cmk_key(self, cmk_key_id): self.cloud_credentials["awsCMKId"] = cmk_key_id - self.save() + return self - def set_static_license(self, license_file=None, license_string=None): + def set_static_license(self, license_file_path=None, license_string=None): """ Set a default static license for the DSS instances - Requires either a license_file or a license_string - - :param str license_file: Optional, load the license from a json file + :param str license_file_path: Optional, load the license from a json file :param str license_string: Optional, load the license from a json string """ - if license_file is not None: - with open(license_file) as json_file: + if license_file_path is not None: + with open(license_file_path) as json_file: license = json.load(json_file) elif license_string is not None: license = json.loads(license_string) else: raise ValueError( - "a valid license_file or license_string needs to be provided" + "a valid license_file_path or license_string needs to be provided" ) self.cloud_credentials["licenseMode"] = "STATIC" self.cloud_credentials["license"] = json.dumps(license, indent=2) - self.save() + return self def set_automatically_updated_license(self, license_token): """ @@ -46,7 +44,7 @@ def set_automatically_updated_license(self, license_token): raise ValueError("a valid license_token needs to be provided") self.cloud_credentials["licenseMode"] = "AUTO_UPDATE" self.cloud_credentials["licenseToken"] = license_token - self.save() + return self def set_authentication(self, authentication): """ @@ -55,7 +53,7 @@ def set_authentication(self, authentication): :param object: a :class:`dataikuapi.fm.tenant.FMCloudAuthentication` """ self.cloud_credentials.update(authentication) - self.save() + return self def save(self): """Saves back the settings to the project""" @@ -70,57 +68,18 @@ class FMCloudTags(object): A Tenant Cloud Tags in the FM instance """ - def __init__(self, client, tenant_id, cloud_tags): + def __init__(self, client, cloud_tags): self.client = client - self.tenant_id = tenant_id self.cloud_tags = json.loads(cloud_tags["msg"]) - def add_tag(self, key, value): - """ - Add a tag to the tenant - - - :param str key: Tag key - :param str value: Tag value - """ - if key in self.cloud_tags: - raise Exception("Key already exists") - self.cloud_tags[key] = value - - def update_tag(self, key, new_key=None, new_value=None): - """ - Update a tag key or value - - - :param str key: Key of the tag to update - :param str new_key: Optional, new key for the tag - :param str new_value: Optional, new value for the tag - """ - if key not in self.cloud_tags: - raise Exception("Key does not exists") - if new_value: - self.cloud_tags[key] = new_value - if new_key: - self.cloud_tags[new_key] = self.cloud_tags[key] - del self.cloud_tags[key] - - def delete_tag(self, key): - """ - Delete a tag - - - :param str key: Key of the tag to delete - """ - if key not in self.cloud_tags: - raise Exception("Key does not exists") - del self.cloud_tags[key] + @property + def tags(self): + return self.cloud_tags def save(self): """Saves the tags on FM""" - self.client._perform_empty( - "PUT", "/tenants/%s/cloud-tags" % (self.tenant_id), body=self.cloud_tags - ) + self.client._perform_tenant_empty("PUT", "/cloud-tags", body=self.cloud_tags) class FMCloudAuthentication(dict): diff --git a/dataikuapi/fmclient.py b/dataikuapi/fmclient.py index 2a6a3d6a..1b8bc2db 100644 --- a/dataikuapi/fmclient.py +++ b/dataikuapi/fmclient.py @@ -13,7 +13,7 @@ FMAWSVirtualNetworkCreator, FMAzureVirtualNetworkCreator, FMAWSVirtualNetwork, - FMAzureVirtualNetwork + FMAzureVirtualNetwork, ) from .fm.instances import ( FMInstance, @@ -21,7 +21,7 @@ FMAWSInstanceCreator, FMAzureInstanceCreator, FMAWSInstance, - FMAzureInstance + FMAzureInstance, ) from .fm.instancesettingstemplates import ( FMInstanceSettingsTemplate, @@ -75,7 +75,7 @@ def get_cloud_credentials(self): creds = self._perform_tenant_json("GET", "/cloud-credentials") return FMCloudCredentials(self, creds) - def get_cloud_tags(self, tenant_id): + def get_cloud_tags(self): """ Get Tenant's Cloud Tags @@ -84,17 +84,17 @@ def get_cloud_tags(self, tenant_id): :return: tenant's cloud tags :rtype: :class:`dataikuapi.fm.tenant.FMCloudTags` """ - tags = self._perform_json("GET", "/tenants/%s/cloud-tags" % tenant_id) - return FMCloudTags(self, tenant_id, tags) + tags = self._perform_tenant_json("GET", "/cloud-tags") + return FMCloudTags(self, tags) ######################################################## # VirtualNetwork ######################################################## def _make_virtual_network(self, vn): - if self.cloud == 'AWS': + if self.cloud == "AWS": return FMAWSVirtualNetwork(self, vn) - elif self.cloud == 'Azure': + elif self.cloud == "Azure": return FMAzureVirtualNetwork(self, vn) else: raise Exception("Unknown cloud type %s" % self.cloud) @@ -156,9 +156,9 @@ def get_instance_template(self, template_id): ######################################################## def _make_instance(self, i): - if self.cloud == 'AWS': + if self.cloud == "AWS": return FMAWSInstance(self, i) - elif self.cloud == 'Azure': + elif self.cloud == "Azure": return FMAzureInstance(self, i) else: raise Exception("Unknown cloud type %s" % self.cloud) From 601788e709dbf5c25cd4c1096d63a315c20f6303 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 23 Sep 2021 12:38:10 +0200 Subject: [PATCH 36/38] FM: Review feedback on VirtualNetworks --- dataikuapi/fm/virtualnetworks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dataikuapi/fm/virtualnetworks.py b/dataikuapi/fm/virtualnetworks.py index 9720a067..35aae048 100644 --- a/dataikuapi/fm/virtualnetworks.py +++ b/dataikuapi/fm/virtualnetworks.py @@ -45,11 +45,11 @@ def with_auto_create_security_groups(self): self.data["awsAutoCreateSecurityGroups"] = True return self - def with_aws_security_groups(self, aws_security_groups): + def with_aws_security_groups(self, *aws_security_groups): """ Use pre-created AWS Security Groups - :param list aws_security_groups: A list of up to 5 security group ids to assign to the instances created in this virtual network. + :param str *aws_security_groups: Up to 5 security group ids to assign to the instances created in this virtual network. """ self.data["awsAutoCreateSecurityGroups"] = False self.data["awsSecurityGroups"] = aws_security_groups @@ -149,7 +149,7 @@ def set_fleet_management( self.vn_data["managedNodesDirectory"] = enable self.vn_data["eventServerNodeLabel"] = event_server self.vn_data["nodesDirectoryDeployerMode"] = deployer_management - self.save() + return self def set_https_strategy(self, https_strategy): """ @@ -158,7 +158,7 @@ def set_https_strategy(self, https_strategy): :param object: a :class:`dataikuapi.fm.virtualnetworks.FMHTTPSStrategy` """ self.vn_data.update(https_strategy) - self.save() + return self class FMAWSVirtualNetwork(FMVirtualNetwork): @@ -183,7 +183,7 @@ def set_dns_strategy( else: self.vn_data["dnsStrategy"] = "NONE" - self.save() + return self class FMAzureVirtualNetwork(FMVirtualNetwork): @@ -201,7 +201,7 @@ def set_dns_strategy(self, assign_domain_name, azure_dns_zone_id=None): else: self.vn_data["dnsStrategy"] = "NONE" - self.save() + return self class FMHTTPSStrategy(dict): From 539bd7e5d844ee834ddc8c8fb9e5f20803e2f62e Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 23 Sep 2021 12:45:42 +0200 Subject: [PATCH 37/38] FM: Review feedback on InstanceSettingsTemplates --- dataikuapi/fm/instancesettingstemplates.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/dataikuapi/fm/instancesettingstemplates.py b/dataikuapi/fm/instancesettingstemplates.py index 6d91d51a..1ca123f0 100644 --- a/dataikuapi/fm/instancesettingstemplates.py +++ b/dataikuapi/fm/instancesettingstemplates.py @@ -38,8 +38,23 @@ def with_setup_actions(self, setup_actions): self.data["setupActions"] = setup_actions return self - def with_license(self, license): - self.data["license"] = license + def with_license(self, license_file_path=None, license_string=None): + """ + Override global license + + :param str license_file_path: Optional, load the license from a json file + :param str license_string: Optional, load the license from a json string + """ + if license_file_path is not None: + with open(license_file_path) as json_file: + license = json.load(json_file) + elif license_string is not None: + license = json.loads(license_string) + else: + raise ValueError( + "a valid license_file_path or license_string needs to be provided" + ) + self.data["license"] = json.dumps(license, indent=2) return self @@ -211,7 +226,7 @@ def add_setup_action(self, setup_action): :param object setup_action: a :class:`dataikuapi.fm.instancesettingstemplates.FMSetupAction` """ self.ist_data["setupActions"].append(setup_action) - self.save() + return self class FMSetupAction(dict): From 2cf3bc56e9aca7a593d7d155008cc2b1d6fdc605 Mon Sep 17 00:00:00 2001 From: Thibaud Baas Date: Thu, 23 Sep 2021 12:47:02 +0200 Subject: [PATCH 38/38] FM: Review feedback on Instances --- dataikuapi/fm/instances.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataikuapi/fm/instances.py b/dataikuapi/fm/instances.py index 8e70c543..51301f21 100644 --- a/dataikuapi/fm/instances.py +++ b/dataikuapi/fm/instances.py @@ -240,7 +240,7 @@ def set_automated_snapshots(self, enable, period, keep=0): self.instance_data["enableAutomatedSnapshot"] = enable self.instance_data["automatedSnapshotPeriod"] = period self.instance_data["automatedSnapshotRetention"] = keep - self.save() + return self def set_custom_certificate(self, pem_data): """ @@ -251,7 +251,7 @@ def set_custom_certificate(self, pem_data): param: str pem_data: The SSL certificate """ self.instance_data["sslCertificatePEM"] = pem_data - self.save() + return self class FMAWSInstance(FMInstance): @@ -264,7 +264,7 @@ def set_elastic_ip(self, enable, elasticip_allocation_id): """ self.instance_data["awsAssignElasticIP"] = enable self.instance_data["awsElasticIPAllocationId"] = elasticip_allocation_id - self.save() + return self class FMAzureInstance(FMInstance): @@ -277,7 +277,7 @@ def set_elastic_ip(self, enable, public_ip_id): """ self.instance_data["azureAssignElasticIP"] = enable self.instance_data["azurePublicIPId"] = public_ip_id - self.save() + return self class FMInstanceEncryptionMode(Enum):