From 9138201b9a651507010f133d6c8ece10f95f111f Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Fri, 26 Jan 2024 17:04:26 +0530 Subject: [PATCH 1/2] manifest onboarding --- src/datapilot/__main__.py | 1 + src/datapilot/cli/main.py | 28 ++++++++ src/datapilot/clients/__init__.py | 0 src/datapilot/clients/altimate/__init__.py | 0 src/datapilot/clients/altimate/client.py | 32 +++++++++ src/datapilot/config/utils.py | 3 +- src/datapilot/core/platforms/dbt/cli/cli.py | 3 +- src/datapilot/core/platforms/dbt/executor.py | 7 +- src/datapilot/core/platforms/dbt/factory.py | 6 +- .../core/platforms/dbt/formatting.py | 3 +- .../core/platforms/dbt/insights/__init__.py | 40 +++++++---- .../core/platforms/dbt/insights/base.py | 20 ++++-- .../dbt_test/missing_primary_key_tests.py | 3 +- .../dbt/insights/dbt_test/test_coverage.py | 3 +- .../dbt/insights/governance/__init__.py | 3 +- .../documentation_on_stale_columns.py | 9 +-- .../exposures_dependent_on_private_models.py | 6 +- .../public_models_without_contracts.py | 9 +-- .../governance/undocumented_columns.py | 9 +-- .../governance/undocumented_public_models.py | 9 +-- .../dbt/insights/modelling/__init__.py | 9 +-- .../modelling/direct_join_to_source.py | 6 +- .../downstream_models_dependent_on_source.py | 6 +- .../insights/modelling/duplicate_sources.py | 6 +- .../modelling/hard_coded_references.py | 6 +- .../modelling/joining_of_upstream_concepts.py | 6 +- .../dbt/insights/modelling/model_fanout.py | 6 +- .../modelling/multiple_sources_joined.py | 6 +- .../dbt/insights/modelling/root_model.py | 6 +- .../dbt/insights/modelling/source_fanout.py | 6 +- ...ng_model_dependent_on_downstream_models.py | 6 +- ...aging_model_dependent_on_staging_models.py | 6 +- .../dbt/insights/modelling/unused_sources.py | 6 +- .../performance/chain_view_linking.py | 6 +- .../exposure_parent_materializations.py | 6 +- .../structure/model_directories_structuire.py | 9 +-- .../structure/model_naming_conventions.py | 9 +-- .../structure/source_directories_structure.py | 6 +- .../structure/test_directory_structure.py | 6 +- src/datapilot/core/platforms/dbt/utils.py | 16 ++--- .../dbt/wrappers/catalog/v1/wrapper.py | 3 +- .../dbt/wrappers/manifest/v11/schemas.py | 14 +++- .../dbt/wrappers/manifest/v11/wrapper.py | 72 ++++++++++++------- .../dbt/wrappers/manifest/wrapper.py | 7 +- .../dbt/wrappers/run_results/run_results.py | 15 ++-- src/datapilot/utils/utils.py | 41 +++++++++++ tests/core/platform/dbt/test_utils.py | 15 ++-- 47 files changed, 288 insertions(+), 212 deletions(-) create mode 100644 src/datapilot/clients/__init__.py create mode 100644 src/datapilot/clients/altimate/__init__.py create mode 100644 src/datapilot/clients/altimate/client.py diff --git a/src/datapilot/__main__.py b/src/datapilot/__main__.py index dc0c0781..bb123c92 100644 --- a/src/datapilot/__main__.py +++ b/src/datapilot/__main__.py @@ -8,6 +8,7 @@ - https://docs.python.org/2/using/cmdline.html#cmdoption-m - https://docs.python.org/3/using/cmdline.html#cmdoption-m """ + from datapilot.cli.main import datapilot if __name__ == "__main__": diff --git a/src/datapilot/cli/main.py b/src/datapilot/cli/main.py index 38f1d7ab..52a801a7 100644 --- a/src/datapilot/cli/main.py +++ b/src/datapilot/cli/main.py @@ -1,6 +1,10 @@ +import os + import click from datapilot.core.platforms.dbt.cli import dbt +from datapilot.core.platforms.dbt.utils import load_manifest +from datapilot.utils.utils import onboard_manifest @click.group() @@ -9,4 +13,28 @@ def datapilot(): pass +@dbt.command() +@click.option("--api-token", prompt="API Token", help="Your API token for authentication.") +@click.option("--tenant", prompt="Tenant", help="Your tenant ID.") +@click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID") +@click.option("--manifest-path", prompt="Manifest Path", help="Path to the manifest file.") +def onboard(token, tenant, dbt_core_integration_id, manifest_path, env): + """Onboard a manifest file to DBT.""" + if not token and env: + token = os.environ.get("DBT_API_TOKEN") + + if not token or not tenant: + print("Error: API Token is required.") + return + + # if not validate_credentials(token, tenant): + # print("Error: Invalid credentials.") + # return + + # This will throw error if manifest file is incorrect + load_manifest(manifest_path) + + onboard_manifest(token, tenant, dbt_core_integration_id, manifest_path) + + datapilot.add_command(dbt) diff --git a/src/datapilot/clients/__init__.py b/src/datapilot/clients/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/datapilot/clients/altimate/__init__.py b/src/datapilot/clients/altimate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py new file mode 100644 index 00000000..c2d7e0bf --- /dev/null +++ b/src/datapilot/clients/altimate/client.py @@ -0,0 +1,32 @@ +import requests + + +class APIClient: + def __init__(self, api_token, base_url, tenant): + self.api_token = api_token + self.base_url = base_url + self.tenant = tenant + + def _get_headers(self): + headers = { + "Authorization": f"Bearer {self.api_token}", + "Content-Type": "application/json", + "X-Tenant": self.tenant, + } + return headers + + def get(self, endpoint, params=None): + url = f"{self.base_url}/{endpoint}" + headers = self._get_headers() + + response = requests.get(url, headers=headers, params=params) + + return response.json() if response.status_code == 200 else None + + def post(self, endpoint, data=None): + url = f"{self.base_url}/{endpoint}" + headers = self._get_headers() + + response = requests.post(url, headers=headers, json=data) + + return response.json() if response.status_code == 201 else None diff --git a/src/datapilot/config/utils.py b/src/datapilot/config/utils.py index 3da157c2..90d5e535 100644 --- a/src/datapilot/config/utils.py +++ b/src/datapilot/config/utils.py @@ -1,8 +1,7 @@ from typing import Dict, Optional, Text from datapilot.core.platforms.dbt.constants import FOLDER, MODEL -from datapilot.schemas.constants import (CONFIG_FOLDER_TYPE_PATTERNS, - CONFIG_MODEL_TYPE_PATTERNS) +from datapilot.schemas.constants import CONFIG_FOLDER_TYPE_PATTERNS, CONFIG_MODEL_TYPE_PATTERNS def get_regex_configuration( diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index a095f114..01586a72 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -5,8 +5,7 @@ from datapilot.config.config import load_config from datapilot.core.platforms.dbt.constants import MODEL, PROJECT from datapilot.core.platforms.dbt.executor import DBTInsightGenerator -from datapilot.core.platforms.dbt.formatting import ( - generate_model_insights_table, generate_project_insights_table) +from datapilot.core.platforms.dbt.formatting import generate_model_insights_table, generate_project_insights_table from datapilot.utils.formatting.utils import tabulate_data logging.basicConfig(level=logging.INFO) diff --git a/src/datapilot/core/platforms/dbt/executor.py b/src/datapilot/core/platforms/dbt/executor.py index 46364cae..3db23b2b 100644 --- a/src/datapilot/core/platforms/dbt/executor.py +++ b/src/datapilot/core/platforms/dbt/executor.py @@ -1,4 +1,5 @@ import logging + # from src.utils.formatting.utils import generate_model_insights_table from typing import Optional, Text @@ -7,12 +8,10 @@ from datapilot.config.config import load_config from datapilot.core.platforms.dbt.constants import MODEL, PROJECT from datapilot.core.platforms.dbt.factory import DBTFactory -from datapilot.core.platforms.dbt.formatting import ( - generate_model_insights_table, generate_project_insights_table) +from datapilot.core.platforms.dbt.formatting import generate_model_insights_table, generate_project_insights_table from datapilot.core.platforms.dbt.insights import INSIGHTS from datapilot.core.platforms.dbt.utils import load_catalog, load_manifest -from datapilot.utils.formatting.utils import (RED, YELLOW, color_text, - tabulate_data) +from datapilot.utils.formatting.utils import RED, YELLOW, color_text, tabulate_data class DBTInsightGenerator: diff --git a/src/datapilot/core/platforms/dbt/factory.py b/src/datapilot/core/platforms/dbt/factory.py index 846d4f9a..e04506b7 100644 --- a/src/datapilot/core/platforms/dbt/factory.py +++ b/src/datapilot/core/platforms/dbt/factory.py @@ -2,10 +2,8 @@ from dbt_artifacts_parser.parsers.manifest.manifest_v11 import ManifestV11 from datapilot.core.platforms.dbt.schemas.manifest import Catalog, Manifest -from datapilot.core.platforms.dbt.wrappers.catalog.v1.wrapper import \ - CatalogV1Wrapper -from datapilot.core.platforms.dbt.wrappers.manifest.v11.wrapper import \ - ManifestV11Wrapper +from datapilot.core.platforms.dbt.wrappers.catalog.v1.wrapper import CatalogV1Wrapper +from datapilot.core.platforms.dbt.wrappers.manifest.v11.wrapper import ManifestV11Wrapper from datapilot.exceptions.exceptions import AltimateNotSupportedError diff --git a/src/datapilot/core/platforms/dbt/formatting.py b/src/datapilot/core/platforms/dbt/formatting.py index c1abd247..de8de092 100644 --- a/src/datapilot/core/platforms/dbt/formatting.py +++ b/src/datapilot/core/platforms/dbt/formatting.py @@ -1,8 +1,7 @@ from typing import Dict, List, Text from datapilot.core.insights.schema import InsightResult, Severity -from datapilot.core.platforms.dbt.insights.schema import ( - DBTModelInsightResponse, DBTProjectInsightResponse) +from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse, DBTProjectInsightResponse from datapilot.utils.formatting.utils import color_based_on_severity diff --git a/src/datapilot/core/platforms/dbt/insights/__init__.py b/src/datapilot/core/platforms/dbt/insights/__init__.py index 02006a08..631ac0c2 100644 --- a/src/datapilot/core/platforms/dbt/insights/__init__.py +++ b/src/datapilot/core/platforms/dbt/insights/__init__.py @@ -1,20 +1,32 @@ -from datapilot.core.platforms.dbt.insights.dbt_test import ( - DBTTestCoverage, MissingPrimaryKeyTests) +from datapilot.core.platforms.dbt.insights.dbt_test import DBTTestCoverage, MissingPrimaryKeyTests from datapilot.core.platforms.dbt.insights.governance import ( - DBTDocumentationStaleColumns, DBTExposureDependentOnPrivateModels, - DBTMissingDocumentation, DBTPublicModelWithoutContracts, - DBTUndocumentedPublicModels) + DBTDocumentationStaleColumns, + DBTExposureDependentOnPrivateModels, + DBTMissingDocumentation, + DBTPublicModelWithoutContracts, + DBTUndocumentedPublicModels, +) from datapilot.core.platforms.dbt.insights.modelling import ( - DBTDirectJoinSource, DBTDownstreamModelsDependentOnSource, - DBTDuplicateSources, DBTHardCodedReferences, DBTModelFanout, - DBTModelsMultipleSourcesJoined, DBTRejoiningOfUpstreamConcepts, - DBTRootModel, DBTSourceFanout, DBTStagingModelsDependentOnDownstreamModels, - DBTStagingModelsDependentOnStagingModels, DBTUnusedSources) -from datapilot.core.platforms.dbt.insights.performance import ( - DBTChainViewLinking, DBTExposureParentMaterialization) + DBTDirectJoinSource, + DBTDownstreamModelsDependentOnSource, + DBTDuplicateSources, + DBTHardCodedReferences, + DBTModelFanout, + DBTModelsMultipleSourcesJoined, + DBTRejoiningOfUpstreamConcepts, + DBTRootModel, + DBTSourceFanout, + DBTStagingModelsDependentOnDownstreamModels, + DBTStagingModelsDependentOnStagingModels, + DBTUnusedSources, +) +from datapilot.core.platforms.dbt.insights.performance import DBTChainViewLinking, DBTExposureParentMaterialization from datapilot.core.platforms.dbt.insights.structure import ( - DBTModelDirectoryStructure, DBTModelNamingConvention, - DBTSourceDirectoryStructure, DBTTestDirectoryStructure) + DBTModelDirectoryStructure, + DBTModelNamingConvention, + DBTSourceDirectoryStructure, + DBTTestDirectoryStructure, +) INSIGHTS = [ DBTDirectJoinSource, diff --git a/src/datapilot/core/platforms/dbt/insights/base.py b/src/datapilot/core/platforms/dbt/insights/base.py index a54410c0..e166be8d 100644 --- a/src/datapilot/core/platforms/dbt/insights/base.py +++ b/src/datapilot/core/platforms/dbt/insights/base.py @@ -5,10 +5,13 @@ from datapilot.core.insights.schema import Severity from datapilot.core.platforms.dbt.constants import NON_MATERIALIZED from datapilot.core.platforms.dbt.schemas.manifest import ( - AltimateManifestExposureNode, AltimateManifestNode, - AltimateManifestSourceNode, AltimateManifestTestNode, AltimateResourceType) -from datapilot.core.platforms.dbt.wrappers.manifest.wrapper import \ - BaseManifestWrapper + AltimateManifestExposureNode, + AltimateManifestNode, + AltimateManifestSourceNode, + AltimateManifestTestNode, + AltimateResourceType, +) +from datapilot.core.platforms.dbt.wrappers.manifest.wrapper import BaseManifestWrapper class DBTInsight(Insight): @@ -42,9 +45,12 @@ def generate(self, *args, **kwargs) -> dict: def check_part_of_project(self, node_project_name: Text) -> bool: return node_project_name == self.project_name - def get_node( - self, node_id: Text - ) -> Union[AltimateManifestNode, AltimateManifestSourceNode, AltimateManifestExposureNode, AltimateManifestTestNode,]: + def get_node(self, node_id: Text) -> Union[ + AltimateManifestNode, + AltimateManifestSourceNode, + AltimateManifestExposureNode, + AltimateManifestTestNode, + ]: if node_id in self.nodes: return self.nodes[node_id] elif node_id in self.sources: diff --git a/src/datapilot/core/platforms/dbt/insights/dbt_test/missing_primary_key_tests.py b/src/datapilot/core/platforms/dbt/insights/dbt_test/missing_primary_key_tests.py index b6ab8934..5cd3a096 100644 --- a/src/datapilot/core/platforms/dbt/insights/dbt_test/missing_primary_key_tests.py +++ b/src/datapilot/core/platforms/dbt/insights/dbt_test/missing_primary_key_tests.py @@ -3,8 +3,7 @@ from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import GENERIC from datapilot.core.platforms.dbt.insights.dbt_test.base import DBTTestInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType diff --git a/src/datapilot/core/platforms/dbt/insights/dbt_test/test_coverage.py b/src/datapilot/core/platforms/dbt/insights/dbt_test/test_coverage.py index 091c8435..5ac61287 100644 --- a/src/datapilot/core/platforms/dbt/insights/dbt_test/test_coverage.py +++ b/src/datapilot/core/platforms/dbt/insights/dbt_test/test_coverage.py @@ -3,8 +3,7 @@ from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import SINGULAR from datapilot.core.platforms.dbt.insights.dbt_test.base import DBTTestInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTProjectInsightResponse) +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTProjectInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.schemas.constants import CONFIG_METRICS diff --git a/src/datapilot/core/platforms/dbt/insights/governance/__init__.py b/src/datapilot/core/platforms/dbt/insights/governance/__init__.py index 190b530b..b8d01f51 100644 --- a/src/datapilot/core/platforms/dbt/insights/governance/__init__.py +++ b/src/datapilot/core/platforms/dbt/insights/governance/__init__.py @@ -1,6 +1,5 @@ from .documentation_on_stale_columns import DBTDocumentationStaleColumns -from .exposures_dependent_on_private_models import \ - DBTExposureDependentOnPrivateModels +from .exposures_dependent_on_private_models import DBTExposureDependentOnPrivateModels from .public_models_without_contracts import DBTPublicModelWithoutContracts from .undocumented_columns import DBTMissingDocumentation from .undocumented_public_models import DBTUndocumentedPublicModels diff --git a/src/datapilot/core/platforms/dbt/insights/governance/documentation_on_stale_columns.py b/src/datapilot/core/platforms/dbt/insights/governance/documentation_on_stale_columns.py index 4c31a563..f3ed8761 100644 --- a/src/datapilot/core/platforms/dbt/insights/governance/documentation_on_stale_columns.py +++ b/src/datapilot/core/platforms/dbt/insights/governance/documentation_on_stale_columns.py @@ -1,13 +1,10 @@ from typing import List, Text, Tuple from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.governance.base import \ - DBTGovernanceInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.governance.base import DBTGovernanceInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType -from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import \ - BaseCatalogWrapper +from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import BaseCatalogWrapper from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/governance/exposures_dependent_on_private_models.py b/src/datapilot/core/platforms/dbt/insights/governance/exposures_dependent_on_private_models.py index 23ea6613..96ec7b81 100644 --- a/src/datapilot/core/platforms/dbt/insights/governance/exposures_dependent_on_private_models.py +++ b/src/datapilot/core/platforms/dbt/insights/governance/exposures_dependent_on_private_models.py @@ -1,10 +1,8 @@ from typing import List from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.governance.base import \ - DBTGovernanceInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.governance.base import DBTGovernanceInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateAccess from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/governance/public_models_without_contracts.py b/src/datapilot/core/platforms/dbt/insights/governance/public_models_without_contracts.py index 4e903686..1b78a504 100644 --- a/src/datapilot/core/platforms/dbt/insights/governance/public_models_without_contracts.py +++ b/src/datapilot/core/platforms/dbt/insights/governance/public_models_without_contracts.py @@ -1,12 +1,9 @@ from typing import List from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.governance.base import \ - DBTGovernanceInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) -from datapilot.core.platforms.dbt.schemas.manifest import ( - AltimateAccess, AltimateResourceType) +from datapilot.core.platforms.dbt.insights.governance.base import DBTGovernanceInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse +from datapilot.core.platforms.dbt.schemas.manifest import AltimateAccess, AltimateResourceType class DBTPublicModelWithoutContracts(DBTGovernanceInsight): diff --git a/src/datapilot/core/platforms/dbt/insights/governance/undocumented_columns.py b/src/datapilot/core/platforms/dbt/insights/governance/undocumented_columns.py index de2b8d94..0d87c0ca 100644 --- a/src/datapilot/core/platforms/dbt/insights/governance/undocumented_columns.py +++ b/src/datapilot/core/platforms/dbt/insights/governance/undocumented_columns.py @@ -1,13 +1,10 @@ from typing import List, Text, Tuple from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.governance.base import \ - DBTGovernanceInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.governance.base import DBTGovernanceInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType -from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import \ - BaseCatalogWrapper +from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import BaseCatalogWrapper from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/governance/undocumented_public_models.py b/src/datapilot/core/platforms/dbt/insights/governance/undocumented_public_models.py index 093e8af6..1f312241 100644 --- a/src/datapilot/core/platforms/dbt/insights/governance/undocumented_public_models.py +++ b/src/datapilot/core/platforms/dbt/insights/governance/undocumented_public_models.py @@ -1,12 +1,9 @@ from typing import List from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.governance.base import \ - DBTGovernanceInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) -from datapilot.core.platforms.dbt.schemas.manifest import ( - AltimateAccess, AltimateManifestNode, AltimateResourceType) +from datapilot.core.platforms.dbt.insights.governance.base import DBTGovernanceInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse +from datapilot.core.platforms.dbt.schemas.manifest import AltimateAccess, AltimateManifestNode, AltimateResourceType from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/__init__.py b/src/datapilot/core/platforms/dbt/insights/modelling/__init__.py index 1dcc776d..3424a0a3 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/__init__.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/__init__.py @@ -1,6 +1,5 @@ from .direct_join_to_source import DBTDirectJoinSource -from .downstream_models_dependent_on_source import \ - DBTDownstreamModelsDependentOnSource +from .downstream_models_dependent_on_source import DBTDownstreamModelsDependentOnSource from .duplicate_sources import DBTDuplicateSources from .hard_coded_references import DBTHardCodedReferences from .joining_of_upstream_concepts import DBTRejoiningOfUpstreamConcepts @@ -8,8 +7,6 @@ from .multiple_sources_joined import DBTModelsMultipleSourcesJoined from .root_model import DBTRootModel from .source_fanout import DBTSourceFanout -from .staging_model_dependent_on_downstream_models import \ - DBTStagingModelsDependentOnDownstreamModels -from .staging_model_dependent_on_staging_models import \ - DBTStagingModelsDependentOnStagingModels +from .staging_model_dependent_on_downstream_models import DBTStagingModelsDependentOnDownstreamModels +from .staging_model_dependent_on_staging_models import DBTStagingModelsDependentOnStagingModels from .unused_sources import DBTUnusedSources diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/direct_join_to_source.py b/src/datapilot/core/platforms/dbt/insights/modelling/direct_join_to_source.py index 47ed422a..e38ae467 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/direct_join_to_source.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/direct_join_to_source.py @@ -2,10 +2,8 @@ from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import MODEL, SOURCE -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/downstream_models_dependent_on_source.py b/src/datapilot/core/platforms/dbt/insights/modelling/downstream_models_dependent_on_source.py index 96f5a419..dc8a27e8 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/downstream_models_dependent_on_source.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/downstream_models_dependent_on_source.py @@ -3,10 +3,8 @@ from datapilot.config.utils import get_regex_configuration from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import INTERMEDIATE, MART -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.core.platforms.dbt.utils import classify_model_type from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/duplicate_sources.py b/src/datapilot/core/platforms/dbt/insights/modelling/duplicate_sources.py index b77535e4..f24728d2 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/duplicate_sources.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/duplicate_sources.py @@ -2,10 +2,8 @@ from typing import List from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTProjectInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTProjectInsightResponse from datapilot.core.platforms.dbt.utils import get_table_name_from_source from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/hard_coded_references.py b/src/datapilot/core/platforms/dbt/insights/modelling/hard_coded_references.py index c74874ea..13ad912f 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/hard_coded_references.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/hard_coded_references.py @@ -2,10 +2,8 @@ from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import SQL -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.core.platforms.dbt.utils import get_hard_coded_references from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/joining_of_upstream_concepts.py b/src/datapilot/core/platforms/dbt/insights/modelling/joining_of_upstream_concepts.py index 788cedd0..3ae3df48 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/joining_of_upstream_concepts.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/joining_of_upstream_concepts.py @@ -1,10 +1,8 @@ from typing import List from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse class DBTRejoiningOfUpstreamConcepts(DBTModellingInsight): diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/model_fanout.py b/src/datapilot/core/platforms/dbt/insights/modelling/model_fanout.py index c0bd47ac..826745f1 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/model_fanout.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/model_fanout.py @@ -1,10 +1,8 @@ from typing import List from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.schemas.constants import CONFIG_METRICS diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/multiple_sources_joined.py b/src/datapilot/core/platforms/dbt/insights/modelling/multiple_sources_joined.py index 3f099b86..95179445 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/multiple_sources_joined.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/multiple_sources_joined.py @@ -1,10 +1,8 @@ from typing import List from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/root_model.py b/src/datapilot/core/platforms/dbt/insights/modelling/root_model.py index f468c5d5..e1f72f90 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/root_model.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/root_model.py @@ -1,10 +1,8 @@ from typing import List from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/source_fanout.py b/src/datapilot/core/platforms/dbt/insights/modelling/source_fanout.py index 62c67e49..f8557011 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/source_fanout.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/source_fanout.py @@ -1,10 +1,8 @@ from typing import List from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.schemas.constants import CONFIG_METRICS diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_downstream_models.py b/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_downstream_models.py index a5c0b6c2..af0d83dd 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_downstream_models.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_downstream_models.py @@ -3,10 +3,8 @@ from datapilot.config.utils import get_regex_configuration from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import INTERMEDIATE, MART, STAGING -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.core.platforms.dbt.utils import classify_model_type from datapilot.schemas.constants import CONFIG_METRICS diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_staging_models.py b/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_staging_models.py index 450d5b2f..701d70b7 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_staging_models.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/staging_model_dependent_on_staging_models.py @@ -3,10 +3,8 @@ from datapilot.config.utils import get_regex_configuration from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import STAGING -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.core.platforms.dbt.utils import classify_model_type from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/modelling/unused_sources.py b/src/datapilot/core/platforms/dbt/insights/modelling/unused_sources.py index 5aaeb2b7..d10b46eb 100644 --- a/src/datapilot/core/platforms/dbt/insights/modelling/unused_sources.py +++ b/src/datapilot/core/platforms/dbt/insights/modelling/unused_sources.py @@ -1,10 +1,8 @@ from typing import List from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.modelling.base import \ - DBTModellingInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse class DBTUnusedSources(DBTModellingInsight): diff --git a/src/datapilot/core/platforms/dbt/insights/performance/chain_view_linking.py b/src/datapilot/core/platforms/dbt/insights/performance/chain_view_linking.py index e2de0b0e..38f9b11c 100644 --- a/src/datapilot/core/platforms/dbt/insights/performance/chain_view_linking.py +++ b/src/datapilot/core/platforms/dbt/insights/performance/chain_view_linking.py @@ -1,10 +1,8 @@ from typing import List, Text from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.performance.base import \ - DBTPerformanceInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTProjectInsightResponse) +from datapilot.core.platforms.dbt.insights.performance.base import DBTPerformanceInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTProjectInsightResponse from datapilot.schemas.constants import CONFIG_METRICS from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/performance/exposure_parent_materializations.py b/src/datapilot/core/platforms/dbt/insights/performance/exposure_parent_materializations.py index fdf5a0d6..c38b55bb 100644 --- a/src/datapilot/core/platforms/dbt/insights/performance/exposure_parent_materializations.py +++ b/src/datapilot/core/platforms/dbt/insights/performance/exposure_parent_materializations.py @@ -2,10 +2,8 @@ from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import SOURCE -from datapilot.core.platforms.dbt.insights.performance.base import \ - DBTPerformanceInsight -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) +from datapilot.core.platforms.dbt.insights.performance.base import DBTPerformanceInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.utils.formatting.utils import numbered_list diff --git a/src/datapilot/core/platforms/dbt/insights/structure/model_directories_structuire.py b/src/datapilot/core/platforms/dbt/insights/structure/model_directories_structuire.py index 4ef5ce5b..58cda4c9 100644 --- a/src/datapilot/core/platforms/dbt/insights/structure/model_directories_structuire.py +++ b/src/datapilot/core/platforms/dbt/insights/structure/model_directories_structuire.py @@ -3,13 +3,10 @@ from datapilot.config.utils import get_regex_configuration from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import OTHER -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) -from datapilot.core.platforms.dbt.insights.structure.base import \ - DBTStructureInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse +from datapilot.core.platforms.dbt.insights.structure.base import DBTStructureInsight from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType -from datapilot.core.platforms.dbt.utils import (_check_model_folder_convention, - classify_model_type) +from datapilot.core.platforms.dbt.utils import _check_model_folder_convention, classify_model_type class DBTModelDirectoryStructure(DBTStructureInsight): diff --git a/src/datapilot/core/platforms/dbt/insights/structure/model_naming_conventions.py b/src/datapilot/core/platforms/dbt/insights/structure/model_naming_conventions.py index e3a48bc1..5c0ac191 100644 --- a/src/datapilot/core/platforms/dbt/insights/structure/model_naming_conventions.py +++ b/src/datapilot/core/platforms/dbt/insights/structure/model_naming_conventions.py @@ -3,13 +3,10 @@ from datapilot.config.utils import get_regex_configuration from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import MODEL, OTHER -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) -from datapilot.core.platforms.dbt.insights.structure.base import \ - DBTStructureInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse +from datapilot.core.platforms.dbt.insights.structure.base import DBTStructureInsight from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType -from datapilot.core.platforms.dbt.utils import (_check_model_naming_convention, - classify_model_type) +from datapilot.core.platforms.dbt.utils import _check_model_naming_convention, classify_model_type class DBTModelNamingConvention(DBTStructureInsight): diff --git a/src/datapilot/core/platforms/dbt/insights/structure/source_directories_structure.py b/src/datapilot/core/platforms/dbt/insights/structure/source_directories_structure.py index db0de5b0..6a07195b 100644 --- a/src/datapilot/core/platforms/dbt/insights/structure/source_directories_structure.py +++ b/src/datapilot/core/platforms/dbt/insights/structure/source_directories_structure.py @@ -3,10 +3,8 @@ from datapilot.config.utils import get_regex_configuration from datapilot.core.insights.utils import get_severity from datapilot.core.platforms.dbt.constants import OTHER -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) -from datapilot.core.platforms.dbt.insights.structure.base import \ - DBTStructureInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse +from datapilot.core.platforms.dbt.insights.structure.base import DBTStructureInsight from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType from datapilot.core.platforms.dbt.utils import _check_source_folder_convention diff --git a/src/datapilot/core/platforms/dbt/insights/structure/test_directory_structure.py b/src/datapilot/core/platforms/dbt/insights/structure/test_directory_structure.py index 6830b39c..7a1c2991 100644 --- a/src/datapilot/core/platforms/dbt/insights/structure/test_directory_structure.py +++ b/src/datapilot/core/platforms/dbt/insights/structure/test_directory_structure.py @@ -1,10 +1,8 @@ from typing import List, Optional from datapilot.core.insights.utils import get_severity -from datapilot.core.platforms.dbt.insights.schema import ( - DBTInsightResult, DBTModelInsightResponse) -from datapilot.core.platforms.dbt.insights.structure.base import \ - DBTStructureInsight +from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse +from datapilot.core.platforms.dbt.insights.structure.base import DBTStructureInsight from datapilot.utils.utils import get_dir_path diff --git a/src/datapilot/core/platforms/dbt/utils.py b/src/datapilot/core/platforms/dbt/utils.py index 0dc1d1f7..23559388 100644 --- a/src/datapilot/core/platforms/dbt/utils.py +++ b/src/datapilot/core/platforms/dbt/utils.py @@ -3,17 +3,11 @@ from dbt_artifacts_parser.parser import parse_catalog, parse_manifest -from datapilot.core.platforms.dbt.constants import (BASE, FOLDER, INTERMEDIATE, - MART, MODEL, OTHER, - STAGING) -from datapilot.core.platforms.dbt.exceptions import \ - AltimateInvalidManifestError -from datapilot.core.platforms.dbt.schemas.manifest import ( - AltimateManifestNode, AltimateManifestSourceNode, Catalog, Manifest) -from datapilot.exceptions.exceptions import (AltimateFileNotFoundError, - AltimateInvalidJSONError) -from datapilot.utils.utils import (extract_dir_name_from_file_path, - extract_folders_in_path, load_json) +from datapilot.core.platforms.dbt.constants import BASE, FOLDER, INTERMEDIATE, MART, MODEL, OTHER, STAGING +from datapilot.core.platforms.dbt.exceptions import AltimateInvalidManifestError +from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestNode, AltimateManifestSourceNode, Catalog, Manifest +from datapilot.exceptions.exceptions import AltimateFileNotFoundError, AltimateInvalidJSONError +from datapilot.utils.utils import extract_dir_name_from_file_path, extract_folders_in_path, load_json MODEL_TYPE_PATTERNS = { STAGING: r"^stg_.*", # Example: models starting with 'stg_' diff --git a/src/datapilot/core/platforms/dbt/wrappers/catalog/v1/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/catalog/v1/wrapper.py index 7ab938f3..9d338a07 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/catalog/v1/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/catalog/v1/wrapper.py @@ -1,7 +1,6 @@ from dbt_artifacts_parser.parsers.catalog.catalog_v1 import CatalogV1 -from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import \ - BaseCatalogWrapper +from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import BaseCatalogWrapper class CatalogV1Wrapper(BaseCatalogWrapper): diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/schemas.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/schemas.py index 92a48183..7c8b437e 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/schemas.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/schemas.py @@ -1,8 +1,18 @@ from typing import Dict, Type, Union from dbt_artifacts_parser.parsers.manifest.manifest_v11 import ( - AnalysisNode, Exposure, GenericTestNode, HookNode, ModelNode, RPCNode, - SeedNode, SingularTestNode, SnapshotNode, SourceDefinition, SqlNode) + AnalysisNode, + Exposure, + GenericTestNode, + HookNode, + ModelNode, + RPCNode, + SeedNode, + SingularTestNode, + SnapshotNode, + SourceDefinition, + SqlNode, +) from datapilot.core.platforms.dbt.constants import GENERIC, SINGULAR diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py index 0d8ea9e2..fcd143de 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py @@ -1,22 +1,38 @@ from typing import Dict, Set, Text, Union -from dbt_artifacts_parser.parsers.manifest.manifest_v11 import ( - GenericTestNode, ManifestV11, SingularTestNode) +from dbt_artifacts_parser.parsers.manifest.manifest_v11 import GenericTestNode, ManifestV11, SingularTestNode -from datapilot.core.platforms.dbt.constants import (GENERIC, OTHER_TEST_NODE, - SEED, SINGULAR) +from datapilot.core.platforms.dbt.constants import GENERIC, OTHER_TEST_NODE, SEED, SINGULAR from datapilot.core.platforms.dbt.schemas.manifest import ( - AltimateDBTContract, AltimateDependsOn, AltimateExposureType, - AltimateExternalTable, AltimateFileHash, AltimateFreshnessThreshold, - AltimateManifestColumnInfo, AltimateManifestExposureNode, - AltimateManifestNode, AltimateManifestSourceNode, AltimateManifestTestNode, - AltimateMaturityEnum, AltimateNodeConfig, AltimateOwner, AltimateQuoting, - AltimateRefArgs, AltimateResourceType, AltimateSourceConfig, - AltimateTestConfig, AltimateTestMetadata) + AltimateDBTContract, + AltimateDependsOn, + AltimateExposureType, + AltimateExternalTable, + AltimateFileHash, + AltimateFreshnessThreshold, + AltimateManifestColumnInfo, + AltimateManifestExposureNode, + AltimateManifestNode, + AltimateManifestSourceNode, + AltimateManifestTestNode, + AltimateMaturityEnum, + AltimateNodeConfig, + AltimateOwner, + AltimateQuoting, + AltimateRefArgs, + AltimateResourceType, + AltimateSourceConfig, + AltimateTestConfig, + AltimateTestMetadata, +) from datapilot.core.platforms.dbt.wrappers.manifest.v11.schemas import ( - TEST_TYPE_TO_NODE_MAP, ExposureNode, ManifestNode, SourceNode, TestNode) -from datapilot.core.platforms.dbt.wrappers.manifest.wrapper import \ - BaseManifestWrapper + TEST_TYPE_TO_NODE_MAP, + ExposureNode, + ManifestNode, + SourceNode, + TestNode, +) +from datapilot.core.platforms.dbt.wrappers.manifest.wrapper import BaseManifestWrapper class ManifestV11Wrapper(BaseManifestWrapper): @@ -183,19 +199,21 @@ def _get_tests(self, test: TestNode) -> AltimateManifestTestNode: config=AltimateTestConfig(**test.config.dict()) if test.config else None, description=test.description, tags=test.tags, - columns={ - name: AltimateManifestColumnInfo( - name=column.name, - description=column.description, - meta=column.meta, - data_type=column.data_type, - quote=column.quote, - tags=column.tags, - ) - for name, column in test.columns.items() - } - if test.columns - else None, + columns=( + { + name: AltimateManifestColumnInfo( + name=column.name, + description=column.description, + meta=column.meta, + data_type=column.data_type, + quote=column.quote, + tags=column.tags, + ) + for name, column in test.columns.items() + } + if test.columns + else None + ), meta=test.meta, relation_name=test.relation_name, group=test.group, diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py index 45af82b8..012e737e 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py @@ -2,8 +2,11 @@ from typing import Dict, Set, Text from datapilot.core.platforms.dbt.schemas.manifest import ( - AltimateManifestExposureNode, AltimateManifestNode, - AltimateManifestSourceNode, AltimateManifestTestNode) + AltimateManifestExposureNode, + AltimateManifestNode, + AltimateManifestSourceNode, + AltimateManifestTestNode, +) class BaseManifestWrapper(ABC): diff --git a/src/datapilot/core/platforms/dbt/wrappers/run_results/run_results.py b/src/datapilot/core/platforms/dbt/wrappers/run_results/run_results.py index e5d18bd5..92f95440 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/run_results/run_results.py +++ b/src/datapilot/core/platforms/dbt/wrappers/run_results/run_results.py @@ -1,15 +1,10 @@ from abc import ABC -from dbt_artifacts_parser.parsers.run_results.run_results_v1 import \ - RunResultsV1 -from dbt_artifacts_parser.parsers.run_results.run_results_v2 import \ - RunResultsV2 -from dbt_artifacts_parser.parsers.run_results.run_results_v3 import \ - RunResultsV3 -from dbt_artifacts_parser.parsers.run_results.run_results_v4 import \ - RunResultsV4 -from dbt_artifacts_parser.parsers.run_results.run_results_v5 import \ - RunResultsV5 +from dbt_artifacts_parser.parsers.run_results.run_results_v1 import RunResultsV1 +from dbt_artifacts_parser.parsers.run_results.run_results_v2 import RunResultsV2 +from dbt_artifacts_parser.parsers.run_results.run_results_v3 import RunResultsV3 +from dbt_artifacts_parser.parsers.run_results.run_results_v4 import RunResultsV4 +from dbt_artifacts_parser.parsers.run_results.run_results_v5 import RunResultsV5 class BaseRunResultsWrapper(ABC): diff --git a/src/datapilot/utils/utils.py b/src/datapilot/utils/utils.py index 8c7146e7..3441d1fa 100644 --- a/src/datapilot/utils/utils.py +++ b/src/datapilot/utils/utils.py @@ -2,6 +2,8 @@ import os from typing import Dict, Text +from datapilot.clients.altimate.client import APIClient + def load_json(file_path: Text) -> Dict: try: @@ -39,3 +41,42 @@ def get_dir_path(path: str) -> str: :return: """ return os.path.dirname(path) + + +# Will need to change this +base_url = "http://localhost:5001" + + +def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path): + api_client = APIClient(api_token, base_url, tenant) + + endpoint = "/dbt/v1/signed_url" + params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"} + signed_url_data = api_client.get(endpoint, params=params) + + if signed_url_data: + signed_url = signed_url_data.get("url") + file_id = signed_url_data.get("dbt_core_integration_file_id") + print(f"Received signed URL: {signed_url}") + + with open(manifest_path, "rb") as file: + file_content = file.read() + + upload_response = api_client.post(signed_url, data=file_content) + + if upload_response: + endpoint = "/verify_upload" + verify_params = {"dbt_core_integration_file_id": file_id} + verify_response = api_client.post(endpoint, params=verify_params) + + if verify_response: + print("File successfully uploaded and verified.") + return + else: + print(f"Error verifying upload: {verify_response.status_code}, {verify_response.text}") + + else: + print(f"Error uploading file: {upload_response.status_code}, {upload_response.text}") + + else: + print("Error getting signed URL.") diff --git a/tests/core/platform/dbt/test_utils.py b/tests/core/platform/dbt/test_utils.py index 97ca164a..2213c97f 100644 --- a/tests/core/platform/dbt/test_utils.py +++ b/tests/core/platform/dbt/test_utils.py @@ -1,12 +1,13 @@ import pytest -from datapilot.core.platforms.dbt.constants import (BASE, INTERMEDIATE, MART, - OTHER, STAGING) -from datapilot.core.platforms.dbt.utils import (MODEL_TYPE_PATTERNS, - _check_model_naming_convention, - classify_model_type_by_folder, - classify_model_type_by_name, - get_hard_coded_references) +from datapilot.core.platforms.dbt.constants import BASE, INTERMEDIATE, MART, OTHER, STAGING +from datapilot.core.platforms.dbt.utils import ( + MODEL_TYPE_PATTERNS, + _check_model_naming_convention, + classify_model_type_by_folder, + classify_model_type_by_name, + get_hard_coded_references, +) test_cases = [ # Test with a simple FROM clause From 91c4b4417ee03ab3d0a3a51cb7899ca36d97ddb0 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Mon, 29 Jan 2024 14:36:22 +0530 Subject: [PATCH 2/2] fix onboard cli --- src/datapilot/cli/main.py | 28 ----------------- src/datapilot/clients/altimate/client.py | 35 ++++++++++++++++----- src/datapilot/core/platforms/dbt/cli/cli.py | 30 ++++++++++++++++++ src/datapilot/utils/utils.py | 21 ++++++++----- 4 files changed, 72 insertions(+), 42 deletions(-) diff --git a/src/datapilot/cli/main.py b/src/datapilot/cli/main.py index 52a801a7..38f1d7ab 100644 --- a/src/datapilot/cli/main.py +++ b/src/datapilot/cli/main.py @@ -1,10 +1,6 @@ -import os - import click from datapilot.core.platforms.dbt.cli import dbt -from datapilot.core.platforms.dbt.utils import load_manifest -from datapilot.utils.utils import onboard_manifest @click.group() @@ -13,28 +9,4 @@ def datapilot(): pass -@dbt.command() -@click.option("--api-token", prompt="API Token", help="Your API token for authentication.") -@click.option("--tenant", prompt="Tenant", help="Your tenant ID.") -@click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID") -@click.option("--manifest-path", prompt="Manifest Path", help="Path to the manifest file.") -def onboard(token, tenant, dbt_core_integration_id, manifest_path, env): - """Onboard a manifest file to DBT.""" - if not token and env: - token = os.environ.get("DBT_API_TOKEN") - - if not token or not tenant: - print("Error: API Token is required.") - return - - # if not validate_credentials(token, tenant): - # print("Error: Invalid credentials.") - # return - - # This will throw error if manifest file is incorrect - load_manifest(manifest_path) - - onboard_manifest(token, tenant, dbt_core_integration_id, manifest_path) - - datapilot.add_command(dbt) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index c2d7e0bf..00f3bd56 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -1,32 +1,53 @@ +import logging + import requests class APIClient: - def __init__(self, api_token, base_url, tenant): + def __init__(self, api_token="", base_url="", tenant=""): self.api_token = api_token self.base_url = base_url self.tenant = tenant + self.logger = logging.getLogger(self.__class__.__name__) def _get_headers(self): headers = { - "Authorization": f"Bearer {self.api_token}", "Content-Type": "application/json", - "X-Tenant": self.tenant, } + + if self.api_token: + headers["Authorization"] = f"Bearer {self.api_token}" + + if self.tenant: + headers["x-tenant"] = self.tenant + return headers def get(self, endpoint, params=None): - url = f"{self.base_url}/{endpoint}" + url = f"{self.base_url}{endpoint}" headers = self._get_headers() + self.logger.debug(f"Sending GET request for tenant {self.tenant} at url: {url}") + print(f"Sending GET request for tenant {self.tenant} at url: {url}") response = requests.get(url, headers=headers, params=params) - + self.logger.debug(f"Received GET response with status: {response.status_code }") return response.json() if response.status_code == 200 else None def post(self, endpoint, data=None): - url = f"{self.base_url}/{endpoint}" + url = f"{self.base_url}{endpoint}" headers = self._get_headers() + self.logger.debug(f"Sending POST request for tenant {self.tenant} at url: {url}") response = requests.post(url, headers=headers, json=data) + self.logger.debug(f"Received POST response with status: {response.status_code }") + + return response + + def put(self, endpoint, data): + url = f"{self.base_url}{endpoint}" + headers = self._get_headers() - return response.json() if response.status_code == 201 else None + self.logger.debug(f"Sending PUT request for tenant {self.tenant} at url: {url}") + response = requests.put(url, data=data) + self.logger.debug(f"Received PUT response with status: {response.status_code}") + return response diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 01586a72..c0a310f8 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -1,4 +1,5 @@ import logging +import os import click @@ -6,7 +7,9 @@ from datapilot.core.platforms.dbt.constants import MODEL, PROJECT from datapilot.core.platforms.dbt.executor import DBTInsightGenerator from datapilot.core.platforms.dbt.formatting import generate_model_insights_table, generate_project_insights_table +from datapilot.core.platforms.dbt.utils import load_manifest from datapilot.utils.formatting.utils import tabulate_data +from datapilot.utils.utils import onboard_manifest logging.basicConfig(level=logging.INFO) @@ -63,3 +66,30 @@ def project_health(manifest_path, catalog_path, config_path=None): click.echo("Project Insights") click.echo("--" * 50) click.echo(tabulate_data(project_report, headers="keys")) + + +@dbt.command("onboard") +@click.option("--token", prompt="API Token", help="Your API token for authentication.") +@click.option("--tenant", prompt="Tenant", help="Your tenant ID.") +@click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID") +@click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.") +def onboard(token, tenant, dbt_core_integration_id, manifest_path, env=None): + """Onboard a manifest file to DBT.""" + if not token and env: + token = os.environ.get("ALTIMATE_API_KEY") + + if not tenant and env: + tenant = os.environ.get("ALTIMATE_INSTANCE_NAME") + + if not token or not tenant: + click.echo("Error: API Token is required.") + return + + # if not validate_credentials(token, tenant): + # print("Error: Invalid credentials.") + # return + + # This will throw error if manifest file is incorrect + # load_manifest(manifest_path) + + onboard_manifest(token, tenant, dbt_core_integration_id, manifest_path) diff --git a/src/datapilot/utils/utils.py b/src/datapilot/utils/utils.py index 3441d1fa..75b9dd0e 100644 --- a/src/datapilot/utils/utils.py +++ b/src/datapilot/utils/utils.py @@ -47,10 +47,19 @@ def get_dir_path(path: str) -> str: base_url = "http://localhost:5001" +def upload_content_to_signed_url(file_path, signed_url): + api_client = APIClient() + + with open(file_path, "rb") as file: + file_content = file.read() + + return api_client.put(signed_url, data=file_content) + + def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path): api_client = APIClient(api_token, base_url, tenant) - endpoint = "/dbt/v1/signed_url" + endpoint = f"/dbt/v1/signed_url" params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"} signed_url_data = api_client.get(endpoint, params=params) @@ -58,16 +67,14 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path): signed_url = signed_url_data.get("url") file_id = signed_url_data.get("dbt_core_integration_file_id") print(f"Received signed URL: {signed_url}") + print(f"Received File ID: {file_id}") - with open(manifest_path, "rb") as file: - file_content = file.read() - - upload_response = api_client.post(signed_url, data=file_content) + upload_response = upload_content_to_signed_url(manifest_path, signed_url) if upload_response: - endpoint = "/verify_upload" + endpoint = f"/dbt/v1/verify_upload" verify_params = {"dbt_core_integration_file_id": file_id} - verify_response = api_client.post(endpoint, params=verify_params) + verify_response = api_client.post(endpoint, data=verify_params) if verify_response: print("File successfully uploaded and verified.")