feat: manifest onboarding

src/datapilot/__main__.py

@@ -8,6 +8,7 @@
 - https://docs.python.org/2/using/cmdline.html#cmdoption-m
 - https://docs.python.org/3/using/cmdline.html#cmdoption-m
 """
+
 from datapilot.cli.main import datapilot
 
 if __name__ == "__main__":

src/datapilot/clients/altimate/client.py

@@ -0,0 +1,53 @@
+import logging
+
+import requests
+
+
+class APIClient:
+    def __init__(self, api_token="", base_url="", tenant=""):
+        self.api_token = api_token
+        self.base_url = base_url
+        self.tenant = tenant
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def _get_headers(self):
+        headers = {
+            "Content-Type": "application/json",
+        }
+
+        if self.api_token:
+            headers["Authorization"] = f"Bearer {self.api_token}"
+
+        if self.tenant:
+            headers["x-tenant"] = self.tenant
+
+        return headers
+
+    def get(self, endpoint, params=None):
+        url = f"{self.base_url}{endpoint}"
+        headers = self._get_headers()
+
+        self.logger.debug(f"Sending GET request for tenant {self.tenant} at url: {url}")
+        print(f"Sending GET request for tenant {self.tenant} at url: {url}")
+        response = requests.get(url, headers=headers, params=params)
+        self.logger.debug(f"Received GET response with status: {response.status_code }")
+        return response.json() if response.status_code == 200 else None
+
+    def post(self, endpoint, data=None):
+        url = f"{self.base_url}{endpoint}"
+        headers = self._get_headers()
+
+        self.logger.debug(f"Sending POST request for tenant {self.tenant} at url: {url}")
+        response = requests.post(url, headers=headers, json=data)
+        self.logger.debug(f"Received POST response with status: {response.status_code }")
+
+        return response
+
+    def put(self, endpoint, data):
+        url = f"{self.base_url}{endpoint}"
+        headers = self._get_headers()
+
+        self.logger.debug(f"Sending PUT request for tenant {self.tenant} at url: {url}")
+        response = requests.put(url, data=data)
+        self.logger.debug(f"Received PUT response with status: {response.status_code}")
+        return response

src/datapilot/config/utils.py

@@ -1,8 +1,7 @@
 from typing import Dict, Optional, Text
 
 from datapilot.core.platforms.dbt.constants import FOLDER, MODEL
-from datapilot.schemas.constants import (CONFIG_FOLDER_TYPE_PATTERNS,
-                                         CONFIG_MODEL_TYPE_PATTERNS)
+from datapilot.schemas.constants import CONFIG_FOLDER_TYPE_PATTERNS, CONFIG_MODEL_TYPE_PATTERNS
 
 
 def get_regex_configuration(

src/datapilot/core/platforms/dbt/cli/cli.py

@@ -1,13 +1,15 @@
 import logging
+import os
 
 import click
 
 from datapilot.config.config import load_config
 from datapilot.core.platforms.dbt.constants import MODEL, PROJECT
 from datapilot.core.platforms.dbt.executor import DBTInsightGenerator
-from datapilot.core.platforms.dbt.formatting import (
-    generate_model_insights_table, generate_project_insights_table)
+from datapilot.core.platforms.dbt.formatting import generate_model_insights_table, generate_project_insights_table
+from datapilot.core.platforms.dbt.utils import load_manifest
 from datapilot.utils.formatting.utils import tabulate_data
+from datapilot.utils.utils import onboard_manifest
 
 logging.basicConfig(level=logging.INFO)
 
@@ -64,3 +66,30 @@ def project_health(manifest_path, catalog_path, config_path=None):
     click.echo("Project Insights")
     click.echo("--" * 50)
     click.echo(tabulate_data(project_report, headers="keys"))
+
+
+@dbt.command("onboard")
+@click.option("--token", prompt="API Token", help="Your API token for authentication.")
+@click.option("--tenant", prompt="Tenant", help="Your tenant ID.")
+@click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID")
+@click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.")
+def onboard(token, tenant, dbt_core_integration_id, manifest_path, env=None):
+    """Onboard a manifest file to DBT."""
+    if not token and env:
+        token = os.environ.get("ALTIMATE_API_KEY")
+
+    if not tenant and env:
+        tenant = os.environ.get("ALTIMATE_INSTANCE_NAME")
+
+    if not token or not tenant:
+        click.echo("Error: API Token is required.")
+        return
+
+    # if not validate_credentials(token, tenant):
+    #     print("Error: Invalid credentials.")
+    #     return
+
+    # This will throw error if manifest file is incorrect
+    # load_manifest(manifest_path)
+
+    onboard_manifest(token, tenant, dbt_core_integration_id, manifest_path)

src/datapilot/core/platforms/dbt/executor.py

@@ -1,4 +1,5 @@
 import logging
+
 # from src.utils.formatting.utils import generate_model_insights_table
 from typing import Optional, Text
 
@@ -7,12 +8,10 @@
 from datapilot.config.config import load_config
 from datapilot.core.platforms.dbt.constants import MODEL, PROJECT
 from datapilot.core.platforms.dbt.factory import DBTFactory
-from datapilot.core.platforms.dbt.formatting import (
-    generate_model_insights_table, generate_project_insights_table)
+from datapilot.core.platforms.dbt.formatting import generate_model_insights_table, generate_project_insights_table
 from datapilot.core.platforms.dbt.insights import INSIGHTS
 from datapilot.core.platforms.dbt.utils import load_catalog, load_manifest
-from datapilot.utils.formatting.utils import (RED, YELLOW, color_text,
-                                              tabulate_data)
+from datapilot.utils.formatting.utils import RED, YELLOW, color_text, tabulate_data
 
 
 class DBTInsightGenerator:

src/datapilot/core/platforms/dbt/factory.py

@@ -2,10 +2,8 @@
 from dbt_artifacts_parser.parsers.manifest.manifest_v11 import ManifestV11
 
 from datapilot.core.platforms.dbt.schemas.manifest import Catalog, Manifest
-from datapilot.core.platforms.dbt.wrappers.catalog.v1.wrapper import \
-    CatalogV1Wrapper
-from datapilot.core.platforms.dbt.wrappers.manifest.v11.wrapper import \
-    ManifestV11Wrapper
+from datapilot.core.platforms.dbt.wrappers.catalog.v1.wrapper import CatalogV1Wrapper
+from datapilot.core.platforms.dbt.wrappers.manifest.v11.wrapper import ManifestV11Wrapper
 from datapilot.exceptions.exceptions import AltimateNotSupportedError
 
 

src/datapilot/core/platforms/dbt/formatting.py

@@ -1,8 +1,7 @@
 from typing import Dict, List, Text
 
 from datapilot.core.insights.schema import InsightResult, Severity
-from datapilot.core.platforms.dbt.insights.schema import (
-    DBTModelInsightResponse, DBTProjectInsightResponse)
+from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse, DBTProjectInsightResponse
 from datapilot.utils.formatting.utils import color_based_on_severity
 
 

src/datapilot/core/platforms/dbt/insights/__init__.py

@@ -1,20 +1,32 @@
-from datapilot.core.platforms.dbt.insights.dbt_test import (
-    DBTTestCoverage, MissingPrimaryKeyTests)
+from datapilot.core.platforms.dbt.insights.dbt_test import DBTTestCoverage, MissingPrimaryKeyTests
 from datapilot.core.platforms.dbt.insights.governance import (
-    DBTDocumentationStaleColumns, DBTExposureDependentOnPrivateModels,
-    DBTMissingDocumentation, DBTPublicModelWithoutContracts,
-    DBTUndocumentedPublicModels)
+    DBTDocumentationStaleColumns,
+    DBTExposureDependentOnPrivateModels,
+    DBTMissingDocumentation,
+    DBTPublicModelWithoutContracts,
+    DBTUndocumentedPublicModels,
+)
 from datapilot.core.platforms.dbt.insights.modelling import (
-    DBTDirectJoinSource, DBTDownstreamModelsDependentOnSource,
-    DBTDuplicateSources, DBTHardCodedReferences, DBTModelFanout,
-    DBTModelsMultipleSourcesJoined, DBTRejoiningOfUpstreamConcepts,
-    DBTRootModel, DBTSourceFanout, DBTStagingModelsDependentOnDownstreamModels,
-    DBTStagingModelsDependentOnStagingModels, DBTUnusedSources)
-from datapilot.core.platforms.dbt.insights.performance import (
-    DBTChainViewLinking, DBTExposureParentMaterialization)
+    DBTDirectJoinSource,
+    DBTDownstreamModelsDependentOnSource,
+    DBTDuplicateSources,
+    DBTHardCodedReferences,
+    DBTModelFanout,
+    DBTModelsMultipleSourcesJoined,
+    DBTRejoiningOfUpstreamConcepts,
+    DBTRootModel,
+    DBTSourceFanout,
+    DBTStagingModelsDependentOnDownstreamModels,
+    DBTStagingModelsDependentOnStagingModels,
+    DBTUnusedSources,
+)
+from datapilot.core.platforms.dbt.insights.performance import DBTChainViewLinking, DBTExposureParentMaterialization
 from datapilot.core.platforms.dbt.insights.structure import (
-    DBTModelDirectoryStructure, DBTModelNamingConvention,
-    DBTSourceDirectoryStructure, DBTTestDirectoryStructure)
+    DBTModelDirectoryStructure,
+    DBTModelNamingConvention,
+    DBTSourceDirectoryStructure,
+    DBTTestDirectoryStructure,
+)
 
 INSIGHTS = [
     DBTDirectJoinSource,

src/datapilot/core/platforms/dbt/insights/base.py

@@ -5,10 +5,13 @@
 from datapilot.core.insights.schema import Severity
 from datapilot.core.platforms.dbt.constants import NON_MATERIALIZED
 from datapilot.core.platforms.dbt.schemas.manifest import (
-    AltimateManifestExposureNode, AltimateManifestNode,
-    AltimateManifestSourceNode, AltimateManifestTestNode, AltimateResourceType)
-from datapilot.core.platforms.dbt.wrappers.manifest.wrapper import \
-    BaseManifestWrapper
+    AltimateManifestExposureNode,
+    AltimateManifestNode,
+    AltimateManifestSourceNode,
+    AltimateManifestTestNode,
+    AltimateResourceType,
+)
+from datapilot.core.platforms.dbt.wrappers.manifest.wrapper import BaseManifestWrapper
 
 
 class DBTInsight(Insight):
@@ -42,9 +45,12 @@ def generate(self, *args, **kwargs) -> dict:
     def check_part_of_project(self, node_project_name: Text) -> bool:
         return node_project_name == self.project_name
 
-    def get_node(
-        self, node_id: Text
-    ) -> Union[AltimateManifestNode, AltimateManifestSourceNode, AltimateManifestExposureNode, AltimateManifestTestNode,]:
+    def get_node(self, node_id: Text) -> Union[
+        AltimateManifestNode,
+        AltimateManifestSourceNode,
+        AltimateManifestExposureNode,
+        AltimateManifestTestNode,
+    ]:
         if node_id in self.nodes:
             return self.nodes[node_id]
         elif node_id in self.sources:

src/datapilot/core/platforms/dbt/insights/dbt_test/missing_primary_key_tests.py

@@ -3,8 +3,7 @@
 from datapilot.core.insights.utils import get_severity
 from datapilot.core.platforms.dbt.constants import GENERIC
 from datapilot.core.platforms.dbt.insights.dbt_test.base import DBTTestInsight
-from datapilot.core.platforms.dbt.insights.schema import (
-    DBTInsightResult, DBTModelInsightResponse)
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse
 from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
 
 

src/datapilot/core/platforms/dbt/insights/dbt_test/test_coverage.py

@@ -3,8 +3,7 @@
 from datapilot.core.insights.utils import get_severity
 from datapilot.core.platforms.dbt.constants import SINGULAR
 from datapilot.core.platforms.dbt.insights.dbt_test.base import DBTTestInsight
-from datapilot.core.platforms.dbt.insights.schema import (
-    DBTInsightResult, DBTProjectInsightResponse)
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTProjectInsightResponse
 from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
 from datapilot.schemas.constants import CONFIG_METRICS
 

src/datapilot/core/platforms/dbt/insights/governance/__init__.py

@@ -1,6 +1,5 @@
 from .documentation_on_stale_columns import DBTDocumentationStaleColumns
-from .exposures_dependent_on_private_models import \
-    DBTExposureDependentOnPrivateModels
+from .exposures_dependent_on_private_models import DBTExposureDependentOnPrivateModels
 from .public_models_without_contracts import DBTPublicModelWithoutContracts
 from .undocumented_columns import DBTMissingDocumentation
 from .undocumented_public_models import DBTUndocumentedPublicModels

src/datapilot/core/platforms/dbt/insights/governance/documentation_on_stale_columns.py

@@ -1,13 +1,10 @@
 from typing import List, Text, Tuple
 
 from datapilot.core.insights.utils import get_severity
-from datapilot.core.platforms.dbt.insights.governance.base import \
-    DBTGovernanceInsight
-from datapilot.core.platforms.dbt.insights.schema import (
-    DBTInsightResult, DBTModelInsightResponse)
+from datapilot.core.platforms.dbt.insights.governance.base import DBTGovernanceInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse
 from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
-from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import \
-    BaseCatalogWrapper
+from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import BaseCatalogWrapper
 from datapilot.utils.formatting.utils import numbered_list
 
 

src/datapilot/core/platforms/dbt/insights/governance/exposures_dependent_on_private_models.py

@@ -1,10 +1,8 @@
 from typing import List
 
 from datapilot.core.insights.utils import get_severity
-from datapilot.core.platforms.dbt.insights.governance.base import \
-    DBTGovernanceInsight
-from datapilot.core.platforms.dbt.insights.schema import (
-    DBTInsightResult, DBTModelInsightResponse)
+from datapilot.core.platforms.dbt.insights.governance.base import DBTGovernanceInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse
 from datapilot.core.platforms.dbt.schemas.manifest import AltimateAccess
 from datapilot.utils.formatting.utils import numbered_list
 

src/datapilot/core/platforms/dbt/insights/governance/public_models_without_contracts.py

@@ -1,12 +1,9 @@
 from typing import List
 
 from datapilot.core.insights.utils import get_severity
-from datapilot.core.platforms.dbt.insights.governance.base import \
-    DBTGovernanceInsight
-from datapilot.core.platforms.dbt.insights.schema import (
-    DBTInsightResult, DBTModelInsightResponse)
-from datapilot.core.platforms.dbt.schemas.manifest import (
-    AltimateAccess, AltimateResourceType)
+from datapilot.core.platforms.dbt.insights.governance.base import DBTGovernanceInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateAccess, AltimateResourceType
 
 
 class DBTPublicModelWithoutContracts(DBTGovernanceInsight):

src/datapilot/core/platforms/dbt/insights/governance/undocumented_columns.py

@@ -1,13 +1,10 @@
 from typing import List, Text, Tuple
 
 from datapilot.core.insights.utils import get_severity
-from datapilot.core.platforms.dbt.insights.governance.base import \
-    DBTGovernanceInsight
-from datapilot.core.platforms.dbt.insights.schema import (
-    DBTInsightResult, DBTModelInsightResponse)
+from datapilot.core.platforms.dbt.insights.governance.base import DBTGovernanceInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse
 from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
-from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import \
-    BaseCatalogWrapper
+from datapilot.core.platforms.dbt.wrappers.catalog.wrapper import BaseCatalogWrapper
 from datapilot.utils.formatting.utils import numbered_list
 
 

src/datapilot/core/platforms/dbt/insights/governance/undocumented_public_models.py

@@ -1,12 +1,9 @@
 from typing import List
 
 from datapilot.core.insights.utils import get_severity
-from datapilot.core.platforms.dbt.insights.governance.base import \
-    DBTGovernanceInsight
-from datapilot.core.platforms.dbt.insights.schema import (
-    DBTInsightResult, DBTModelInsightResponse)
-from datapilot.core.platforms.dbt.schemas.manifest import (
-    AltimateAccess, AltimateManifestNode, AltimateResourceType)
+from datapilot.core.platforms.dbt.insights.governance.base import DBTGovernanceInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse
+from datapilot.core.platforms.dbt.schemas.manifest import AltimateAccess, AltimateManifestNode, AltimateResourceType
 from datapilot.utils.formatting.utils import numbered_list
 
 

src/datapilot/core/platforms/dbt/insights/modelling/__init__.py

@@ -1,15 +1,12 @@
 from .direct_join_to_source import DBTDirectJoinSource
-from .downstream_models_dependent_on_source import \
-    DBTDownstreamModelsDependentOnSource
+from .downstream_models_dependent_on_source import DBTDownstreamModelsDependentOnSource
 from .duplicate_sources import DBTDuplicateSources
 from .hard_coded_references import DBTHardCodedReferences
 from .joining_of_upstream_concepts import DBTRejoiningOfUpstreamConcepts
 from .model_fanout import DBTModelFanout
 from .multiple_sources_joined import DBTModelsMultipleSourcesJoined
 from .root_model import DBTRootModel
 from .source_fanout import DBTSourceFanout
-from .staging_model_dependent_on_downstream_models import \
-    DBTStagingModelsDependentOnDownstreamModels
-from .staging_model_dependent_on_staging_models import \
-    DBTStagingModelsDependentOnStagingModels
+from .staging_model_dependent_on_downstream_models import DBTStagingModelsDependentOnDownstreamModels
+from .staging_model_dependent_on_staging_models import DBTStagingModelsDependentOnStagingModels
 from .unused_sources import DBTUnusedSources

src/datapilot/core/platforms/dbt/insights/modelling/direct_join_to_source.py

@@ -2,10 +2,8 @@
 
 from datapilot.core.insights.utils import get_severity
 from datapilot.core.platforms.dbt.constants import MODEL, SOURCE
-from datapilot.core.platforms.dbt.insights.modelling.base import \
-    DBTModellingInsight
-from datapilot.core.platforms.dbt.insights.schema import (
-    DBTInsightResult, DBTModelInsightResponse)
+from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse
 from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
 from datapilot.utils.formatting.utils import numbered_list
 

src/datapilot/core/platforms/dbt/insights/modelling/downstream_models_dependent_on_source.py

@@ -3,10 +3,8 @@
 from datapilot.config.utils import get_regex_configuration
 from datapilot.core.insights.utils import get_severity
 from datapilot.core.platforms.dbt.constants import INTERMEDIATE, MART
-from datapilot.core.platforms.dbt.insights.modelling.base import \
-    DBTModellingInsight
-from datapilot.core.platforms.dbt.insights.schema import (
-    DBTInsightResult, DBTModelInsightResponse)
+from datapilot.core.platforms.dbt.insights.modelling.base import DBTModellingInsight
+from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult, DBTModelInsightResponse
 from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
 from datapilot.core.platforms.dbt.utils import classify_model_type
 from datapilot.utils.formatting.utils import numbered_list