Dcv 2515 dbt coves setup datacoves

README.md

@@ -11,7 +11,6 @@ The Datacoves platform helps enterprises overcome their data delivery challenges
 
 Hosted VS Code, dbt-core, SqlFluff, and Airflow, find out more at [Datacoves.com](https://datacoves.com/product).
 
-
 ## What is dbt-coves?
 
 dbt-coves is a CLI tool that automates certain tasks for [dbt](https://www.getdbt.com), making life simpler for the dbt user.
@@ -66,31 +65,12 @@ dbt-coves <command> -h
 
 ## Environment setup
 
-You can configure different components:
-
-Set up `git` repository of dbt-coves project
-
-```console
-dbt-coves setup git
-```
-
-Set up `dbt` within the project (delegates to dbt init)
-
-```console
-dbt-coves setup dbt
-```
-
-Set up SSH Keys for dbt project. Supports the argument `--open_ssl_public_key` which generates an extra Public Key in Open SSL format, useful for configuring certain providers (i.e. Snowflake authentication)
-
-```console
-dbt-coves setup ssh
-```
-
-Set up pre-commit for your dbt project. In this, you can configure different tools that we consider essential for proper dbt usage: `sqlfluff`, `yaml-lint`, and `dbt-checkpoint`
+By running `dbt-coves setup datacoves` you can set up:
 
-```console
-dbt-coves setup precommit
-```
+- dbt profile for automated runs
+- initial CI/CD scripts
+- linting with SQLFluff, dbt-checkpoint and/or YMLLint
+- sample Airflow DAGs
 
 ## Models generation
 

dbt_coves/config/config.py

@@ -129,17 +129,12 @@ class LoadModel(BaseModel):
     fivetran: Optional[LoadFivetranModel] = LoadFivetranModel()
 
 
-class SetupSshModel(BaseModel):
-    open_ssl_public_key: Optional[bool] = False
-
-
-class SetupGitModel(BaseModel):
+class SetupDatacovesModel(BaseModel):
     no_prompt: Optional[bool] = False
 
 
 class SetupModel(BaseModel):
-    ssh: Optional[SetupSshModel] = SetupSshModel()
-    git: Optional[SetupGitModel] = SetupGitModel()
+    datacoves: Optional[SetupDatacovesModel] = SetupDatacovesModel()
 
 
 class RunDbtModel(BaseModel):
@@ -233,8 +228,7 @@ class DbtCovesConfig:
         "load.airbyte.secrets_project",
         "load.airbyte.secrets_tags",
         "load.airbyte.secrets_key",
-        "setup.ssh.open_ssl_public_key",
-        "setup.git.no_prompt",
+        "setup.datacoves.no_prompt",
         "dbt.command",
         "dbt.project_dir",
         "dbt.virtualenv",

dbt_coves/tasks/dbt/main.py

@@ -137,6 +137,7 @@ def run_command(
                 f"{Text.from_ansi(output.decode())}\n"
                 f"[green]{command} :heavy_check_mark:[/green]"
             )
+            print(output.decode())
         except subprocess.CalledProcessError as e:
             formatted = f"{Text.from_ansi(e.stderr.decode()) if e.stderr else Text.from_ansi(e.stdout.decode())}"
             e.stderr = f"An error has occurred running [red]{command}[/red]:\n{formatted}"

dbt_coves/tasks/setup/datacoves.py

@@ -0,0 +1,115 @@
+"""
+Task that helps Datacoves users set-up their Environment.
+On a first stage, it'll generate:
+- automate/dbt/profiles.yml
+- orchestrate/dags/sample_dag.py
+- .github/workflows/push_to_main.yml
+"""
+
+import os
+from pathlib import Path
+
+import copier
+import questionary
+from rich.console import Console
+
+from dbt_coves.tasks.base import NonDbtBaseTask
+from dbt_coves.utils.tracking import trackable
+
+from .utils import get_dbt_projects
+
+AVAILABLE_SERVICES = {
+    "dbt profile for automated runs": "setup_dbt_profile",
+    "Initial CI/CD scripts": "setup_ci_cd",
+    "Linting with SQLFluff, dbt-checkpoint and/or YMLLint": "setup_precommit",
+    "Sample Airflow DAG": "setup_airflow_dag",
+}
+
+console = Console()
+
+
+class SetupDatacovesTask(NonDbtBaseTask):
+    """
+    Task that runs pre-commit setup
+    """
+
+    @classmethod
+    def register_parser(cls, sub_parsers, base_subparser):
+        subparser = sub_parsers.add_parser(
+            "datacoves",
+            parents=[base_subparser],
+            help="Set up Datacoves CI, profiles and DAGs structure",
+        )
+        subparser.add_argument(
+            "--no-prompt",
+            action="store_true",
+            help="Generate all Datacoves components without prompting for confirmation",
+            default=False,
+        )
+        subparser.set_defaults(cls=cls, which="datacoves")
+        return subparser
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def get_config_value(self, key):
+        return self.coves_config.integrated["setup"]["datacoves"][key]
+
+    @trackable
+    def run(self) -> int:
+        self.repo_path = os.environ.get("DATACOVES__REPO_PATH", "/config/workspace")
+        self.copier_context = {"no_prompt": self.get_config_value("no_prompt")}
+        return self.setup_datacoves()
+
+    def _get_path_rel_to_root(self, path):
+        return str(Path(path).relative_to(self.repo_path))
+
+    def setup_datacoves(self):
+        # dbt profile data gathering
+        choices = questionary.checkbox(
+            "What services would you like to set up?",
+            choices=list(AVAILABLE_SERVICES.keys()),
+        ).ask()
+        services = [AVAILABLE_SERVICES[service] for service in choices]
+
+        airflow_profile_path = os.environ.get(
+            "DATACOVES__AIRFLOW_DBT_PROFILE_PATH", f"{self.repo_path}/automate/dbt"
+        )
+        if not airflow_profile_path:
+            airflow_profile_path = f"{self.repo_path}/automate/dbt"
+
+        self.copier_context["airflow_profile_path"] = self._get_path_rel_to_root(
+            airflow_profile_path
+        )
+
+        dbt_adapter = os.environ.get("DATACOVES__DBT_ADAPTER")
+        if dbt_adapter:
+            self.copier_context["dbt_adapter"] = dbt_adapter
+
+        # sample DAG data
+        airflow_dags_path = os.environ.get(
+            "DATACOVES__AIRFLOW_DAGS_PATH", f"{self.repo_path}/orchestrate/dags"
+        )
+
+        self.copier_context["airflow_dags_path"] = self._get_path_rel_to_root(airflow_dags_path)
+        if "setup_precommit" in services:
+            dbt_project_paths = get_dbt_projects(self.repo_path)
+            if not dbt_project_paths:
+                console.print(
+                    "Your repository doesn't contain any dbt project where to install [red]pre-commit[/red] into"
+                )
+                services.remove("setup_precommit")
+            elif len(dbt_project_paths) == 1:
+                self.copier_context["dbt_project_dir"] = dbt_project_paths[0]
+            else:
+                self.copier_context["dbt_project_dir"] = questionary.select(
+                    "In which dbt project would you like to install pre-commit?",
+                    choices=dbt_project_paths,
+                ).ask()
+        self.copier_context["services"] = services
+        copier.run_auto(
+            src_path=str(Path(__file__).parent.joinpath("templates", "datacoves").resolve()),
+            dst_path=self.repo_path,
+            data=self.copier_context,
+        )
+        return 0