Add dlt commands as Jupyter Notebook magic commands

.github/workflows/lint.yml

@@ -25,7 +25,7 @@ jobs:
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}    
+    runs-on: ${{ matrix.os }}
 
     steps:
 
@@ -42,7 +42,7 @@ jobs:
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
-          installer-parallel: true         
+          installer-parallel: true
 
       - name: Load cached venv
         id: cached-poetry-dependencies
@@ -53,11 +53,11 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --all-extras --with airflow,providers,pipeline,sentry-sdk
+        run: poetry install --all-extras --with airflow,providers,pipeline,sentry-sdk,notebook
 
       - name: Run make lint
         run: |
-          export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows        
+          export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows
           make lint
 
       # - name: print envs
@@ -75,4 +75,4 @@ jobs:
       - name: Check matrix job results
         if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
         run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1      
+          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1

.github/workflows/test_local_destinations.yml

@@ -81,7 +81,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-destinations
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline
+        run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk,pipeline,notebook
 
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml

dlt/__init__.py

@@ -41,6 +41,13 @@
 from dlt.pipeline import progress
 from dlt import destinations
 
+try:
+    from dlt.cli.magics import register_notebook_magics
+
+    register_notebook_magics()
+except Exception:
+    pass
+
 pipeline = _pipeline
 current = _current
 mark = _mark

dlt/cli/echo.py

@@ -47,9 +47,13 @@ def note(msg: str) -> None:
 
 def confirm(text: str, default: Optional[bool] = None) -> bool:
     if ALWAYS_CHOOSE_VALUE:
+        warning(f"Automatically choosing {ALWAYS_CHOOSE_VALUE} for: {text}")
         return bool(ALWAYS_CHOOSE_VALUE)
     if ALWAYS_CHOOSE_DEFAULT:
-        assert default is not None
+        assert (
+            default is not None
+        ), "Default value must be provided when ALWAYS_CHOOSE_DEFAULT is enabled."
+        warning(f"Automatically choosing default ({default}) value for: {text}")
         return default
     return click.confirm(text, default=default)
 

dlt/cli/init_command.py

@@ -1,41 +1,42 @@
-import os
 import ast
+import os
 import shutil
 from types import ModuleType
 from typing import Dict, List, Sequence, Tuple
-from importlib.metadata import version as pkg_version
 
+import dlt.reflection.names as n
+from dlt.cli import echo as fmt, pipeline_files as files_ops, source_detection, utils
+from dlt.cli.config_toml_writer import WritableConfigValue, write_values
+from dlt.cli.exceptions import CliCommandException
+from dlt.cli.pipeline_files import (
+    TVerifiedSourceFileEntry,
+    TVerifiedSourceFileIndex,
+    VerifiedSourceFiles,
+)
+from dlt.cli.requirements import SourceRequirements
 from dlt.common import git
-from dlt.common.configuration.paths import get_dlt_settings_dir, make_dlt_settings_path
-from dlt.common.configuration.specs import known_sections
+from dlt.common.configuration.paths import (
+    create_symlink_to_dlt,
+    get_dlt_settings_dir,
+    make_dlt_settings_path,
+)
 from dlt.common.configuration.providers import (
     CONFIG_TOML,
     SECRETS_TOML,
     ConfigTomlProvider,
     SecretsTomlProvider,
 )
-from dlt.common.pipeline import get_dlt_repos_dir
-from dlt.common.source import _SOURCES
-from dlt.version import DLT_PKG_NAME, __version__
+from dlt.common.configuration.specs import known_sections
 from dlt.common.destination import Destination
+from dlt.common.pipeline import get_dlt_repos_dir
 from dlt.common.reflection.utils import rewrite_python_script
-from dlt.common.schema.utils import is_valid_schema_name
+from dlt.common.runtime.exec_info import is_notebook
 from dlt.common.schema.exceptions import InvalidSchemaName
+from dlt.common.schema.utils import is_valid_schema_name
+from dlt.common.source import _SOURCES
 from dlt.common.storages.file_storage import FileStorage
-
-import dlt.reflection.names as n
 from dlt.reflection.script_inspector import inspect_pipeline_script, load_script_module
-
-from dlt.cli import echo as fmt, pipeline_files as files_ops, source_detection
-from dlt.cli import utils
-from dlt.cli.config_toml_writer import WritableConfigValue, write_values
-from dlt.cli.pipeline_files import (
-    VerifiedSourceFiles,
-    TVerifiedSourceFileEntry,
-    TVerifiedSourceFileIndex,
-)
-from dlt.cli.exceptions import CliCommandException
-from dlt.cli.requirements import SourceRequirements
+from dlt.version import DLT_PKG_NAME
 
 DLT_INIT_DOCS_URL = "https://dlthub.com/docs/reference/command-line-interface#dlt-init"
 DEFAULT_VERIFIED_SOURCES_REPO = "https://github.com/dlt-hub/verified-sources.git"
@@ -92,7 +93,10 @@ def _select_source_files(
         prompt = (
             "Should incoming changes be Skipped, Applied (local changes will be lost) or Merged (%s"
             " UPDATED | %s DELETED | all local changes remain)?"
-            % (fmt.bold(",".join(can_update_files)), fmt.bold(",".join(can_delete_files)))
+            % (
+                fmt.bold(",".join(can_update_files)),
+                fmt.bold(",".join(can_delete_files)),
+            )
         )
         choices = "sam"
     else:
@@ -171,7 +175,10 @@ def _welcome_message(
     if is_new_source:
         fmt.echo(
             "* Add credentials for %s and other secrets in %s"
-            % (fmt.bold(destination_type), fmt.bold(make_dlt_settings_path(SECRETS_TOML)))
+            % (
+                fmt.bold(destination_type),
+                fmt.bold(make_dlt_settings_path(SECRETS_TOML)),
+            )
         )
 
     if dependency_system:
@@ -191,7 +198,7 @@ def _welcome_message(
             )
         elif dependency_system == utils.PYPROJECT_TOML:
             fmt.echo("  If you are using poetry you may issue the following command:")
-            fmt.echo(fmt.bold("  poetry add %s -E %s" % (DLT_PKG_NAME, destination_type)))
+            fmt.echo(fmt.bold(f"  poetry add {DLT_PKG_NAME} -E {destination_type}"))
         fmt.echo()
     else:
         fmt.echo(
@@ -216,7 +223,7 @@ def list_verified_sources_command(repo_location: str, branch: str = None) -> Non
     for source_name, source_files in _list_verified_sources(repo_location, branch).items():
         reqs = source_files.requirements
         dlt_req_string = str(reqs.dlt_requirement_base)
-        msg = "%s: %s" % (fmt.bold(source_name), source_files.doc)
+        msg = f"{fmt.bold(source_name)}: {source_files.doc}"
         if not reqs.is_installed_dlt_compatible():
             msg += fmt.warning_style(" [needs update: %s]" % (dlt_req_string))
         fmt.echo(msg)
@@ -279,7 +286,11 @@ def init_command(
         if conflict_modified or conflict_deleted:
             # select source files that can be copied/updated
             _, remote_modified, remote_deleted = _select_source_files(
-                source_name, remote_modified, remote_deleted, conflict_modified, conflict_deleted
+                source_name,
+                remote_modified,
+                remote_deleted,
+                conflict_modified,
+                conflict_deleted,
             )
         if not remote_deleted and not remote_modified:
             fmt.echo("No files to update, exiting")
@@ -327,7 +338,8 @@ def init_command(
         )
         fmt.warning(msg)
         if not fmt.confirm(
-            "Would you like to continue anyway? (you can update dlt after this step)", default=True
+            "Would you like to continue anyway? (you can update dlt after this step)",
+            default=True,
         ):
             fmt.echo(
                 "You can update dlt with: pip3 install -U"
@@ -387,9 +399,11 @@ def init_command(
         )
         # template sources are always in module starting with "pipeline"
         # for templates, place config and secrets into top level section
-        required_secrets, required_config, checked_sources = source_detection.detect_source_configs(
-            _SOURCES, "pipeline", ()
-        )
+        (
+            required_secrets,
+            required_config,
+            checked_sources,
+        ) = source_detection.detect_source_configs(_SOURCES, "pipeline", ())
         # template has a strict rules where sources are placed
         for source_q_name, source_config in checked_sources.items():
             if source_q_name not in visitor.known_sources_resources:
@@ -410,7 +424,11 @@ def init_command(
         )
         # pipeline sources are in module with name starting from {pipeline_name}
         # for verified pipelines place in the specific source section
-        required_secrets, required_config, checked_sources = source_detection.detect_source_configs(
+        (
+            required_secrets,
+            required_config,
+            checked_sources,
+        ) = source_detection.detect_source_configs(
             _SOURCES, source_name, (known_sections.SOURCES, source_name)
         )
 
@@ -450,6 +468,7 @@ def init_command(
             )
             if use_generic_template:
                 fmt.warning("--generic parameter is meaningless if verified source is found")
+
         if not fmt.confirm("Do you want to proceed?", default=True):
             raise CliCommandException("init", "Aborted")
 
@@ -502,3 +521,6 @@ def init_command(
     if dependency_system is None:
         requirements_txt = "\n".join(source_files.requirements.compiled())
         dest_storage.save(utils.REQUIREMENTS_TXT, requirements_txt)
+
+    if is_notebook():
+        create_symlink_to_dlt("_dlt")