Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove strict source dialect checks #284

Merged
merged 2 commits into from
Apr 26, 2024
Merged

Remove strict source dialect checks #284

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c8c514
Unlock dialect keys limitation
sundarshankar89 Apr 26, 2024
78532a3
fmt fixes
sundarshankar89 Apr 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions src/databricks/labs/remorph/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from databricks.labs.blueprint.cli import App
from databricks.labs.blueprint.entrypoint import get_logger
from databricks.labs.blueprint.installation import Installation
from databricks.labs.remorph.config import MorphConfig
from databricks.labs.remorph.config import SQLGLOT_DIALECTS, MorphConfig
from databricks.labs.remorph.helpers.recon_config_utils import ReconConfigPrompts
from databricks.labs.remorph.lineage import lineage_generator
from databricks.labs.remorph.reconcile.execute import recon
Expand Down Expand Up @@ -35,8 +35,8 @@ def transpile(
installation = Installation.current(w, 'remorph')
default_config = installation.load(MorphConfig)
mode = mode if mode else "current" # not checking for default config as it will always be current

if source.lower() not in {"snowflake", "tsql"}:
dialects = SQLGLOT_DIALECTS.keys()
if source.lower() not in dialects:
raise_validation_exception(
f"Error: Invalid value for '--source': '{source}' is not one of 'snowflake', 'tsql'. "
)
Expand Down Expand Up @@ -92,11 +92,9 @@ def reconcile(w: WorkspaceClient, recon_conf: str, conn_profile: str, source: st
def generate_lineage(w: WorkspaceClient, source: str, input_sql: str, output_folder: str):
"""Generates a lineage of source SQL files or folder"""
logger.info(f"User: {w.current_user.me()}")
expected_sources = {'snowflake', 'tsql'}
if source.lower() not in expected_sources:
raise_validation_exception(
f"Error: Invalid value for '--source': '{source}' is not one of {expected_sources}. "
)
dialects = SQLGLOT_DIALECTS.keys()
if source.lower() not in dialects:
raise_validation_exception(f"Error: Invalid value for '--source': '{source}' is not one of {dialects}. ")
if not os.path.exists(input_sql) or input_sql in {None, ""}:
raise_validation_exception(f"Error: Invalid value for '--input_sql': Path '{input_sql}' does not exist.")
if not os.path.exists(output_folder) or output_folder in {None, ""}:
Expand Down
4 changes: 2 additions & 2 deletions src/databricks/labs/remorph/install.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from databricks.labs.blueprint.tui import Prompts
from databricks.labs.blueprint.wheels import ProductInfo
from databricks.labs.remorph.__about__ import __version__
from databricks.labs.remorph.config import MorphConfig
from databricks.labs.remorph.config import SQLGLOT_DIALECTS, MorphConfig
from databricks.sdk import WorkspaceClient
from databricks.sdk.errors import NotFound
from databricks.sdk.retries import retried
Expand Down Expand Up @@ -61,7 +61,7 @@ def configure(self) -> MorphConfig:
schema_name = "convertor_test"
ws_config = None

source_prompt = self._prompts.choice("Select the source", ["snowflake", "tsql"])
source_prompt = self._prompts.choice("Select the source", SQLGLOT_DIALECTS.keys())
source = source_prompt.lower()

skip_validation = self._prompts.confirm("Do you want to Skip Validation")
Expand Down
14 changes: 7 additions & 7 deletions tests/unit/test_install.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def mock_installation():
def test_install(ws, mock_installation):
prompts = MockPrompts(
{
r"Select the source": "0",
r"Select the source": "10",
r"Do you want to Skip Validation": "yes",
r"Enter catalog_name": "remorph_test",
r".*Do you want to create a new one?": "yes",
Expand All @@ -57,7 +57,7 @@ def test_install_dbr(ws, mock_installation, monkeypatch):
monkeypatch.setenv("DATABRICKS_RUNTIME_VERSION", "14.1")
prompts = MockPrompts(
{
r"Select the source": "0",
r"Select the source": "10",
r"Do you want to Skip Validation": "yes",
r".*": "",
}
Expand All @@ -70,7 +70,7 @@ def test_install_dbr(ws, mock_installation, monkeypatch):
def test_save_config(ws, mock_installation):
prompts = MockPrompts(
{
r"Select the source": "0",
r"Select the source": "10",
r"Do you want to Skip Validation": "yes",
r".*": "",
}
Expand All @@ -95,7 +95,7 @@ def test_save_config(ws, mock_installation):
def test_create_sql_warehouse(ws, mock_installation):
prompts = MockPrompts(
{
r"Select the source": "0",
r"Select the source": "10",
r"Do you want to Skip Validation": "No",
r"Do you want to use SQL Warehouse for validation?": "yes",
r"Select PRO or SERVERLESS SQL warehouse to run validation on": "0",
Expand Down Expand Up @@ -127,7 +127,7 @@ def test_create_sql_warehouse(ws, mock_installation):
def test_create_catalog_schema(ws, mock_installation):
prompts = MockPrompts(
{
r"Select the source": "0",
r"Select the source": "10",
r"Do you want to Skip Validation": "No",
r"Do you want to use SQL Warehouse for validation?": "No",
r"Enter a valid cluster_id to proceed": "test_cluster",
Expand Down Expand Up @@ -159,7 +159,7 @@ def test_create_catalog_schema(ws, mock_installation):
def test_create_catalog_no(ws, mock_installation):
prompts = MockPrompts(
{
r"Select the source": "0",
r"Select the source": "10",
r"Do you want to Skip Validation": "No",
r"Enter catalog_name": "test",
r".*Do you want to create a new one?": "no",
Expand All @@ -174,7 +174,7 @@ def test_create_catalog_no(ws, mock_installation):
def test_create_schema_no(ws, mock_installation):
prompts = MockPrompts(
{
r"Select the source": "0",
r"Select the source": "10",
r"Do you want to Skip Validation": "No",
r"Enter catalog_name": "test",
r".*Do you want to create a new one?": "yes",
Expand Down
Loading