From 988becdce10ca6c80a960e3daa10f8cd56ad069c Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Mon, 16 Oct 2023 11:15:21 -0400 Subject: [PATCH 01/32] Skip IdentityReflectionTest as it doesn't seem possible at this time Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 4b13dbeee..fe24f638f 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -548,3 +548,12 @@ class DifficultParametersTest: we have opted to defer implementing fixes to a later time, guided by customer feedback. Passage of these tests is not an acceptance criteria for our dialect. """ + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Identity reflection is not implemented in this dialect. See test_suite.py") +class IdentityReflectionTest(IdentityReflectionTest): + """It's not clear _how_ to implement this for SQLAlchemy. Columns created with GENERATED ALWAYS AS IDENTITY + are not specially demarked in the output of TGetColumnsResponse or DESCRIBE TABLE EXTENDED. + + We could theoretically parse this from the contents of `SHOW CREATE TABLE` but that feels like a hack. + """ \ No newline at end of file From fad7325e18f617ca36e0a174bc0f642817408d5e Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Mon, 16 Oct 2023 11:23:44 -0400 Subject: [PATCH 02/32] Make TrueDivTest pass For SQLAlchemy's floor division to work on databricks, SQLA needs to render FLOOR() functions. It does not do this by default. By setting div_is_floordiv = False, SQLA knows to render the FLOOR() function. Copped this from the MySQL dialect Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/base.py | 1 + src/databricks/sqlalchemy/test/test_suite.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/base.py b/src/databricks/sqlalchemy/base.py index fa100f4f7..00c579770 100644 --- a/src/databricks/sqlalchemy/base.py +++ b/src/databricks/sqlalchemy/base.py @@ -60,6 +60,7 @@ class DatabricksDialect(default.DefaultDialect): supports_identity_columns: bool = True supports_schemas: bool = True paramstyle: str = "named" + div_is_floordiv: bool = False colspecs = { sqlalchemy.types.DateTime: dialect_type_impl.DatabricksDateTimeNoTimezoneType, diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index fe24f638f..ecf72e972 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -556,4 +556,8 @@ class IdentityReflectionTest(IdentityReflectionTest): are not specially demarked in the output of TGetColumnsResponse or DESCRIBE TABLE EXTENDED. We could theoretically parse this from the contents of `SHOW CREATE TABLE` but that feels like a hack. - """ \ No newline at end of file + """ + +@pytest.mark.reviewed +class TrueDivTest(TrueDivTest): + pass \ No newline at end of file From b64d2f71427ed89fd80e8c7cb89de4cce9492059 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Mon, 16 Oct 2023 11:24:17 -0400 Subject: [PATCH 03/32] Remove duplicate test skip directive Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index ecf72e972..ca8bdae66 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -359,15 +359,6 @@ def test_get_foreign_key_options(self): pass -class DifficultParametersTest(DifficultParametersTest): - @pytest.mark.skip(reason="Error during execution. Requires investigation.") - def test_round_trip_same_named_column(self): - """ - Exception: - - sqlalchemy.exc.DatabaseError: (databricks.sql.exc.ServerOperationError) Found invalid character(s) among ' ,;{}()\n\t=' in the column names of your schema. - """ - - class InsertBehaviorTest(InsertBehaviorTest): @pytest.mark.skip(reason="Error during execution. Requires investigation.") def test_autoclose_on_insert(self): From 814093dbea487487346f920ba96dc231f9a60619 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Mon, 16 Oct 2023 13:17:40 -0400 Subject: [PATCH 04/32] Add skip markers for InsertBehaviorTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/base.py | 1 + src/databricks/sqlalchemy/test/test_suite.py | 81 +++++++++++--------- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/src/databricks/sqlalchemy/base.py b/src/databricks/sqlalchemy/base.py index 00c579770..c59c066ea 100644 --- a/src/databricks/sqlalchemy/base.py +++ b/src/databricks/sqlalchemy/base.py @@ -61,6 +61,7 @@ class DatabricksDialect(default.DefaultDialect): supports_schemas: bool = True paramstyle: str = "named" div_is_floordiv: bool = False + supports_default_values: bool = False colspecs = { sqlalchemy.types.DateTime: dialect_type_impl.DatabricksDateTimeNoTimezoneType, diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index ca8bdae66..9c4108b69 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -359,52 +359,53 @@ def test_get_foreign_key_options(self): pass +@pytest.mark.reviewed class InsertBehaviorTest(InsertBehaviorTest): - @pytest.mark.skip(reason="Error during execution. Requires investigation.") - def test_autoclose_on_insert(self): - """ - Exception: - - sqlalchemy.exc.DatabaseError: (databricks.sql.exc.ServerOperationError) Column id is not specified in INSERT + @pytest.mark.skip( + reason="Databricks dialect doesn't implement empty inserts. See test_suite.py" + ) + def test_empty_insert(self): + """Empty inserts are possible using DEFAULT VALUES on Databricks. To implement it, we need + to hook into the SQLCompiler to render a no-op column list. With SQLAlchemy's default implementation + the request fails with a syntax error """ + pass - @pytest.mark.skip(reason="Error during execution. Requires investigation.") - def test_empty_insert(self): + @pytest.mark.skip( + reason="Databricks dialect doesn't implement empty inserts. See test_suite.py" + ) + def test_empty_insert_multiple(self): + """Empty inserts are possible using DEFAULT VALUES on Databricks. To implement it, we need + to hook into the SQLCompiler to render a no-op column list. With SQLAlchemy's default implementation + the request fails with a syntax error """ - Exception: - - sqlalchemy.exc.DatabaseError: (databricks.sql.exc.ServerOperationError) + pass + + @pytest.mark.skip( + reason="Test setup relies on implicit autoincrement. See test_suite.py" + ) + def test_autoclose_on_insert(self): + """The setup for this test creates a column with implicit autoincrement enabled. + This dialect does not implement implicit autoincrement - users must declare Identity() explicitly. """ + pass - @pytest.mark.skip(reason="Error during execution. Requires investigation.") + @pytest.mark.skip( + reason="Test setup relies on implicit autoincrement. See test_suite.py" + ) def test_insert_from_select_autoinc(self): - """ - Exception: - - sqlalchemy.exc.DatabaseError: (databricks.sql.exc.ServerOperationError) Column id is not specified in INSERT - """ + """Implicit autoincrement is not implemented in this dialect.""" + pass - @pytest.mark.skip(reason="Error during execution. Requires investigation.") + @pytest.mark.skip( + reason="Test setup relies on implicit autoincrement. See test_suite.py" + ) def test_insert_from_select_autoinc_no_rows(self): - """ - Exception: - - sqlalchemy.exc.DatabaseError: (databricks.sql.exc.ServerOperationError) Column id is not specified in INSERT - """ + pass - @pytest.mark.skip(reason="Databricks doesn't support empty INSERT.") - def test_empty_insert_multiple(self): - """ - Exception: - sqlalchemy.exc.DatabaseError: (databricks.sql.exc.ServerOperationError) - - E sqlalchemy.exc.DatabaseError: (databricks.sql.exc.ServerOperationError) - E [PARSE_SYNTAX_ERROR] Syntax error at or near ')'.(line 1, pos 24) - E - E == SQL == - E INSERT INTO autoinc_pk () VALUES () - E ------------------------^^^ - E - E [SQL: INSERT INTO autoinc_pk () VALUES ()] - E [parameters: ({}, {}, {})] - E (Background on this error at: https://sqlalche.me/e/14/4xp6) - """ + @pytest.mark.skip(reason="Databricks doesn't support INSERT ... RETURNING syntax") + def test_autoclose_on_insert_implicit_returning(self): + pass @pytest.mark.reviewed @@ -540,8 +541,11 @@ class DifficultParametersTest: these tests is not an acceptance criteria for our dialect. """ + @pytest.mark.reviewed -@pytest.mark.skip(reason="Identity reflection is not implemented in this dialect. See test_suite.py") +@pytest.mark.skip( + reason="Identity reflection is not implemented in this dialect. See test_suite.py" +) class IdentityReflectionTest(IdentityReflectionTest): """It's not clear _how_ to implement this for SQLAlchemy. Columns created with GENERATED ALWAYS AS IDENTITY are not specially demarked in the output of TGetColumnsResponse or DESCRIBE TABLE EXTENDED. @@ -549,6 +553,7 @@ class IdentityReflectionTest(IdentityReflectionTest): We could theoretically parse this from the contents of `SHOW CREATE TABLE` but that feels like a hack. """ + @pytest.mark.reviewed class TrueDivTest(TrueDivTest): - pass \ No newline at end of file + pass From 67fb1de329966657ee092d1398a4fd94fa08beee Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Mon, 16 Oct 2023 13:39:05 -0400 Subject: [PATCH 05/32] Explicitly mark these test cases as reviewed All tests in these cases pass. Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 64 ++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 9c4108b69..4b74481dd 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -557,3 +557,67 @@ class IdentityReflectionTest(IdentityReflectionTest): @pytest.mark.reviewed class TrueDivTest(TrueDivTest): pass + +@pytest.mark.reviewed +class ArgSignatureTest(ArgSignatureTest): + pass + +@pytest.mark.reviewed +class CompoundSelectTest(CompoundSelectTest): + pass + +@pytest.mark.reviewed +class DeprecatedCompoundSelectTest(DeprecatedCompoundSelectTest): + pass + +@pytest.mark.reviewed +class CastTypeDecoratorTest(CastTypeDecoratorTest): + pass + +@pytest.mark.reviewed +class DistinctOnTest(DistinctOnTest): + pass + +@pytest.mark.reviewed +class EscapingTest(EscapingTest): + pass + +@pytest.mark.reviewed +class ExistsTest(ExistsTest): + pass + +@pytest.mark.reviewed +class IntegerTest(IntegerTest): + pass + +@pytest.mark.reviewed +class IsOrIsNotDistinctFromTest(IsOrIsNotDistinctFromTest): + pass + +@pytest.mark.reviewed +class JoinTest(JoinTest): + pass + +@pytest.mark.reviewed +class OrderByLabelTest(OrderByLabelTest): + pass + +@pytest.mark.reviewed +class PingTest(PingTest): + pass + +@pytest.mark.reviewed +class ReturningGuardsTest(ReturningGuardsTest): + pass + +@pytest.mark.reviewed +class SameNamedSchemaTableTest(SameNamedSchemaTableTest): + pass + +@pytest.mark.reviewed +class UnicodeTextTest(UnicodeTextTest): + pass + +@pytest.mark.reviewed +class UnicodeVarcharTest(UnicodeVarcharTest): + pass \ No newline at end of file From 472bcb29e7e2ae51e19cc23623795bc06637fa78 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Mon, 16 Oct 2023 13:41:36 -0400 Subject: [PATCH 06/32] Call out tests for types we can't bind Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 4b74481dd..44bdd4525 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -24,6 +24,7 @@ # See further: https://github.com/sqlalchemy/sqlalchemy/blob/rel_1_4_48/README.dialects.rst +@pytest.mark.reviewed @pytest.mark.skip(reason="pysql doesn't support binding of BINARY type parameters") class BinaryTest(BinaryTest): pass @@ -620,4 +621,11 @@ class UnicodeTextTest(UnicodeTextTest): @pytest.mark.reviewed class UnicodeVarcharTest(UnicodeVarcharTest): - pass \ No newline at end of file + pass + +@pytest.mark.skip(reason="pysql doesn't support binding of array parameters. See test_suite.py") +class ArrayTest(ArrayTest): + """While Databricks supports ARRAY types, DBR cannot handle bound parameters of this type. + This makes them unusable to SQLAlchemy without some workaround. Potentially we could inline + the values of these parameters (which risks sql injection). + """ \ No newline at end of file From 07c73a6ef7a8af8d231903e5eefa5f488d241aff Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Mon, 16 Oct 2023 13:46:37 -0400 Subject: [PATCH 07/32] Skip JSONTest. Will use PM input to guide when we implement this Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 44bdd4525..ff1dadd77 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -628,4 +628,10 @@ class ArrayTest(ArrayTest): """While Databricks supports ARRAY types, DBR cannot handle bound parameters of this type. This makes them unusable to SQLAlchemy without some workaround. Potentially we could inline the values of these parameters (which risks sql injection). - """ \ No newline at end of file + """ + +@pytest.mark.skip(reason="Databricks dialect doesn't implement JSON column types. See test_suite.py") +class JSONTest(JSONTest): + """Databricks supports JSON path expressions in queries it's just not implemented in this dialect. + """ + pass \ No newline at end of file From df710cf586d38d3bce30c5b8f7b9fd787fbb5846 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Mon, 16 Oct 2023 14:29:44 -0400 Subject: [PATCH 08/32] Add more skip markers for unsupported features Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/requirements.py | 9 +++ src/databricks/sqlalchemy/test/test_suite.py | 70 ++++++++++++++++++-- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/src/databricks/sqlalchemy/requirements.py b/src/databricks/sqlalchemy/requirements.py index 6fb252dbc..85538794e 100644 --- a/src/databricks/sqlalchemy/requirements.py +++ b/src/databricks/sqlalchemy/requirements.py @@ -186,4 +186,13 @@ def unique_constraint_reflection(self): def reflects_pk_names(self): """Target driver reflects the name of primary key constraints.""" + return sqlalchemy.testing.exclusions.open() + + @property + def datetime_implicit_bound(self): + """target dialect when given a datetime object will bind it such + that the database server knows the object is a date, and not + a plain string. + """ + return sqlalchemy.testing.exclusions.open() \ No newline at end of file diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index ff1dadd77..4369e7885 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -559,79 +559,141 @@ class IdentityReflectionTest(IdentityReflectionTest): class TrueDivTest(TrueDivTest): pass + @pytest.mark.reviewed class ArgSignatureTest(ArgSignatureTest): pass + @pytest.mark.reviewed class CompoundSelectTest(CompoundSelectTest): pass + @pytest.mark.reviewed class DeprecatedCompoundSelectTest(DeprecatedCompoundSelectTest): pass + @pytest.mark.reviewed class CastTypeDecoratorTest(CastTypeDecoratorTest): pass + @pytest.mark.reviewed class DistinctOnTest(DistinctOnTest): pass + @pytest.mark.reviewed class EscapingTest(EscapingTest): pass + @pytest.mark.reviewed class ExistsTest(ExistsTest): pass + @pytest.mark.reviewed class IntegerTest(IntegerTest): pass + @pytest.mark.reviewed class IsOrIsNotDistinctFromTest(IsOrIsNotDistinctFromTest): pass + @pytest.mark.reviewed class JoinTest(JoinTest): pass + @pytest.mark.reviewed class OrderByLabelTest(OrderByLabelTest): pass + @pytest.mark.reviewed class PingTest(PingTest): pass + @pytest.mark.reviewed class ReturningGuardsTest(ReturningGuardsTest): pass + @pytest.mark.reviewed class SameNamedSchemaTableTest(SameNamedSchemaTableTest): pass + @pytest.mark.reviewed class UnicodeTextTest(UnicodeTextTest): pass + @pytest.mark.reviewed class UnicodeVarcharTest(UnicodeVarcharTest): pass -@pytest.mark.skip(reason="pysql doesn't support binding of array parameters. See test_suite.py") + +@pytest.mark.skip( + reason="pysql doesn't support binding of array parameters. See test_suite.py" +) class ArrayTest(ArrayTest): """While Databricks supports ARRAY types, DBR cannot handle bound parameters of this type. - This makes them unusable to SQLAlchemy without some workaround. Potentially we could inline + This makes them unusable to SQLAlchemy without some workaround. Potentially we could inline the values of these parameters (which risks sql injection). """ -@pytest.mark.skip(reason="Databricks dialect doesn't implement JSON column types. See test_suite.py") + +@pytest.mark.skip( + reason="Databricks dialect doesn't implement JSON column types. See test_suite.py" +) class JSONTest(JSONTest): - """Databricks supports JSON path expressions in queries it's just not implemented in this dialect. + """Databricks supports JSON path expressions in queries it's just not implemented in this dialect.""" + + pass + + +@pytest.mark.skip(reason="Databricks doesn't support INSERT ... RETURNING syntax") +class ReturningText(ReturningTest): + pass + + +@pytest.mark.reviewed +class LikeFunctionsTest(LikeFunctionsTest): + @pytest.mark.skip( + reason="Databricks dialect doesn't implement regexp features. See test_suite.py" + ) + def test_not_regexp_match(self): + """The defaul dialect doesn't implement _visit_regexp methods so we don't get them automatically.""" + pass + + @pytest.mark.skip( + reason="Databricks dialect doesn't implement regexp features. See test_suite.py" + ) + def test_regexp_match(self): + """The defaul dialect doesn't implement _visit_regexp methods so we don't get them automatically.""" + pass + + +@pytest.mark.reviewed +class UuidTest(UuidTest): + + @pytest.mark.skip(reason="Databricks doesn't support INSERT ... RETURNING syntax") + def test_uuid_returning(self): + pass + + +@pytest.mark.skip(reason="Datetime handling doesn't handle timezones well. Priority to fix.") +class DateTimeTZTest(DateTimeTZTest): + """When I initially implemented DateTime type handling, I started using TIMESTAMP_NTZ because + that's the default behaviour of the DateTime() type and the other tests passed. I simply missed + this group of tests. Will need to modify the compilation and result_processor for our type override + so that we can pass both DateTimeTZTest and DateTimeTest. Currently, only DateTimeTest passes. """ pass \ No newline at end of file From e6a75c7ae5379802bc08090724f279b3c1ee76fa Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Mon, 16 Oct 2023 14:44:37 -0400 Subject: [PATCH 09/32] Add support for ExpandingBoundInTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/requirements.py | 4 +++ src/databricks/sqlalchemy/test/test_suite.py | 32 ++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/databricks/sqlalchemy/requirements.py b/src/databricks/sqlalchemy/requirements.py index 85538794e..e32ab5dd3 100644 --- a/src/databricks/sqlalchemy/requirements.py +++ b/src/databricks/sqlalchemy/requirements.py @@ -195,4 +195,8 @@ def datetime_implicit_bound(self): a plain string. """ + return sqlalchemy.testing.exclusions.open() + + @property + def tuple_in(self): return sqlalchemy.testing.exclusions.open() \ No newline at end of file diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 4369e7885..012fcd430 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -683,17 +683,43 @@ def test_regexp_match(self): @pytest.mark.reviewed class UuidTest(UuidTest): - @pytest.mark.skip(reason="Databricks doesn't support INSERT ... RETURNING syntax") def test_uuid_returning(self): pass -@pytest.mark.skip(reason="Datetime handling doesn't handle timezones well. Priority to fix.") +@pytest.mark.skip( + reason="Datetime handling doesn't handle timezones well. Priority to fix." +) class DateTimeTZTest(DateTimeTZTest): """When I initially implemented DateTime type handling, I started using TIMESTAMP_NTZ because that's the default behaviour of the DateTime() type and the other tests passed. I simply missed this group of tests. Will need to modify the compilation and result_processor for our type override so that we can pass both DateTimeTZTest and DateTimeTest. Currently, only DateTimeTest passes. """ - pass \ No newline at end of file + + pass + + +TUPLES_READ_AS_STRUCT_MSG = ( + "Databricks interprets tuple-like IN markers as though they are structs." +) + + +@pytest.mark.reviewed +class ExpandingBoundInTest(ExpandingBoundInTest): + @pytest.mark.skip(reason=TUPLES_READ_AS_STRUCT_MSG) + def test_empty_heterogeneous_tuples_bindparam(self): + pass + + @pytest.mark.skip(reason=TUPLES_READ_AS_STRUCT_MSG) + def test_empty_heterogeneous_tuples_direct(self): + pass + + @pytest.mark.skip(reason=TUPLES_READ_AS_STRUCT_MSG) + def test_empty_homogeneous_tuples_bindparam(self): + pass + + @pytest.mark.skip(reason=TUPLES_READ_AS_STRUCT_MSG) + def test_empty_homogeneous_tuples_direct(self): + pass From 08114a4e6f023134aed20c87add1427f4c909092 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Mon, 16 Oct 2023 14:47:08 -0400 Subject: [PATCH 10/32] Skip sequence related tests Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 012fcd430..54ca03da6 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -723,3 +723,13 @@ def test_empty_homogeneous_tuples_bindparam(self): @pytest.mark.skip(reason=TUPLES_READ_AS_STRUCT_MSG) def test_empty_homogeneous_tuples_direct(self): pass + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks doesn't support SEQUENCE server defaults") +class HasSequenceTest(HasSequenceTest): + pass + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks doesn't support SEQUENCE server defaults") +class HasSequenceTestEmpty(HasSequenceTestEmpty): + pass \ No newline at end of file From 907e2ecf14d9cd5d8428a8c3f482c0a6730e610d Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Tue, 17 Oct 2023 15:07:24 -0400 Subject: [PATCH 11/32] Add CTETest skip markers and justification Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/requirements.py | 13 ++++++++++ src/databricks/sqlalchemy/test/test_suite.py | 27 +++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/requirements.py b/src/databricks/sqlalchemy/requirements.py index e32ab5dd3..02e8d986d 100644 --- a/src/databricks/sqlalchemy/requirements.py +++ b/src/databricks/sqlalchemy/requirements.py @@ -199,4 +199,17 @@ def datetime_implicit_bound(self): @property def tuple_in(self): + return sqlalchemy.testing.exclusions.open() + + @property + def ctes(self): + return sqlalchemy.testing.exclusions.open() + + @property + def ctes_with_update_delete(self): + return sqlalchemy.testing.exclusions.open() + + @property + def delete_from(self): + """Target must support DELETE FROM..FROM or DELETE..USING syntax""" return sqlalchemy.testing.exclusions.open() \ No newline at end of file diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 54ca03da6..270aa6f42 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -732,4 +732,29 @@ class HasSequenceTest(HasSequenceTest): @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks doesn't support SEQUENCE server defaults") class HasSequenceTestEmpty(HasSequenceTestEmpty): - pass \ No newline at end of file + pass + +@pytest.mark.reviewed +class CTETest(CTETest): + """During the teardown for this test block, it tries to drop a constraint that it never named which raises + a compilation error. This could point to poor constraint reflection but our other constraint reflection + tests pass. Requires investigation. + """ + + @pytest.mark.skip(reason="Databricks dialect doesn't implement multiple-table criteria within DELETE") + def test_delete_from_round_trip(self): + """This may be supported by Databricks but has not been implemented here. + """ + pass + + @pytest.mark.skip(reason="Databricks doesn't support recursive CTE") + def test_select_recursive_round_trip(self): + pass + + @pytest.mark.skip(reason="Unsupported by Databricks. See test_suite.py") + def test_delete_scalar_subq_round_trip(self): + """Error received is [UNSUPPORTED_SUBQUERY_EXPRESSION_CATEGORY.MUST_AGGREGATE_CORRELATED_SCALAR_SUBQUERY] + + This suggests a limitation of the platform. But a workaround may be possible if customers require it. + """ + pass From 28e63063831f5e108a3b603e3522f6017515dfef Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Tue, 17 Oct 2023 15:12:56 -0400 Subject: [PATCH 12/32] Skip WeCanSetDefaultSchemaWEventsTest with reasoning Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 270aa6f42..07918c06b 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -758,3 +758,14 @@ def test_delete_scalar_subq_round_trip(self): This suggests a limitation of the platform. But a workaround may be possible if customers require it. """ pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Dialect doesn't implement provision.py See test_suite.py") +class WeCanSetDefaultSchemaWEventsTest(WeCanSetDefaultSchemaWEventsTest): + """provision.py allows us to define event listeners that emit DDL for things like setting up a test schema + or, in this case, changing the default schema for the connection after it's been built. This would override + the schema defined in the sqlalchemy connection string. This support is possible but is not implemented + in the dialect. Deferred for now. + """ + pass \ No newline at end of file From f64cb874e114bbad9bcc733706b607c59363e4a5 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Tue, 17 Oct 2023 15:14:06 -0400 Subject: [PATCH 13/32] Stop skipping ValuesExpressionTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/requirements.py | 4 ++++ src/databricks/sqlalchemy/test/test_suite.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/databricks/sqlalchemy/requirements.py b/src/databricks/sqlalchemy/requirements.py index 02e8d986d..f0d3684d1 100644 --- a/src/databricks/sqlalchemy/requirements.py +++ b/src/databricks/sqlalchemy/requirements.py @@ -212,4 +212,8 @@ def ctes_with_update_delete(self): @property def delete_from(self): """Target must support DELETE FROM..FROM or DELETE..USING syntax""" + return sqlalchemy.testing.exclusions.open() + + @property + def table_value_constructor(self): return sqlalchemy.testing.exclusions.open() \ No newline at end of file diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 07918c06b..d008841ea 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -768,4 +768,8 @@ class WeCanSetDefaultSchemaWEventsTest(WeCanSetDefaultSchemaWEventsTest): the schema defined in the sqlalchemy connection string. This support is possible but is not implemented in the dialect. Deferred for now. """ + pass + +@pytest.mark.reviewed +class ValuesExpressionTest(ValuesExpressionTest): pass \ No newline at end of file From 266df13cfc2f572ab84a1fa5d9f2584592a353bf Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Tue, 17 Oct 2023 15:16:50 -0400 Subject: [PATCH 14/32] Skip UnicodeSchemaTest with reasoning Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index d008841ea..54c6f3f56 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -772,4 +772,9 @@ class WeCanSetDefaultSchemaWEventsTest(WeCanSetDefaultSchemaWEventsTest): @pytest.mark.reviewed class ValuesExpressionTest(ValuesExpressionTest): + pass + +@pytest.mark.reviewed +@pytest.mark.skipped(reason="Databricks doesn't support unicode in symbol names") +class UnicodeSchemaTest(UnicodeSchemaTest): pass \ No newline at end of file From c8cae780a70fc8855901ccf37e1f2f0692b955ab Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 11:33:01 -0400 Subject: [PATCH 15/32] Stop skipping TableNoColumnsTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/base.py | 7 ++++++- src/databricks/sqlalchemy/requirements.py | 4 ++++ src/databricks/sqlalchemy/test/test_suite.py | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/base.py b/src/databricks/sqlalchemy/base.py index c59c066ea..0f5bb9ee0 100644 --- a/src/databricks/sqlalchemy/base.py +++ b/src/databricks/sqlalchemy/base.py @@ -111,7 +111,12 @@ def get_columns( ).fetchall() if not resp: - raise sqlalchemy.exc.NoSuchTableError(table_name) + # TGetColumnsRequest will not raise an exception if passed a table that doesn't exist + # But Databricks supports tables with no columns. So if the result is an empty list, + # we need to check if the table exists (and raise an exception if not) or simply return + # an empty list. + self._describe_table_extended(connection, table_name, self.catalog, schema or self.schema, expect_result=False) + return resp columns = [] for col in resp: row_dict = parse_column_info_from_tgetcolumnsresponse(col) diff --git a/src/databricks/sqlalchemy/requirements.py b/src/databricks/sqlalchemy/requirements.py index f0d3684d1..7196d24a0 100644 --- a/src/databricks/sqlalchemy/requirements.py +++ b/src/databricks/sqlalchemy/requirements.py @@ -216,4 +216,8 @@ def delete_from(self): @property def table_value_constructor(self): + return sqlalchemy.testing.exclusions.open() + + @property + def reflect_tables_no_columns(self): return sqlalchemy.testing.exclusions.open() \ No newline at end of file diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 54c6f3f56..a86bb0971 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -777,4 +777,5 @@ class ValuesExpressionTest(ValuesExpressionTest): @pytest.mark.reviewed @pytest.mark.skipped(reason="Databricks doesn't support unicode in symbol names") class UnicodeSchemaTest(UnicodeSchemaTest): +class TableNoColumnsTest(TableNoColumnsTest): pass \ No newline at end of file From 691b5aab8916b6eb5be127aee4e8f2ebbe9966e3 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 11:34:56 -0400 Subject: [PATCH 16/32] Skip ServerSideCursorsTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/base.py | 1 + src/databricks/sqlalchemy/test/test_suite.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/databricks/sqlalchemy/base.py b/src/databricks/sqlalchemy/base.py index 0f5bb9ee0..fda295c1e 100644 --- a/src/databricks/sqlalchemy/base.py +++ b/src/databricks/sqlalchemy/base.py @@ -62,6 +62,7 @@ class DatabricksDialect(default.DefaultDialect): paramstyle: str = "named" div_is_floordiv: bool = False supports_default_values: bool = False + supports_server_side_cursors: bool = False colspecs = { sqlalchemy.types.DateTime: dialect_type_impl.DatabricksDateTimeNoTimezoneType, diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index a86bb0971..5ce63ee19 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -777,5 +777,13 @@ class ValuesExpressionTest(ValuesExpressionTest): @pytest.mark.reviewed @pytest.mark.skipped(reason="Databricks doesn't support unicode in symbol names") class UnicodeSchemaTest(UnicodeSchemaTest): + pass + +@pytest.mark.reviewed class TableNoColumnsTest(TableNoColumnsTest): + pass + +@pytes.mark.reviewed +@pytest.mark.skip(reason="Databricks doesn't support server-side cursors.") +class ServerSideCursorsTest(ServerSideCursorsTest): pass \ No newline at end of file From d4e0df9b78ff624464a8bfe2659b074baf2fa2c5 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 11:36:28 -0400 Subject: [PATCH 17/32] Skip SequenceTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/base.py | 1 + src/databricks/sqlalchemy/test/test_suite.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/base.py b/src/databricks/sqlalchemy/base.py index fda295c1e..7300609c3 100644 --- a/src/databricks/sqlalchemy/base.py +++ b/src/databricks/sqlalchemy/base.py @@ -63,6 +63,7 @@ class DatabricksDialect(default.DefaultDialect): div_is_floordiv: bool = False supports_default_values: bool = False supports_server_side_cursors: bool = False + supports_sequences: bool = False colspecs = { sqlalchemy.types.DateTime: dialect_type_impl.DatabricksDateTimeNoTimezoneType, diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 5ce63ee19..36dbf5887 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -783,7 +783,12 @@ class UnicodeSchemaTest(UnicodeSchemaTest): class TableNoColumnsTest(TableNoColumnsTest): pass -@pytes.mark.reviewed +@pytest.mark.reviewed @pytest.mark.skip(reason="Databricks doesn't support server-side cursors.") class ServerSideCursorsTest(ServerSideCursorsTest): + pass + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks does not support sequences.") +class SequenceTest(SequenceTest): pass \ No newline at end of file From 03dd219885b2c8e55a675201ebcb0309ac5ce070 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 11:44:47 -0400 Subject: [PATCH 18/32] Skip RowCountTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/base.py | 2 +- src/databricks/sqlalchemy/test/test_suite.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/base.py b/src/databricks/sqlalchemy/base.py index 7300609c3..3a8445b1c 100644 --- a/src/databricks/sqlalchemy/base.py +++ b/src/databricks/sqlalchemy/base.py @@ -45,7 +45,7 @@ class DatabricksImpl(DefaultImpl): class DatabricksDialect(default.DefaultDialect): """This dialect implements only those methods required to pass our e2e tests""" - # Possible attributes are defined here: https://docs.sqlalchemy.org/en/14/core/internals.html#sqlalchemy.engine.Dialect + # See sqlalchemy.engine.interfaces for descriptions of each of these properties name: str = "databricks" driver: str = "databricks" default_schema_name: str = "default" diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 36dbf5887..dcdf0205c 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -791,4 +791,9 @@ class ServerSideCursorsTest(ServerSideCursorsTest): @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks does not support sequences.") class SequenceTest(SequenceTest): + pass + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks dialect does not implement sane rowcount.") +class RowCountTest(RowCountTest): pass \ No newline at end of file From a7a7968795cf37097934fe6f53c5d763e9df5b3b Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 11:49:17 -0400 Subject: [PATCH 19/32] Add PostCompileParamsTest Was originally skipped before i defined that requirements.tuple_in is open() Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index dcdf0205c..a4e50d604 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -796,4 +796,8 @@ class SequenceTest(SequenceTest): @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks dialect does not implement sane rowcount.") class RowCountTest(RowCountTest): + pass + +@pytest.mark.reviewed +class PostCompileParamsTest(PostCompileParamsTest): pass \ No newline at end of file From 86c4788bef33fa7496019cfa7cc4bb907d29d5e7 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 11:52:54 -0400 Subject: [PATCH 20/32] Skip NativeUUIDTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index a4e50d604..e17f18c5e 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -800,4 +800,13 @@ class RowCountTest(RowCountTest): @pytest.mark.reviewed class PostCompileParamsTest(PostCompileParamsTest): - pass \ No newline at end of file + pass + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks dialect doesn't implement UUID type. See test_suite.py") +class NativeUUIDTest(NativeUUIDTest): + """Type implementation will be straightforward. Since Databricks doesn't have a native UUID type we can use + a STRING field, create a custom TypeDecorator for sqlalchemy.types.Uuid and add it to the dialect's colspecs. + + Then mark requirements.uuid_data_type as open() so this test can run. + """ \ No newline at end of file From 7ddd1b2001c746b60aa6aa5ec1a1b2ca45745157 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 11:55:04 -0400 Subject: [PATCH 21/32] Skip PercentSchemaNamesTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index e17f18c5e..49a0048a0 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -809,4 +809,9 @@ class NativeUUIDTest(NativeUUIDTest): a STRING field, create a custom TypeDecorator for sqlalchemy.types.Uuid and add it to the dialect's colspecs. Then mark requirements.uuid_data_type as open() so this test can run. - """ \ No newline at end of file + """ + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks doesn't allow percent signs in identifiers") +class PercentSchemaNamesTest(PercentSchemaNamesTest): + pass \ No newline at end of file From 9d323a0884c370f0273c527f0a91f16eb15150be Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 11:56:52 -0400 Subject: [PATCH 22/32] Skip IsolationLevelTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 49a0048a0..b58d8660b 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -814,4 +814,9 @@ class NativeUUIDTest(NativeUUIDTest): @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks doesn't allow percent signs in identifiers") class PercentSchemaNamesTest(PercentSchemaNamesTest): + pass + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks does not support transactions") +class IsolationLevelTest(IsolationLevelTest): pass \ No newline at end of file From 53db2ef75ad17c5bbd04a77d807febf61abe0d4e Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 11:57:39 -0400 Subject: [PATCH 23/32] Skip FutureWeCanSetDefaultSchemaWEventsTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index b58d8660b..9ceb15c10 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -770,6 +770,17 @@ class WeCanSetDefaultSchemaWEventsTest(WeCanSetDefaultSchemaWEventsTest): """ pass +@pytest.mark.reviewed +@pytest.mark.skip(reason="Dialect doesn't implement provision.py See test_suite.py") +class FutureWeCanSetDefaultSchemaWEventsTest(FutureWeCanSetDefaultSchemaWEventsTest): + """provision.py allows us to define event listeners that emit DDL for things like setting up a test schema + or, in this case, changing the default schema for the connection after it's been built. This would override + the schema defined in the sqlalchemy connection string. This support is possible but is not implemented + in the dialect. Deferred for now. + """ + pass + + @pytest.mark.reviewed class ValuesExpressionTest(ValuesExpressionTest): pass From 5abfa188b8986f4568a9ad8272d1d4452c2c912a Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 12:00:18 -0400 Subject: [PATCH 24/32] Skip JSONLegacyStringCastIndexTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 9ceb15c10..d3e9f70f6 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -650,6 +650,7 @@ class ArrayTest(ArrayTest): """ +@pytest.mark.reviewed @pytest.mark.skip( reason="Databricks dialect doesn't implement JSON column types. See test_suite.py" ) @@ -658,6 +659,14 @@ class JSONTest(JSONTest): pass +@pytest.mark.reviewed +@pytest.mark.skip( + reason="Databricks dialect doesn't implement JSON column types. See test_suite.py" +) +class JSONLegacyStringCastIndexTest(JSONLegacyStringCastIndexTest): + """Same comment applies as JSONTest + """ + pass @pytest.mark.skip(reason="Databricks doesn't support INSERT ... RETURNING syntax") class ReturningText(ReturningTest): From 0730eb9def1694705016e376d01e7f228c6fe215 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 12:01:15 -0400 Subject: [PATCH 25/32] Skip AutocommitIsolationTest( Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index d3e9f70f6..cb114e956 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -839,4 +839,9 @@ class PercentSchemaNamesTest(PercentSchemaNamesTest): @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks does not support transactions") class IsolationLevelTest(IsolationLevelTest): + pass + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks does not support transactions") +class AutocommitIsolationTest(AutocommitIsolationTest): pass \ No newline at end of file From 8b9a1402a45067fa558f5e09e35e825c5a6a4380 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 12:05:23 -0400 Subject: [PATCH 26/32] Skip CollateTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index cb114e956..76649f344 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -844,4 +844,10 @@ class IsolationLevelTest(IsolationLevelTest): @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks does not support transactions") class AutocommitIsolationTest(AutocommitIsolationTest): - pass \ No newline at end of file + pass + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks dialect does not implement COLLATE support") +class CollateTest(CollateTest): + """This is supported in Databricks. Not implemented here. + """ \ No newline at end of file From ed549ba46d860290340cb4ac3bc6940671d5cb71 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 12:07:42 -0400 Subject: [PATCH 27/32] Skip ComputedReflectionTest and ComputedColumnTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 76649f344..d7351e497 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -850,4 +850,14 @@ class AutocommitIsolationTest(AutocommitIsolationTest): @pytest.mark.skip(reason="Databricks dialect does not implement COLLATE support") class CollateTest(CollateTest): """This is supported in Databricks. Not implemented here. - """ \ No newline at end of file + """ + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks does not support computed / generated columns") +class ComputedColumnTest(ComputedColumnTest): + pass + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks does not support computed / generated columns") +class ComputedReflectionTest(ComputedReflectionTest): + pass \ No newline at end of file From 4a53facfc5383e84ebfe9465d91b16b100a8c4ad Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 12:08:25 -0400 Subject: [PATCH 28/32] Skip SimpleUpdateDeleteTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index d7351e497..7e9b3c9a4 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -818,6 +818,11 @@ class SequenceTest(SequenceTest): class RowCountTest(RowCountTest): pass +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks dialect does not implement sane rowcount.") +class SimpleUpdateDeleteTest(SimpleUpdateDeleteTest): + pass + @pytest.mark.reviewed class PostCompileParamsTest(PostCompileParamsTest): pass From 7ccac16b2f1a74aa75d4069a30cf3b38103a8990 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 12:09:15 -0400 Subject: [PATCH 29/32] Skip SequenceCompilerTest Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 7e9b3c9a4..75bec8196 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -813,6 +813,11 @@ class ServerSideCursorsTest(ServerSideCursorsTest): class SequenceTest(SequenceTest): pass +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks does not support sequences.") +class SequenceCompilerTest(SequenceCompilerTest): + pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks dialect does not implement sane rowcount.") class RowCountTest(RowCountTest): From 61c791168abb1320726ee7ce2ea2423b7af0c06d Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 12:17:33 -0400 Subject: [PATCH 30/32] Skip some of NormalizedNameTest with justification Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/requirements.py | 7 +++++++ src/databricks/sqlalchemy/test/test_suite.py | 18 +++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/requirements.py b/src/databricks/sqlalchemy/requirements.py index 7196d24a0..08d770766 100644 --- a/src/databricks/sqlalchemy/requirements.py +++ b/src/databricks/sqlalchemy/requirements.py @@ -220,4 +220,11 @@ def table_value_constructor(self): @property def reflect_tables_no_columns(self): + return sqlalchemy.testing.exclusions.open() + + @property + def denormalized_names(self): + """Target database must have 'denormalized', i.e. + UPPERCASE as case insensitive names.""" + return sqlalchemy.testing.exclusions.open() \ No newline at end of file diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 75bec8196..9be9e41b1 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -870,4 +870,20 @@ class ComputedColumnTest(ComputedColumnTest): @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks does not support computed / generated columns") class ComputedReflectionTest(ComputedReflectionTest): - pass \ No newline at end of file + pass + +@pytest.mark.reviewed +class NormalizedNameTest(NormalizedNameTest): + + @pytest.mark.skip(reason="Poor test design? See test_suite.py") + def test_get_table_names(self): + """I'm not clear how this test can ever pass given that it's assertion looks like this: + + ```python + eq_(tablenames[0].upper(), tablenames[0].lower()) + eq_(tablenames[1].upper(), tablenames[1].lower()) + ``` + + It's forcibly calling .upper() and .lower() on the same string and expecting them to be equal. + """ + pass \ No newline at end of file From d180144a45e79956e3fa4a75e79f58628a3110cc Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 12:22:23 -0400 Subject: [PATCH 31/32] Add final reviewed markers Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/test/test_suite.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 9be9e41b1..3ee1b159a 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -640,6 +640,7 @@ class UnicodeVarcharTest(UnicodeVarcharTest): pass +@pytest.mark.reviewed @pytest.mark.skip( reason="pysql doesn't support binding of array parameters. See test_suite.py" ) @@ -668,6 +669,7 @@ class JSONLegacyStringCastIndexTest(JSONLegacyStringCastIndexTest): """ pass +@pytest.mark.reviewed @pytest.mark.skip(reason="Databricks doesn't support INSERT ... RETURNING syntax") class ReturningText(ReturningTest): pass @@ -697,6 +699,7 @@ def test_uuid_returning(self): pass +@pytest.mark.reviewed @pytest.mark.skip( reason="Datetime handling doesn't handle timezones well. Priority to fix." ) @@ -886,4 +889,17 @@ def test_get_table_names(self): It's forcibly calling .upper() and .lower() on the same string and expecting them to be equal. """ - pass \ No newline at end of file + pass + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks doesn't support INSERT ... RETURNING syntax") +class ReturningTest(ReturningTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason="Databricks dialect does not implement timezone support for Timestamp() types. See test_suite.py") +class TimeTZTest(TimeTZTest): + """Similar to DateTimeTZTest, this should be possible for the dialect since we can override type compilation + and processing in _types.py. Implementation has been deferred. + """ \ No newline at end of file From 2de4e91b4187059736c53e7a9eaa0587e9c85497 Mon Sep 17 00:00:00 2001 From: Jesse Whitehouse Date: Wed, 18 Oct 2023 12:22:58 -0400 Subject: [PATCH 32/32] Black all sqlalchemy source files Signed-off-by: Jesse Whitehouse --- src/databricks/sqlalchemy/__init__.py | 2 +- src/databricks/sqlalchemy/base.py | 8 +++- src/databricks/sqlalchemy/requirements.py | 22 ++++----- src/databricks/sqlalchemy/test/test_suite.py | 50 +++++++++++++++----- 4 files changed, 58 insertions(+), 24 deletions(-) diff --git a/src/databricks/sqlalchemy/__init__.py b/src/databricks/sqlalchemy/__init__.py index 0eed85f33..e1c0d7554 100644 --- a/src/databricks/sqlalchemy/__init__.py +++ b/src/databricks/sqlalchemy/__init__.py @@ -1 +1 @@ -from databricks.sqlalchemy.base import DatabricksDialect \ No newline at end of file +from databricks.sqlalchemy.base import DatabricksDialect diff --git a/src/databricks/sqlalchemy/base.py b/src/databricks/sqlalchemy/base.py index 3a8445b1c..937d0a9ec 100644 --- a/src/databricks/sqlalchemy/base.py +++ b/src/databricks/sqlalchemy/base.py @@ -117,7 +117,13 @@ def get_columns( # But Databricks supports tables with no columns. So if the result is an empty list, # we need to check if the table exists (and raise an exception if not) or simply return # an empty list. - self._describe_table_extended(connection, table_name, self.catalog, schema or self.schema, expect_result=False) + self._describe_table_extended( + connection, + table_name, + self.catalog, + schema or self.schema, + expect_result=False, + ) return resp columns = [] for col in resp: diff --git a/src/databricks/sqlalchemy/requirements.py b/src/databricks/sqlalchemy/requirements.py index 08d770766..b6ff46641 100644 --- a/src/databricks/sqlalchemy/requirements.py +++ b/src/databricks/sqlalchemy/requirements.py @@ -158,7 +158,7 @@ def temporary_tables(self): def table_reflection(self): """target database has general support for table reflection""" return sqlalchemy.testing.exclusions.open() - + @property def temp_table_reflection(self): """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. @@ -181,13 +181,13 @@ def unique_constraint_reflection(self): Databricks doesn't support UNIQUE constraints. """ return sqlalchemy.testing.exclusions.closed() - + @property def reflects_pk_names(self): """Target driver reflects the name of primary key constraints.""" return sqlalchemy.testing.exclusions.open() - + @property def datetime_implicit_bound(self): """target dialect when given a datetime object will bind it such @@ -196,35 +196,35 @@ def datetime_implicit_bound(self): """ return sqlalchemy.testing.exclusions.open() - + @property def tuple_in(self): return sqlalchemy.testing.exclusions.open() - + @property def ctes(self): return sqlalchemy.testing.exclusions.open() - + @property def ctes_with_update_delete(self): return sqlalchemy.testing.exclusions.open() - + @property def delete_from(self): """Target must support DELETE FROM..FROM or DELETE..USING syntax""" return sqlalchemy.testing.exclusions.open() - + @property def table_value_constructor(self): return sqlalchemy.testing.exclusions.open() - + @property def reflect_tables_no_columns(self): return sqlalchemy.testing.exclusions.open() - + @property def denormalized_names(self): """Target database must have 'denormalized', i.e. UPPERCASE as case insensitive names.""" - return sqlalchemy.testing.exclusions.open() \ No newline at end of file + return sqlalchemy.testing.exclusions.open() diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py index 3ee1b159a..ea141336b 100644 --- a/src/databricks/sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -660,15 +660,17 @@ class JSONTest(JSONTest): pass + @pytest.mark.reviewed @pytest.mark.skip( reason="Databricks dialect doesn't implement JSON column types. See test_suite.py" ) class JSONLegacyStringCastIndexTest(JSONLegacyStringCastIndexTest): - """Same comment applies as JSONTest - """ + """Same comment applies as JSONTest""" + pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks doesn't support INSERT ... RETURNING syntax") class ReturningText(ReturningTest): @@ -736,16 +738,19 @@ def test_empty_homogeneous_tuples_bindparam(self): def test_empty_homogeneous_tuples_direct(self): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks doesn't support SEQUENCE server defaults") class HasSequenceTest(HasSequenceTest): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks doesn't support SEQUENCE server defaults") class HasSequenceTestEmpty(HasSequenceTestEmpty): pass + @pytest.mark.reviewed class CTETest(CTETest): """During the teardown for this test block, it tries to drop a constraint that it never named which raises @@ -753,10 +758,11 @@ class CTETest(CTETest): tests pass. Requires investigation. """ - @pytest.mark.skip(reason="Databricks dialect doesn't implement multiple-table criteria within DELETE") + @pytest.mark.skip( + reason="Databricks dialect doesn't implement multiple-table criteria within DELETE" + ) def test_delete_from_round_trip(self): - """This may be supported by Databricks but has not been implemented here. - """ + """This may be supported by Databricks but has not been implemented here.""" pass @pytest.mark.skip(reason="Databricks doesn't support recursive CTE") @@ -780,8 +786,10 @@ class WeCanSetDefaultSchemaWEventsTest(WeCanSetDefaultSchemaWEventsTest): the schema defined in the sqlalchemy connection string. This support is possible but is not implemented in the dialect. Deferred for now. """ + pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Dialect doesn't implement provision.py See test_suite.py") class FutureWeCanSetDefaultSchemaWEventsTest(FutureWeCanSetDefaultSchemaWEventsTest): @@ -790,6 +798,7 @@ class FutureWeCanSetDefaultSchemaWEventsTest(FutureWeCanSetDefaultSchemaWEventsT the schema defined in the sqlalchemy connection string. This support is possible but is not implemented in the dialect. Deferred for now. """ + pass @@ -797,46 +806,57 @@ class FutureWeCanSetDefaultSchemaWEventsTest(FutureWeCanSetDefaultSchemaWEventsT class ValuesExpressionTest(ValuesExpressionTest): pass + @pytest.mark.reviewed @pytest.mark.skipped(reason="Databricks doesn't support unicode in symbol names") class UnicodeSchemaTest(UnicodeSchemaTest): pass + @pytest.mark.reviewed class TableNoColumnsTest(TableNoColumnsTest): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks doesn't support server-side cursors.") class ServerSideCursorsTest(ServerSideCursorsTest): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks does not support sequences.") class SequenceTest(SequenceTest): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks does not support sequences.") class SequenceCompilerTest(SequenceCompilerTest): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks dialect does not implement sane rowcount.") class RowCountTest(RowCountTest): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks dialect does not implement sane rowcount.") class SimpleUpdateDeleteTest(SimpleUpdateDeleteTest): pass + @pytest.mark.reviewed class PostCompileParamsTest(PostCompileParamsTest): pass + @pytest.mark.reviewed -@pytest.mark.skip(reason="Databricks dialect doesn't implement UUID type. See test_suite.py") +@pytest.mark.skip( + reason="Databricks dialect doesn't implement UUID type. See test_suite.py" +) class NativeUUIDTest(NativeUUIDTest): """Type implementation will be straightforward. Since Databricks doesn't have a native UUID type we can use a STRING field, create a custom TypeDecorator for sqlalchemy.types.Uuid and add it to the dialect's colspecs. @@ -844,40 +864,45 @@ class NativeUUIDTest(NativeUUIDTest): Then mark requirements.uuid_data_type as open() so this test can run. """ + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks doesn't allow percent signs in identifiers") class PercentSchemaNamesTest(PercentSchemaNamesTest): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks does not support transactions") class IsolationLevelTest(IsolationLevelTest): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks does not support transactions") class AutocommitIsolationTest(AutocommitIsolationTest): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks dialect does not implement COLLATE support") class CollateTest(CollateTest): - """This is supported in Databricks. Not implemented here. - """ + """This is supported in Databricks. Not implemented here.""" + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks does not support computed / generated columns") class ComputedColumnTest(ComputedColumnTest): pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks does not support computed / generated columns") class ComputedReflectionTest(ComputedReflectionTest): pass + @pytest.mark.reviewed class NormalizedNameTest(NormalizedNameTest): - @pytest.mark.skip(reason="Poor test design? See test_suite.py") def test_get_table_names(self): """I'm not clear how this test can ever pass given that it's assertion looks like this: @@ -891,6 +916,7 @@ def test_get_table_names(self): """ pass + @pytest.mark.reviewed @pytest.mark.skip(reason="Databricks doesn't support INSERT ... RETURNING syntax") class ReturningTest(ReturningTest): @@ -898,8 +924,10 @@ class ReturningTest(ReturningTest): @pytest.mark.reviewed -@pytest.mark.skip(reason="Databricks dialect does not implement timezone support for Timestamp() types. See test_suite.py") +@pytest.mark.skip( + reason="Databricks dialect does not implement timezone support for Timestamp() types. See test_suite.py" +) class TimeTZTest(TimeTZTest): """Similar to DateTimeTZTest, this should be possible for the dialect since we can override type compilation and processing in _types.py. Implementation has been deferred. - """ \ No newline at end of file + """