Skip to content
Permalink
Browse files
fix: Select expressions no-longer force use of labels (#129)
* Fixed a dependency problem that caysed test failures in Python 3.6.

The source of the dependency bug is in old versions of google-cloud-core that
depend on too-old versions of google-api-core.

* Provide a bigquery mock based on sqlite

So we don't have t mock at the api level.

* Don't force labels in select.

#78
  • Loading branch information
jimfulton committed Apr 20, 2021
1 parent 4d169b9 commit 669b301359f9f37c5c7286a245080b8af2567186
Showing with 128 additions and 11 deletions.
  1. +0 −9 pybigquery/sqlalchemy_bigquery.py
  2. +2 −1 setup.py
  3. +1 −1 testing/constraints-3.6.txt
  4. +16 −0 tests/unit/conftest.py
  5. +98 −0 tests/unit/fauxdbi.py
  6. +11 −0 tests/unit/test_select.py
@@ -165,15 +165,6 @@ def __init__(self, dialect, statement, column_keys=None, inline=False, **kwargs)
dialect, statement, column_keys, inline, **kwargs
)

def visit_select(self, *args, **kwargs):
"""
Use labels for every column.
This ensures that fields won't contain duplicate names
"""

args[0].use_labels = True
return super(BigQueryCompiler, self).visit_select(*args, **kwargs)

def visit_column(
self, column, add_to_result_map=None, include_table=True, **kwargs
):
@@ -81,8 +81,9 @@ def readme():
platforms="Posix; MacOS X; Windows",
install_requires=[
"sqlalchemy>=1.1.9,<1.4.0dev",
"google-auth>=1.2.0,<2.0dev",
"google-auth>=1.14.0,<2.0dev", # Work around pip wack.
"google-cloud-bigquery>=1.12.0",
"google-api-core>=1.19.1", # Work-around bug in cloud core deps.
"future",
],
python_requires=">=3.6, <3.10",
@@ -5,5 +5,5 @@
#
# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev",
sqlalchemy==1.1.9
google-auth==1.2.0
google-auth==1.14.0
google-cloud-bigquery==1.12.0
@@ -0,0 +1,16 @@
import mock
import pytest
import sqlalchemy

import fauxdbi


@pytest.fixture()
def faux_conn():
with mock.patch(
"google.cloud.bigquery.dbapi.connection.Connection", fauxdbi.Connection
):
engine = sqlalchemy.create_engine("bigquery://myproject/mydataset")
conn = engine.connect()
yield conn
conn.close()
@@ -0,0 +1,98 @@
import google.api_core.exceptions
import google.cloud.bigquery.schema
import google.cloud.bigquery.table
import contextlib
import sqlite3


class Connection:

connection = None

def __init__(self, client=None, bqstorage_client=None):
# share a single connection:
if self.connection is None:
self.__class__.connection = sqlite3.connect(":memory:")
self._client = FauxClient(client, self.connection)

def cursor(self):
return Cursor(self.connection)

def commit(self):
pass

def rollback(self):
pass

def close(self):
self.connection.close()


class Cursor:

arraysize = 1

def __init__(self, connection):
self.connection = connection
self.cursor = connection.cursor()

def execute(self, operation, parameters=None):
if parameters:
parameters = {
name: "null" if value is None else repr(value)
for name, value in parameters.items()
}
operation %= parameters
self.cursor.execute(operation, parameters)
self.description = self.cursor.description
self.rowcount = self.cursor.rowcount

def executemany(self, operation, parameters_list):
for parameters in parameters_list:
self.execute(operation, parameters)

def close(self):
self.cursor.close()

def fetchone(self):
return self.cursor.fetchone()

def fetchmany(self, size=None):
self.cursor.fetchmany(size or self.arraysize)

def fetchall(self):
return self.cursor.fetchall()

def setinputsizes(self, sizes):
pass

def setoutputsize(self, size, column=None):
pass


class FauxClient:
def __init__(self, client, connection):
self._client = client
self.project = client.project
self.connection = connection

def get_table(self, table_ref):
table_name = table_ref.table_id
with contextlib.closing(self.connection.cursor()) as cursor:
cursor.execute(
f"select name from sqlite_master"
f" where type='table' and name='{table_name}'"
)
if list(cursor):
cursor.execute("PRAGMA table_info('{table_name}')")
schema = [
google.cloud.bigquery.schema.SchemaField(
name=name,
field_type=type_,
mode="REQUIRED" if notnull else "NULLABLE",
)
for cid, name, type_, notnull, dflt_value, pk in cursor
]
return google.cloud.bigquery.table.Table(table_ref, schema)
else:
raise google.api_core.exceptions.NotFound(table_ref)
@@ -0,0 +1,11 @@
import sqlalchemy


def test_labels_not_forced(faux_conn):
metadata = sqlalchemy.MetaData()
table = sqlalchemy.Table(
"some_table", metadata, sqlalchemy.Column("id", sqlalchemy.Integer)
)
metadata.create_all(faux_conn.engine)
result = faux_conn.execute(sqlalchemy.select([table.c.id]))
assert result.keys() == ["id"] # Look! Just the column name!

0 comments on commit 669b301

Please sign in to comment.