From 4868a2fe33debf6ed9d0ea0de99f87fcde2d6d2e Mon Sep 17 00:00:00 2001 From: Ereli Eran Date: Fri, 27 Apr 2018 12:56:51 -0400 Subject: [PATCH 1/2] WIP, inital changes: adding Pipfile for the deps needs to run nose tests. adding the std sql as optional but default --- Pipfile | 19 +++++++++++++++ bigquery/query_builder.py | 14 +++++++---- bigquery/tests/test_query_builder.py | 36 ++++++++++++++-------------- 3 files changed, 47 insertions(+), 22 deletions(-) create mode 100644 Pipfile diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..4aad5b4 --- /dev/null +++ b/Pipfile @@ -0,0 +1,19 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +nose = "*" +rednose = "*" +mock = "==1.0.1" +coverage = "*" +nose-exclude = "*" +tox = "*" + +[dev-packages] +google-api-python-client = "*" +python-dateutil = "*" + +[requires] +python_version = "3.6" diff --git a/bigquery/query_builder.py b/bigquery/query_builder.py index 435bb73..d6eb0ad 100644 --- a/bigquery/query_builder.py +++ b/bigquery/query_builder.py @@ -5,7 +5,7 @@ def render_query(dataset, tables, select=None, conditions=None, - groupings=None, having=None, order_by=None, limit=None): + groupings=None, having=None, order_by=None, limit=None, use_legacy_sql=None): """Render a query that will run over the given tables using the specified parameters. @@ -36,7 +36,8 @@ def render_query(dataset, tables, select=None, conditions=None, {'field':'TimeStamp, 'direction':'desc'} or similar limit : int, optional Limit the amount of data needed to be returned. - + use_legacy_sql: bool, optional + Use legacy SQL syntax instead of Standard SQL. Returns ------- str @@ -131,7 +132,7 @@ def _format_select(formatter, name): return name -def _render_sources(dataset, tables): +def _render_sources(dataset, tables, use_legacy_sql=None ): """Render the source part of a query. Parameters @@ -160,9 +161,14 @@ def _render_sources(dataset, tables): 'Missing parameter %s in selecting sources' % (exp)) else: - return "FROM " + ", ".join( + if use_legacy_sql is True: + return "FROM " + ", ".join( ["[%s.%s]" % (dataset, table) for table in tables]) + else: + return "FROM " + ", ".join( + ["`%s.%s`" % (dataset, table) for table in tables]) + def _render_conditions(conditions): """Render the conditions part of a query. diff --git a/bigquery/tests/test_query_builder.py b/bigquery/tests/test_query_builder.py index 6e9e9ee..77f84a1 100644 --- a/bigquery/tests/test_query_builder.py +++ b/bigquery/tests/test_query_builder.py @@ -62,7 +62,7 @@ def test_multi_tables(self): result = _render_sources('spider', ['man', 'pig', 'bro']) self.assertEqual( - result, 'FROM [spider.man], [spider.pig], [spider.bro]') + result, 'FROM `spider.man`, `spider.pig`, `spider.bro`') def test_no_tables(self): """Ensure that render sources can handle no tables.""" @@ -78,7 +78,7 @@ def test_no_dataset(self): result = _render_sources('', ['man', 'pig', 'bro']) - self.assertEqual(result, 'FROM [.man], [.pig], [.bro]') + self.assertEqual(result, 'FROM `.man`, `.pig`, `.bro`') def test_tables_in_date_range(self): """Ensure that render sources can handle tables in DATE RANGE.""" @@ -417,7 +417,7 @@ def test_full_query(self): limit=10) expected_query = ("SELECT status as status, start_time as timestamp, " - "resource as url FROM [dataset.2013_06_appspot_1]" + "resource as url FROM `dataset.2013_06_appspot_1`" " WHERE (start_time <= INTEGER('1371566954')) AND " "(start_time >= INTEGER('1371556954')) GROUP BY " "timestamp, status HAVING (status == INTEGER('1')) " @@ -448,7 +448,7 @@ def test_empty_conditions(self): expected_query = ("SELECT status as status, start_time as timestamp, " "resource as url FROM " - "[dataset.2013_06_appspot_1] ORDER BY " + "`dataset.2013_06_appspot_1` ORDER BY " "timestamp desc ") expected_select = (expected_query[len('SELECT '):] .split('FROM')[0].strip().split(', ')) @@ -485,7 +485,7 @@ def test_incorrect_conditions(self): expected_query = ("SELECT status as status, start_time as timestamp, " "resource as url FROM " - "[dataset.2013_06_appspot_1] ORDER BY " + "`dataset.2013_06_appspot_1` ORDER BY " "timestamp desc ") expected_select = (expected_query[len('SELECT '):] .split('FROM')[0].strip().split(', ')) @@ -533,7 +533,7 @@ def test_multiple_condition_values(self): expected_query = ("SELECT status as status, start_time as timestamp, " "resource as url FROM " - "[dataset.2013_06_appspot_1] WHERE (start_time " + "`dataset.2013_06_appspot_1` WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) AND " "((resource CONTAINS STRING('foo') AND resource " @@ -571,7 +571,7 @@ def test_negated_condition_value(self): expected_query = ("SELECT status as status, start_time as timestamp, " "resource as url FROM " - "[dataset.2013_06_appspot_1] WHERE (NOT resource " + "`dataset.2013_06_appspot_1` WHERE (NOT resource " "CONTAINS STRING('foo')) ORDER BY timestamp desc ") expected_select = (expected_query[len('SELECT '):] .split('FROM')[0].strip().split(', ')) @@ -612,7 +612,7 @@ def test_multiple_negated_condition_values(self): expected_query = ("SELECT status as status, start_time as timestamp, " "resource as url FROM " - "[dataset.2013_06_appspot_1] WHERE (NOT resource " + "`dataset.2013_06_appspot_1` WHERE (NOT resource " "CONTAINS STRING('foo') AND NOT resource CONTAINS " "STRING('baz') AND NOT resource CONTAINS " "STRING('bar')) ORDER BY timestamp desc ") @@ -651,7 +651,7 @@ def test_empty_order(self): expected_query = ("SELECT status as status, start_time as timestamp, " "resource as url FROM " - "[dataset.2013_06_appspot_1] WHERE (start_time " + "`dataset.2013_06_appspot_1` WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) ") expected_select = (expected_query[len('SELECT '):] @@ -689,7 +689,7 @@ def test_incorrect_order(self): expected_query = ("SELECT status as status, start_time as timestamp, " "resource as url FROM " - "[dataset.2013_06_appspot_1] WHERE (start_time " + "`dataset.2013_06_appspot_1` WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) ") expected_select = (expected_query[len('SELECT '):] @@ -721,7 +721,7 @@ def test_empty_select(self): ], order_by={'fields': ['timestamp'], 'direction': 'desc'}) - expected_query = ("SELECT * FROM [dataset.2013_06_appspot_1] " + expected_query = ("SELECT * FROM `dataset.2013_06_appspot_1` " "WHERE (start_time <= INTEGER('1371566954')) AND " "(start_time >= INTEGER('1371556954')) ORDER BY " "timestamp desc ") @@ -752,7 +752,7 @@ def test_no_alias(self): order_by={'fields': ['start_time'], 'direction': 'desc'}) expected_query = ("SELECT status , start_time , resource FROM " - "[dataset.2013_06_appspot_1] WHERE (start_time " + "`dataset.2013_06_appspot_1` WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) ORDER BY start_time desc ") expected_select = (field.strip() for field in @@ -797,7 +797,7 @@ def test_formatting(self): expected_query = ("SELECT status as status, " "FORMAT_UTC_USEC(INTEGER(start_time)) as timestamp, " "resource as url FROM " - "[dataset.2013_06_appspot_1] WHERE (start_time " + "`dataset.2013_06_appspot_1` WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) ORDER BY timestamp desc ") expected_select = (expected_query[len('SELECT '):] @@ -849,7 +849,7 @@ def test_formatting_duplicate_columns(self): "FORMAT_UTC_USEC(INTEGER(start_time)) as timestamp, " "LEFT(FORMAT_UTC_USEC(INTEGER(start_time*1000000))," "10) as day, resource as url FROM " - "[dataset.2013_06_appspot_1] WHERE " + "`dataset.2013_06_appspot_1` WHERE " "(start_time <= INTEGER('1371566954')) AND " "(start_time >= INTEGER('1371556954')) ORDER BY " "timestamp desc ") @@ -894,7 +894,7 @@ def test_sec_to_micro_formatting(self): expected_query = ("SELECT status as status, " "SEC_TO_TIMESTAMP(INTEGER(start_time*1000000)) as " "timestamp, resource as url FROM " - "[dataset.2013_06_appspot_1] WHERE (start_time " + "`dataset.2013_06_appspot_1` WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) ORDER BY timestamp desc ") expected_select = (expected_query[len('SELECT '):] @@ -952,7 +952,7 @@ def test_empty_groupings(self): expected_query = ("SELECT status as status, start_time as timestamp, " "resource as url FROM " - "[dataset.2013_06_appspot_1] ORDER BY " + "`dataset.2013_06_appspot_1` ORDER BY " "timestamp desc ") expected_select = (expected_query[len('SELECT '):] .split('FROM')[0].strip().split(', ')) @@ -990,8 +990,8 @@ def test_multi_tables(self): expected_query = ("SELECT status as status, start_time as timestamp, " "resource as url FROM " - "[dataset.2013_06_appspot_1], " - "[dataset.2013_07_appspot_1] WHERE (start_time " + "`dataset.2013_06_appspot_1`, " + "`dataset.2013_07_appspot_1` WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) GROUP BY timestamp, status " "ORDER BY timestamp desc ") From f33a74caa251c1d8c3d30781f21dd0e8ae01e1fe Mon Sep 17 00:00:00 2001 From: Ereli Eran Date: Fri, 27 Apr 2018 13:02:26 -0400 Subject: [PATCH 2/2] moving the dev only package just to dev mode --- Pipfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Pipfile b/Pipfile index 4aad5b4..538c46b 100644 --- a/Pipfile +++ b/Pipfile @@ -4,6 +4,11 @@ verify_ssl = true name = "pypi" [packages] + + +[dev-packages] +google-api-python-client = "*" +python-dateutil = "*" nose = "*" rednose = "*" mock = "==1.0.1" @@ -11,9 +16,5 @@ coverage = "*" nose-exclude = "*" tox = "*" -[dev-packages] -google-api-python-client = "*" -python-dateutil = "*" - [requires] python_version = "3.6"