diff --git a/.travis.yml b/.travis.yml index eb0ba39..f18482d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,12 @@ language: python -python: - - "2.7" - install: - - pip install -r requirements.txt - - pip install -r requirements_dev.txt - -script: make test - + - pip install tox +script: tox -e $TOXENV notifications: email: false +env: + - TOXENV=py27 + - TOXENV=py33 + - TOXENV=py34 + - TOXENV=nightly + - TOXENV=pypy diff --git a/bigquery/__init__.py b/bigquery/__init__.py index 2ae326f..ef22544 100644 --- a/bigquery/__init__.py +++ b/bigquery/__init__.py @@ -1,5 +1,6 @@ -from client import get_client -from client import ( +from __future__ import absolute_import +from .client import get_client +from .client import ( BIGQUERY_SCOPE, BIGQUERY_SCOPE_READ_ONLY, JOB_CREATE_IF_NEEDED, @@ -14,4 +15,4 @@ JOB_ENCODING_ISO_8859_1 ) -from schema_builder import schema_from_record +from .schema_builder import schema_from_record diff --git a/bigquery/client.py b/bigquery/client.py index 331940f..0b04ff9 100644 --- a/bigquery/client.py +++ b/bigquery/client.py @@ -1,21 +1,19 @@ import calendar +import json +import logging from collections import defaultdict from datetime import datetime, timedelta -from time import sleep -from time import time from hashlib import sha256 -import json -import logging +from time import sleep, time +import httplib2 +import six from apiclient.discovery import build from apiclient.errors import HttpError -import httplib2 +from bigquery.errors import (BigQueryTimeoutException, JobExecutingException, + JobInsertException, UnfinishedQueryException) from bigquery.schema_builder import schema_from_record -from bigquery.errors import ( - JobExecutingException, JobInsertException, - UnfinishedQueryException, BigQueryTimeoutException -) BIGQUERY_SCOPE = 'https://www.googleapis.com/auth/bigquery' BIGQUERY_SCOPE_READ_ONLY = 'https://www.googleapis.com/auth/bigquery.readonly' @@ -154,7 +152,7 @@ def _submit_query_job(self, query_data): projectId=self.project_id, body=query_data).execute() except HttpError as e: if query_data.get("dryRun", False): - return None, json.loads(e.content) + return None, json.loads(e.content.decode('utf8')) raise job_id = query_reply['jobReference'].get('jobId') @@ -266,7 +264,7 @@ def get_table_schema(self, dataset, table): projectId=self.project_id, tableId=table, datasetId=dataset).execute() - except HttpError, e: + except HttpError as e: if int(e.resp['status']) == 404: logging.warn('Table %s.%s does not exist', dataset, table) return None @@ -651,7 +649,7 @@ def import_data_from_uris( skip_leading_rows=skip_leading_rows, quote=quote) non_null_values = dict((k, v) for k, v - in all_values.items() + in list(all_values.items()) if v) raise Exception("Parameters field_delimiter, allow_jagged_rows, " "allow_quoted_newlines, quote and " @@ -837,6 +835,7 @@ def wait_for_job(self, job, interval=5, timeout=60): Waits until the job indicated by job_resource is done or has failed Args: job: dict, representing a BigQuery job resource + or str, representing a BigQuery job id interval: optional float polling interval in seconds, default = 5 timeout: optional float timeout in seconds, default = 60 Returns: @@ -848,7 +847,9 @@ def wait_for_job(self, job, interval=5, timeout=60): BigQueryTimeoutException on timeout """ complete = False - job_id = job['jobReference']['jobId'] + job_id = str(job if isinstance(job, + (six.binary_type, six.text_type, int)) + else job['jobReference']['jobId']) job_resource = None start_time = time() @@ -1048,7 +1049,7 @@ def _filter_tables_by_time(self, tables, start_time, end_time): A list of table names that are inside the time range. """ - return [table_name for (table_name, unix_seconds) in tables.iteritems() + return [table_name for (table_name, unix_seconds) in tables.items() if self._in_range(start_time, end_time, unix_seconds)] def _in_range(self, start_time, end_time, time): @@ -1167,7 +1168,7 @@ def _generate_hex_for_uris(self, uris): Returns: string of hexed uris """ - return sha256(":".join(uris) + str(time())).hexdigest() + return sha256((":".join(uris) + str(time())).encode()).hexdigest() def _raise_insert_exception_if_error(self, job): error_http = job.get('error') diff --git a/bigquery/query_builder.py b/bigquery/query_builder.py index 942f78e..1cfa72a 100644 --- a/bigquery/query_builder.py +++ b/bigquery/query_builder.py @@ -77,7 +77,7 @@ def _render_select(selections): return 'SELECT *' rendered_selections = [] - for name, options in selections.iteritems(): + for name, options in selections.items(): if not isinstance(options, list): options = [options] @@ -200,7 +200,8 @@ def _render_condition(field, field_type, comparators): if condition == "IN": if isinstance(value, (list, tuple, set)): value = ', '.join( - [_render_condition_value(v, field_type) for v in value] + sorted([_render_condition_value(v, field_type) + for v in value]) ) else: value = _render_condition_value(value, field_type) diff --git a/bigquery/schema_builder.py b/bigquery/schema_builder.py index 195fc3d..09084a7 100644 --- a/bigquery/schema_builder.py +++ b/bigquery/schema_builder.py @@ -1,10 +1,12 @@ +from __future__ import absolute_import __author__ = 'Aneil Mallavarapu (http://github.com/aneilbaboo)' from datetime import datetime +import six import dateutil.parser -from errors import InvalidTypeException +from .errors import InvalidTypeException def default_timestamp_parser(s): @@ -30,7 +32,7 @@ def schema_from_record(record, timestamp_parser=default_timestamp_parser): schema: list """ return [describe_field(k, v, timestamp_parser=timestamp_parser) - for k, v in record.items()] + for k, v in list(record.items())] def describe_field(k, v, timestamp_parser=default_timestamp_parser): @@ -76,7 +78,7 @@ def bq_schema_field(name, bq_type, mode): if bq_type == "record": try: field['fields'] = schema_from_record(v, timestamp_parser) - except InvalidTypeException, e: + except InvalidTypeException as e: # recursively construct the key causing the error raise InvalidTypeException("%s.%s" % (k, e.key), e.value) @@ -100,7 +102,7 @@ def bigquery_type(o, timestamp_parser=default_timestamp_parser): t = type(o) if t == int: return "integer" - elif t == str or t == unicode: + elif (t == six.binary_type and six.PY2) or t == six.text_type: if timestamp_parser and timestamp_parser(o): return "timestamp" else: diff --git a/bigquery/tests/test_client.py b/bigquery/tests/test_client.py index b09cba4..12ed294 100644 --- a/bigquery/tests/test_client.py +++ b/bigquery/tests/test_client.py @@ -1,6 +1,7 @@ import unittest import mock +import six from nose.tools import raises from apiclient.errors import HttpError @@ -101,7 +102,7 @@ def test_initialize_read_write(self, mock_build, mock_return_cred): @mock.patch('bigquery.client._credentials') @mock.patch('bigquery.client.build') - @mock.patch('__builtin__.open') + @mock.patch('__builtin__.open' if six.PY2 else 'builtins.open') def test_initialize_key_file(self, mock_open, mock_build, mock_return_cred): """Ensure that a BigQueryClient is initialized and returned with @@ -295,7 +296,7 @@ def test_query_dry_run_invalid(self): mock_query_job = mock.Mock() mock_query_job.execute.side_effect = HttpError( - 'crap', '{"message": "Bad query"}') + 'crap', '{"message": "Bad query"}'.encode('utf8')) self.mock_job_collection.query.return_value = mock_query_job @@ -370,7 +371,8 @@ def test_get_response(self): page_token = "token" timeout = 1 - actual = self.client.get_query_results(job_id, offset, limit, page_token, timeout) + actual = self.client.get_query_results(job_id, offset, limit, + page_token, timeout) self.mock_job_collection.getQueryResults.assert_called_once_with( projectId=self.project_id, jobId=job_id, startIndex=offset, @@ -586,6 +588,44 @@ def test_wait_job_error_result(self): interval=.01, timeout=.01) + def test_accepts_job_id(self): + """Ensure it accepts a job Id rather than a full job resource""" + + return_values = [{'status': {'state': u'RUNNING'}, + 'jobReference': {'jobId': "testJob"}}, + {'status': {'state': u'DONE'}, + 'jobReference': {'jobId': "testJob"}}] + + def side_effect(*args, **kwargs): + return return_values.pop(0) + + self.api_mock.jobs().get().execute.side_effect = side_effect + + job_resource = self.client.wait_for_job("testJob", + interval=.01, + timeout=5) + + self.assertEqual(self.api_mock.jobs().get().execute.call_count, 2) + self.assertIsInstance(job_resource, dict) + + def test_accepts_integer_job_id(self): + return_values = [{'status': {'state': u'RUNNING'}, + 'jobReference': {'jobId': "testJob"}}, + {'status': {'state': u'DONE'}, + 'jobReference': {'jobId': "testJob"}}] + + def side_effect(*args, **kwargs): + return return_values.pop(0) + + self.api_mock.jobs().get().execute.side_effect = side_effect + + job_resource = self.client.wait_for_job(1234567, + interval=.01, + timeout=600) + + self.assertEqual(self.api_mock.jobs().get().execute.call_count, 2) + self.assertIsInstance(job_resource, dict) + class TestImportDataFromURIs(unittest.TestCase): @@ -857,8 +897,8 @@ def test_export(self, mock_generate_hex): body = { "jobReference": { "projectId": self.project_id, - "jobId": "%s-%s-destinationuri" % - (self.dataset_id, self.table_id) + "jobId": "%s-%s-destinationuri" % (self.dataset_id, + self.table_id) }, "configuration": { "extract": { @@ -1042,8 +1082,9 @@ def test_multi_inside_range(self): }, 1370002000, 1370000000) self.assertEqual( - ['Daenerys Targaryen', 'William Shatner', 'Gordon Freeman'], - tables + sorted( + ['Daenerys Targaryen', 'William Shatner', 'Gordon Freeman']), + sorted(tables) ) def test_not_inside_range(self): @@ -1242,7 +1283,7 @@ def test_table_exists(self): def test_table_does_not_exist(self): """Ensure that None is returned if the table doesn't exist.""" self.mock_tables.get.return_value.execute.side_effect = \ - HttpError({'status': "404"}, '{}') + HttpError({'status': "404"}, '{}'.encode('utf8')) self.assertIsNone( self.client.get_table_schema(self.dataset, self.table)) @@ -1394,7 +1435,7 @@ def test_table_does_not_exist(self): """Ensure that if the table does not exist, False is returned.""" self.mock_tables.get.return_value.execute.side_effect = ( - HttpError(HttpResponse(404), 'There was an error')) + HttpError(HttpResponse(404), 'There was an error'.encode('utf8'))) actual = self.client.check_table(self.dataset, self.table) @@ -1447,7 +1488,7 @@ def test_table_create_failed(self): or if swallow_results is False an empty dict is returned.""" self.mock_tables.insert.return_value.execute.side_effect = ( - HttpError(HttpResponse(404), 'There was an error')) + HttpError(HttpResponse(404), 'There was an error'.encode('utf8'))) actual = self.client.create_table(self.dataset, self.table, self.schema) @@ -1518,7 +1559,7 @@ def test_view_create_failed(self): or if swallow_results is False an empty dict is returned.""" self.mock_tables.insert.return_value.execute.side_effect = ( - HttpError(HttpResponse(404), 'There was an error')) + HttpError(HttpResponse(404), 'There was an error'.encode('utf8'))) actual = self.client.create_view(self.dataset, self.table, self.query) @@ -1582,7 +1623,7 @@ def test_delete_table_fail(self): or the actual response is swallow_results is False.""" self.mock_tables.delete.return_value.execute.side_effect = ( - HttpError(HttpResponse(404), 'There was an error')) + HttpError(HttpResponse(404), 'There was an error'.encode('utf8'))) actual = self.client.delete_table(self.dataset, self.table) @@ -1784,7 +1825,7 @@ def test_push_failed_swallow_results_false(self): def test_push_exception(self): """Ensure that if insertAll raises an exception, False is returned.""" - e = HttpError(HttpResponse(404), 'There was an error') + e = HttpError(HttpResponse(404), 'There was an error'.encode('utf8')) self.mock_table_data.insertAll.return_value.execute.side_effect = e actual = self.client.push_rows(self.dataset, self.table, self.rows, @@ -1973,7 +2014,7 @@ def test_get_tables(self): bq = client.BigQueryClient(mock_bq_service, 'project') tables = bq.get_tables('dataset', 'appspot-1', 0, 10000000000) - self.assertItemsEqual(tables, ['2013_06_appspot_1']) + six.assertCountEqual(self, tables, ['2013_06_appspot_1']) def test_get_tables_from_datetimes(self): """Ensure tables falling in the time window, specified with datetimes, @@ -1996,7 +2037,7 @@ def test_get_tables_from_datetimes(self): end = datetime(2013, 7, 10) tables = bq.get_tables('dataset', 'appspot-1', start, end) - self.assertItemsEqual(tables, ['2013_06_appspot_1']) + six.assertCountEqual(self, tables, ['2013_06_appspot_1']) # @@ -2027,7 +2068,7 @@ def test_dataset_create_failed(self): """Ensure that if creating the table fails, False is returned.""" self.mock_datasets.insert.return_value.execute.side_effect = \ - HttpError(HttpResponse(404), 'There was an error') + HttpError(HttpResponse(404), 'There was an error'.encode('utf8')) actual = self.client.create_dataset(self.dataset, friendly_name=self.friendly_name, @@ -2096,7 +2137,7 @@ def test_delete_datasets_fail(self): """Ensure that if deleting table fails, False is returned.""" self.mock_datasets.delete.return_value.execute.side_effect = \ - HttpError(HttpResponse(404), 'There was an error') + HttpError(HttpResponse(404), 'There was an error'.encode('utf8')) actual = self.client.delete_dataset(self.dataset) @@ -2254,7 +2295,8 @@ def test_get_datasets(self): bq = client.BigQueryClient(mock_bq_service, 'project') datasets = bq.get_datasets() - self.assertItemsEqual(datasets, FULL_DATASET_LIST_RESPONSE['datasets']) + six.assertCountEqual(self, datasets, + FULL_DATASET_LIST_RESPONSE['datasets']) def test_get_datasets_returns_no_list(self): """Ensure we handle the no datasets case""" @@ -2273,7 +2315,7 @@ def test_get_datasets_returns_no_list(self): bq = client.BigQueryClient(mock_bq_service, 'project') datasets = bq.get_datasets() - self.assertItemsEqual(datasets, []) + six.assertCountEqual(self, datasets, []) class TestUpdateDataset(unittest.TestCase): @@ -2301,7 +2343,7 @@ def test_dataset_update_failed(self): """Ensure that if creating the table fails, False is returned.""" self.mock_datasets.update.return_value.execute.side_effect = \ - HttpError(HttpResponse(404), 'There was an error') + HttpError(HttpResponse(404), 'There was an error'.encode('utf8')) actual = self.client.update_dataset(self.dataset, friendly_name=self.friendly_name, diff --git a/bigquery/tests/test_query_builder.py b/bigquery/tests/test_query_builder.py index b2e2de1..8591c6b 100644 --- a/bigquery/tests/test_query_builder.py +++ b/bigquery/tests/test_query_builder.py @@ -1,5 +1,8 @@ +import six import unittest +unittest.TestCase.maxDiff = None + class TestRenderSelect(unittest.TestCase): @@ -18,11 +21,13 @@ def test_multiple_selects(self): 'ip': {'alias': 'IP'}, 'app_logs': {'alias': 'AppLogs'}}) - expected = 'SELECT status as Status, latency as Latency, ' \ - 'max_log_level as MaxLogLevel, resource as URL, user as ' \ - 'User, ip as IP, start_time as TimeStamp, version_id as ' \ - 'Version, app_logs as AppLogs' - self.assertEqual(expected, result) + expected = ('SELECT status as Status, latency as Latency, ' + 'max_log_level as MaxLogLevel, resource as URL, user as ' + 'User, ip as IP, start_time as TimeStamp, version_id as ' + 'Version, app_logs as AppLogs') + six.assertCountEqual( + self, sorted(expected[len('SELECT '):].split(', ')), + sorted(result[len('SELECT '):].split(', '))) def test_casting(self): """Ensure that render select can handle custom casting.""" @@ -202,14 +207,16 @@ def test_in_comparator(self): } ]) - self.assertEqual(result, "WHERE ((foobar IN (STRING('a'), STRING('b'))" - " AND foobar IN (STRING('c'), STRING('d')) " - "AND foobar IN (STRING('e'), STRING('f')) AND" - " foobar IN (STRING('g'))) AND (NOT foobar IN" - " (STRING('h'), STRING('i')) AND NOT foobar " - "IN (STRING('k'), STRING('j')) AND NOT foobar" - " IN (STRING('l'), STRING('m')) AND NOT " - "foobar IN (STRING('n'))))") + six.assertCountEqual(self, result[len('WHERE '):].split(' AND '), + "WHERE ((foobar IN (STRING('a'), STRING('b'))" + " AND foobar IN (STRING('c'), STRING('d')) " + "AND foobar IN (STRING('e'), STRING('f')) AND" + " foobar IN (STRING('g'))) AND (NOT foobar IN" + " (STRING('h'), STRING('i')) AND NOT foobar " + "IN (STRING('j'), STRING('k')) AND NOT foobar" + " IN (STRING('l'), STRING('m')) AND NOT " + "foobar IN (STRING('n'))))" [len('WHERE '):] + .split(' AND ')) class TestRenderOrder(unittest.TestCase): @@ -298,7 +305,14 @@ def test_full_query(self): " WHERE (start_time <= INTEGER('1371566954')) AND " "(start_time >= INTEGER('1371556954')) GROUP BY " "timestamp, status ORDER BY timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_empty_conditions(self): """Ensure that render query can handle an empty list of conditions.""" @@ -319,7 +333,14 @@ def test_empty_conditions(self): "resource as url FROM " "[dataset.2013_06_appspot_1] ORDER BY " "timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_incorrect_conditions(self): """Ensure that render query can handle incorrectly formatted @@ -348,7 +369,14 @@ def test_incorrect_conditions(self): "resource as url FROM " "[dataset.2013_06_appspot_1] ORDER BY " "timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_multiple_condition_values(self): """Ensure that render query can handle conditions with multiple values. @@ -393,7 +421,14 @@ def test_multiple_condition_values(self): "((resource CONTAINS STRING('foo') AND resource " "CONTAINS STRING('baz')) AND (NOT resource CONTAINS " "STRING('bar'))) ORDER BY timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_negated_condition_value(self): """Ensure that render query can handle conditions with negated values. @@ -420,7 +455,14 @@ def test_negated_condition_value(self): "resource as url FROM " "[dataset.2013_06_appspot_1] WHERE (NOT resource " "CONTAINS STRING('foo')) ORDER BY timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_multiple_negated_condition_values(self): """Ensure that render query can handle conditions with multiple negated @@ -456,7 +498,14 @@ def test_multiple_negated_condition_values(self): "CONTAINS STRING('foo') AND NOT resource CONTAINS " "STRING('baz') AND NOT resource CONTAINS " "STRING('bar')) ORDER BY timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_empty_order(self): """Ensure that render query can handle an empty formatted order.""" @@ -487,7 +536,14 @@ def test_empty_order(self): "[dataset.2013_06_appspot_1] WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) ") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_incorrect_order(self): """Ensure that render query can handle inccorectly formatted order.""" @@ -518,7 +574,14 @@ def test_incorrect_order(self): "[dataset.2013_06_appspot_1] WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) ") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_empty_select(self): """Ensure that render query corrently handles no selection.""" @@ -574,7 +637,17 @@ def test_no_alias(self): "[dataset.2013_06_appspot_1] WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) ORDER BY start_time desc") - self.assertEqual(result, expected_query) + expected_select = (field.strip() for field in + expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = (expected_query[len('SELECT '):].split('FROM')[1] + .strip()) + result_select = (field.strip() for field in + result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1].strip() + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_formatting(self): """Ensure that render query runs with formatting a select.""" @@ -609,7 +682,14 @@ def test_formatting(self): "[dataset.2013_06_appspot_1] WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) ORDER BY timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_formatting_duplicate_columns(self): """Ensure that render query runs with formatting a select for a @@ -655,7 +735,14 @@ def test_formatting_duplicate_columns(self): "(start_time <= INTEGER('1371566954')) AND " "(start_time >= INTEGER('1371556954')) ORDER BY " "timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_sec_to_micro_formatting(self): """Ensure that render query runs sec_to_micro formatting on a @@ -692,7 +779,14 @@ def test_sec_to_micro_formatting(self): "[dataset.2013_06_appspot_1] WHERE (start_time " "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) ORDER BY timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) def test_no_table_or_dataset(self): """Ensure that render query returns None if there is no dataset or @@ -741,7 +835,15 @@ def test_empty_groupings(self): "resource as url FROM " "[dataset.2013_06_appspot_1] ORDER BY " "timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) + def test_multi_tables(self): """Ensure that render query arguments work with multiple tables.""" @@ -775,4 +877,11 @@ def test_multi_tables(self): "<= INTEGER('1371566954')) AND (start_time >= " "INTEGER('1371556954')) GROUP BY timestamp, status " "ORDER BY timestamp desc") - self.assertEqual(result, expected_query) + expected_select = (expected_query[len('SELECT '):] + .split('FROM')[0].strip().split(', ')) + expected_from = expected_query[len('SELECT '):].split('FROM')[1] + result_select = (result[len('SELECT '):].split('FROM')[0] + .strip().split(', ')) + result_from = result[len('SELECT '):].split('FROM')[1] + six.assertCountEqual(self, expected_select, result_select) + six.assertCountEqual(self, expected_from, result_from) diff --git a/bigquery/tests/test_schema_builder.py b/bigquery/tests/test_schema_builder.py index eef1298..060162b 100644 --- a/bigquery/tests/test_schema_builder.py +++ b/bigquery/tests/test_schema_builder.py @@ -1,7 +1,8 @@ +from six.moves.builtins import object from datetime import datetime import unittest - +import six from bigquery.schema_builder import schema_from_record from bigquery.schema_builder import describe_field from bigquery.schema_builder import bigquery_type @@ -11,48 +12,49 @@ class TestBigQueryTypes(unittest.TestCase): def test_str_is_string(self): - self.assertItemsEqual(bigquery_type("Bob"), 'string') + six.assertCountEqual(self, bigquery_type("Bob"), 'string') def test_unicode_is_string(self): - self.assertItemsEqual(bigquery_type(u"Here is a happy face \u263A"), - 'string') + six.assertCountEqual(self, bigquery_type(u"Here is a happy face \u263A"), + 'string') def test_int_is_integer(self): - self.assertItemsEqual(bigquery_type(123), 'integer') + six.assertCountEqual(self, bigquery_type(123), 'integer') def test_datetime_is_timestamp(self): - self.assertItemsEqual(bigquery_type(datetime.now()), 'timestamp') + six.assertCountEqual(self, bigquery_type(datetime.now()), 'timestamp') def test_isoformat_timestring(self): - self.assertItemsEqual(bigquery_type(datetime.now().isoformat()), - 'timestamp') + six.assertCountEqual(self, bigquery_type(datetime.now().isoformat()), + 'timestamp') def test_timestring_feb_20_1973(self): - self.assertItemsEqual(bigquery_type("February 20th 1973"), 'timestamp') + six.assertCountEqual(self, bigquery_type("February 20th 1973"), + 'timestamp') def test_timestring_thu_1_july_2004_22_30_00(self): - self.assertItemsEqual(bigquery_type("Thu, 1 July 2004 22:30:00"), - 'timestamp') + six.assertCountEqual(self, bigquery_type("Thu, 1 July 2004 22:30:00"), + 'timestamp') def test_today_is_not_timestring(self): - self.assertItemsEqual(bigquery_type("today"), 'string') + six.assertCountEqual(self, bigquery_type("today"), 'string') def test_timestring_next_thursday(self): - self.assertItemsEqual(bigquery_type("February 20th 1973"), 'timestamp') + six.assertCountEqual(self, bigquery_type("February 20th 1973"), 'timestamp') def test_timestring_arbitrary_fn_success(self): - self.assertItemsEqual( - bigquery_type("whatever", timestamp_parser=lambda x: True), + six.assertCountEqual( + self, bigquery_type("whatever", timestamp_parser=lambda x: True), 'timestamp') def test_timestring_arbitrary_fn_fail(self): - self.assertItemsEqual( - bigquery_type("February 20th 1973", - timestamp_parser=lambda x: False), + six.assertCountEqual( + self, bigquery_type("February 20th 1973", + timestamp_parser=lambda x: False), 'string') def test_class_instance_is_invalid_type(self): - class SomeClass: + class SomeClass(object): pass self.assertIsNone(bigquery_type(SomeClass())) @@ -61,15 +63,15 @@ def test_list_is_invalid_type(self): self.assertIsNone(bigquery_type([1, 2, 3])) def test_dict_is_record(self): - self.assertItemsEqual(bigquery_type({"a": 1}), 'record') + six.assertCountEqual(self, bigquery_type({"a": 1}), 'record') class TestFieldDescription(unittest.TestCase): def test_simple_string_field(self): - self.assertItemsEqual(describe_field("user", "Bob"), - {"name": "user", "type": "string", "mode": - "nullable"}) + six.assertCountEqual(self, describe_field("user", "Bob"), + {"name": "user", "type": "string", "mode": + "nullable"}) class TestSchemaGenerator(unittest.TestCase): @@ -79,7 +81,7 @@ def test_simple_record(self): schema = [{"name": "username", "type": "string", "mode": "nullable"}, {"name": "id", "type": "integer", "mode": "nullable"}] - self.assertItemsEqual(schema_from_record(record), schema) + six.assertCountEqual(self, schema_from_record(record), schema) def test_hierarchical_record(self): record = {"user": {"username": "Bob", "id": 123}} @@ -87,8 +89,11 @@ def test_hierarchical_record(self): "fields": [{"name": "username", "type": "string", "mode": "nullable"}, {"name": "id", "type": "integer", "mode": "nullable"}]}] - - self.assertItemsEqual(schema_from_record(record), schema) + generated_schema = schema_from_record(record) + schema_fields = schema[0].pop('fields') + generated_fields = generated_schema[0].pop('fields') + six.assertCountEqual(self, schema_fields, generated_fields) + six.assertCountEqual(self, generated_schema, schema) def test_hierarchical_record_with_timestamps(self): record = {"global": "2001-01-01", "user": {"local": "2001-01-01"}} @@ -109,19 +114,17 @@ def test_hierarchical_record_with_timestamps(self): "type": "string", "mode": "nullable"}]}] - self.assertItemsEqual( - schema_from_record(record), - schema_with_ts) + six.assertCountEqual(self, schema_from_record(record), schema_with_ts) - self.assertItemsEqual( - schema_from_record(record, timestamp_parser=lambda x: False), + six.assertCountEqual( + self, schema_from_record(record, timestamp_parser=lambda x: False), schema_without_ts) def test_repeated_field(self): record = {"ids": [1, 2, 3, 4, 5]} schema = [{"name": "ids", "type": "integer", "mode": "repeated"}] - self.assertItemsEqual(schema_from_record(record), schema) + six.assertCountEqual(self, schema_from_record(record), schema) def test_nested_invalid_type_reported_correctly(self): key = "wrong answer" @@ -129,7 +132,7 @@ def test_nested_invalid_type_reported_correctly(self): try: schema_from_record({"a": {"b": [{"c": None}]}}) - except InvalidTypeException, e: + except InvalidTypeException as e: key = e.key value = e.value diff --git a/requirements_dev.txt b/requirements_dev.txt index a36ba42..a1292b0 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -3,3 +3,5 @@ rednose mock coverage nose-exclude +tox +-r requirements.txt diff --git a/setup.py b/setup.py index 6a91a69..2ab7020 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -VERSION = '1.2.0' +VERSION = '1.3.0' setup_args = dict( name='BigQuery-Python', @@ -20,10 +20,10 @@ 'Environment :: Web Environment', 'Intended Audience :: Developers', 'Operating System :: OS Independent', - 'Programming Language :: Python', + 'Programming Language :: Python2', + 'Programming Language :: Python3', ], ) if __name__ == '__main__': setup(**setup_args) - diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..3a3c16f --- /dev/null +++ b/tox.ini @@ -0,0 +1,12 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py27, py33, py34, nightly, pypy + +[testenv] +commands = nosetests +deps = -rrequirements_dev.txt +skip_missing_interpreters = True \ No newline at end of file