diff --git a/README.md b/README.md index a533ea9..762917d 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,19 @@ rows = [ inserted = client.push_rows('dataset', 'table', rows, 'id') ``` +# Import data from Google cloud storage +```python +schema = [ {"name": "username", "type": "string", "mode": "nullable"} ] +job = client.import_data_from_uris( ['gs://mybucket/mydata.json'], + 'dataset', + 'table', + schema, + source_format=JOB_SOURCE_FORMAT_JSON) + +job = client.wait_for_job(job, timeout=60) +print(job) +``` + # Managing Datasets The client provides an API for listing, creating, deleting, updating and patching datasets. diff --git a/bigquery/__init__.py b/bigquery/__init__.py index b8ad97e..8102877 100644 --- a/bigquery/__init__.py +++ b/bigquery/__init__.py @@ -3,4 +3,3 @@ logger = logging.getLogger('bigquery') logger.setLevel(logging.DEBUG) - diff --git a/bigquery/client.py b/bigquery/client.py index d94b6b8..7a1d8f3 100644 --- a/bigquery/client.py +++ b/bigquery/client.py @@ -1,6 +1,8 @@ import calendar from collections import defaultdict from datetime import datetime +from time import sleep, time +from hashlib import sha256 import json from apiclient.discovery import build @@ -11,9 +13,21 @@ from bigquery.errors import UnfinishedQueryException from bigquery.schema_builder import schema_from_record + BIGQUERY_SCOPE = 'https://www.googleapis.com/auth/bigquery' BIGQUERY_SCOPE_READ_ONLY = 'https://www.googleapis.com/auth/bigquery.readonly' +JOB_CREATE_IF_NEEDED = 'CREATE_IF_NEEDED' +JOB_CREATE_NEVER = 'CREATE_NEVER' +JOB_SOURCE_FORMAT_NEWLINE_DELIMITED_JSON = 'NEWLINE_DELIMITED_JSON' +JOB_SOURCE_FORMAT_DATASTORE_BACKUP = 'DATASTORE_BACKUP' +JOB_SOURCE_FORMAT_CSV = 'CSV' +JOB_WRITE_TRUNCATE = 'WRITE_TRUNCATE' +JOB_WRITE_APPEND = 'WRITE_APPEND' +JOB_WRITE_EMPTY = 'WRITE_EMPTY' +JOB_ENCODING_UTF_8 = 'UTF-8' +JOB_ENCODING_ISO_8859_1 = 'ISO-8859-1' + def get_client(project_id, credentials=None, service_account=None, private_key=None, readonly=True): @@ -67,11 +81,11 @@ def _get_bq_service(credentials=None, service_account=None, private_key=None, def _credentials(): """Import and return SignedJwtAssertionCredentials class""" from oauth2client.client import SignedJwtAssertionCredentials + return SignedJwtAssertionCredentials class BigQueryClient(object): - def __init__(self, bq_service, project_id): self.bigquery = bq_service self.project_id = project_id @@ -309,6 +323,191 @@ def get_tables(self, dataset_id, app_id, start_time, end_time): return self._filter_tables_by_time(app_tables, start_time, end_time) + def import_data_from_uris( + self, + source_uris, + dataset, + table, + schema=None, + job=None, + source_format=None, + create_disposition=None, + write_disposition=None, + encoding=None, + ignore_unknown_values=None, + max_bad_records=None, + allow_jagged_rows=None, + allow_quoted_newlines=None, + field_delimiter=None, + quote=None, + skip_leading_rows=None, + ): + """ + Imports data into a BigQuery table from cloud storage. + Args: + source_uris: required string or list of strings representing + the uris on cloud storage of the form: + gs://bucket/filename + dataset: required string id of the dataset + table: required string id of the table + job: optional string identifying the job (a unique jobid + is automatically generated if not provided) + schema: optional list representing the bigquery schema + source_format: optional string + (one of the JOB_SOURCE_FORMAT_* constants) + create_disposition: optional string + (one of the JOB_CREATE_* constants) + write_disposition: optional string + (one of the JOB_WRITE_* constants) + encoding: optional string default + (one of the JOB_ENCODING_* constants) + ignore_unknown_values: optional boolean + max_bad_records: optional boolean + allow_jagged_rows: optional boolean for csv only + allow_quoted_newlines: optional boolean for csv only + field_delimiter: optional string for csv only + quote: optional string the quote character for csv only + skip_leading_rows: optional int for csv only + + Optional arguments with value None are determined by + BigQuery as described: + https://developers.google.com/bigquery/docs/reference/v2/jobs + + Returns: + dict, a BigQuery job resource or None on failure + """ + source_uris = source_uris if isinstance(source_uris, list) \ + else [source_uris] + + configuration = { + "destinationTable": { + "projectId": self.project_id, + "tableId": table, + "datasetId": dataset + }, + "sourceUris": source_uris, + } + + if max_bad_records: + configuration['maxBadRecords'] = max_bad_records + + if ignore_unknown_values: + configuration['ignoreUnknownValues'] = ignore_unknown_values + + if create_disposition: + configuration['createDisposition'] = create_disposition + + if write_disposition: + configuration['writeDisposition'] = write_disposition + + if encoding: + configuration['encoding'] = encoding + + if schema: + configuration['schema'] = schema + + if source_format: + configuration['sourceFormat'] = source_format + + if not job: + hex = sha256(":".join(source_uris) + str(time())).hexdigest() + job = "{dataset}-{table}-{digest}".format( + dataset=dataset, + table=table, + digest=hex + ) + + if source_format == JOB_SOURCE_FORMAT_CSV: + if field_delimiter: + configuration['fieldDelimiter'] = field_delimiter + + if allow_jagged_rows: + configuration['allowJaggedRows'] = allow_jagged_rows + + if allow_quoted_newlines: + configuration['allowQuotedNewlines'] = allow_quoted_newlines + + if quote: + configuration['quote'] = quote + + if skip_leading_rows: + configuration['skipLeadingRows'] = skip_leading_rows + + elif field_delimiter or allow_jagged_rows \ + or allow_quoted_newlines or quote or skip_leading_rows: + all_values = dict(field_delimiter=field_delimiter, + allow_jagged_rows=allow_jagged_rows, + allow_quoted_newlines=allow_quoted_newlines, + skip_leading_rows=skip_leading_rows, + quote=quote) + non_null_values = dict((k, v) for k, v + in all_values.items() + if v) + raise Exception("Parameters field_delimiter, allow_jagged_rows, " + "allow_quoted_newlines, quote and " + "skip_leading_rows are only allowed when " + "source_format=JOB_SOURCE_FORMAT_CSV: %s" + % non_null_values) + + body = { + "configuration": { + 'load': configuration + }, + "jobReference": { + "projectId": self.project_id, + "jobId": job + } + } + + try: + logger.info("Creating load job %s" % body) + job_resource = self.bigquery.jobs() \ + .insert(projectId=self.project_id, body=body) \ + .execute() + return job_resource + except Exception, e: + logger.error("Failed while starting uri import job: {0}" + .format(e)) + return None + + def wait_for_job(self, job, interval=5, timeout=None): + """ + Waits until the job indicated by job_resource is done or has failed + Args: + job: dict, representing a BigQuery job resource or jobId + interval: optional float polling interval in seconds, default = 5 + timeout: optional float timeout in seconds, default = None + Returns: + dict, final state of the job_resource, as described here: + https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/python/latest/bigquery_v2.jobs.html#get + Raises: + standard exceptions on http / auth failures (you must retry) + """ + if isinstance(job, dict): # job is a job resource + complete = job.get('jobComplete') + job_id = job['jobReference']['jobId'] + else: # job is the jobId + complete = False + job_id = job + job_resource = None + + start_time = time() + elapsed_time = 0 + while not (complete + or (timeout is not None and elapsed_time > timeout)): + sleep(interval) + request = self.bigquery.jobs().get(projectId=self.project_id, + jobId=job_id) + job_resource = request.execute() + error = job_resource.get('error') + if error: + raise Exception("{message} ({code}). Errors: {errors}", + **error) + complete = job_resource.get('status').get('state') == u'DONE' + elapsed_time = time() - start_time + + return job_resource + def push_rows(self, dataset, table, rows, insert_id_key=None): """Upload rows to BigQuery table. @@ -471,8 +670,8 @@ def _in_range(self, start_time, end_time, time): ONE_MONTH = 2764800 # 32 days return start_time <= time <= end_time or \ - time <= start_time <= time + ONE_MONTH or \ - time <= end_time <= time + ONE_MONTH + time <= start_time <= time + ONE_MONTH or \ + time <= end_time <= time + ONE_MONTH def _get_query_results(self, job_collection, project_id, job_id, offset=None, limit=None): @@ -594,8 +793,8 @@ def create_dataset(self, dataset_id, friendly_name=None, description=None, datasets.insert(projectId=self.project_id, body=dataset_data).execute() return True - except: - logger.error('Cannot create dataset %s' % dataset_id) + except Exception, e: + logger.error('Cannot create dataset %s, %s' % (dataset_id, e)) return False def get_datasets(self): diff --git a/bigquery/schema_builder.py b/bigquery/schema_builder.py index 9446e45..4316ca1 100644 --- a/bigquery/schema_builder.py +++ b/bigquery/schema_builder.py @@ -102,4 +102,3 @@ def bigquery_type(o, timestamp_parser=default_timestamp_parser): else: raise Exception( "Invalid object for BigQuery schema: {0} ({1}".format(o, t)) - diff --git a/bigquery/tests/test_client.py b/bigquery/tests/test_client.py index 6808eb1..86546e9 100644 --- a/bigquery/tests/test_client.py +++ b/bigquery/tests/test_client.py @@ -1,12 +1,12 @@ import unittest import mock +from nose.tools import raises from bigquery import client class TestGetClient(unittest.TestCase): - def setUp(self): client._bq_client = None @@ -84,7 +84,6 @@ def test_initialize_read_write(self, mock_build, mock_return_cred): class TestQuery(unittest.TestCase): - def setUp(self): client._bq_client = None @@ -168,8 +167,8 @@ def test_query_timeout(self): self.mock_job_collection.query.assert_called_once_with( projectId=self.project_id, - body={'query': self.query, 'timeoutMs': timeout * 1000, 'dryRun': - False, 'maxResults': None} + body={'query': self.query, 'timeoutMs': timeout * 1000, + 'dryRun': False, 'maxResults': None} ) self.assertEquals(job_id, 'spiderman') self.assertEquals(results, []) @@ -249,7 +248,6 @@ def test_query_with_results(self): class TestGetQueryResults(unittest.TestCase): - def setUp(self): client._bq_client = None @@ -289,7 +287,6 @@ def test_get_response(self): class TestTransformRow(unittest.TestCase): - def setUp(self): client._bq_client = None @@ -368,7 +365,6 @@ def test_transform_row_with_nested_repeated(self): @mock.patch('bigquery.client.BigQueryClient._get_query_results') class TestCheckJob(unittest.TestCase): - def setUp(self): client._bq_client = None self.project_id = 'project' @@ -408,8 +404,266 @@ def test_query_complete(self, mock_exec): self.assertEquals(total_rows, 2) -class TestFilterTablesByTime(unittest.TestCase): +class TestWaitForJob(unittest.TestCase): + def setUp(self): + client._bq_client = None + self.project_id = 'project' + self.api_mock = mock.Mock() + self.client = client.BigQueryClient(self.api_mock, self.project_id) + + def test_detects_completion(self): + """Ensure we can detect completed jobs""" + + return_values = [{'status': {'state': u'RUNNING'}, + 'jobReference': {'jobId': "testJob"}}, + {'status': {'state': u'DONE'}, + 'jobReference': {'jobId': "testJob"}}] + + def side_effect(*args, **kwargs): + return return_values.pop(0) + + self.api_mock.jobs().get().execute.side_effect = side_effect + + job_resource = self.client.wait_for_job( + {'jobComplete': False, + 'jobReference': {'jobId': "testJob"}}, + interval=.01, + timeout=None) + + self.assertEqual(self.api_mock.jobs().get().execute.call_count, 2) + self.assertIsInstance(job_resource, dict) + + def test_accepts_job_id(self): + """Ensure that a jobId argument is accepted""" + return_values = [{'status': {'state': u'RUNNING'}, + 'jobReference': {'jobId': "testJob"}}, + {'status': {'state': u'DONE'}, + 'jobReference': {'jobId': "testJob"}}] + + def side_effect(*args, **kwargs): + return return_values.pop(0) + + self.api_mock.jobs().get('status').execute.side_effect = side_effect + + self.client.wait_for_job('jobId', + interval=.01, + timeout=None) + + self.assertEqual(self.api_mock.jobs().get().execute.call_count, 2) + + def test_respects_timeout(self): + """Ensure that the wait can be timed out""" + incomplete_job = {'status': {'state': u'RUNNING'}, + 'jobReference': {'jobId': "testJob"}} + + self.api_mock.jobs().get().execute.return_value = incomplete_job + + job_resource = self.client.wait_for_job(incomplete_job, + interval=.1, + timeout=.25) + + self.assertEqual(self.api_mock.jobs().get().execute.call_count, 3) + self.assertEqual(job_resource['status']['state'], u'RUNNING') + + def test_respects_zero_timeout(self): + """Ensure that the wait times out even if timeout is zero""" + incomplete_job = {'status': {'state': u'RUNNING'}, + 'jobReference': {'jobId': "testJob"}} + + self.api_mock.jobs().get().execute.return_value = incomplete_job + + job_resource = self.client.wait_for_job(incomplete_job, + interval=.01, + timeout=0) + + self.assertEqual(self.api_mock.jobs().get().execute.call_count, 1) + self.assertEqual(job_resource['status']['state'], u'RUNNING') + + +class TestImportDataFromURIs(unittest.TestCase): + def setUp(self): + pass + + def test_csv_job_body_constructed_correctly(self): + mock_api = mock.Mock() + bq = client.BigQueryClient(mock_api, "project") + bq.import_data_from_uris(["sourceuri"], "dataset", "table", ["schema"], + job="job", + create_disposition="a", + write_disposition="b", + field_delimiter="c", + skip_leading_rows="d", + encoding="e", + quote="f", + max_bad_records="g", + allow_quoted_newlines="h", + source_format="CSV", + allow_jagged_rows="j", + ignore_unknown_values="k") + + body = { + "jobReference": { + "projectId": "project", + "jobId": "job" + }, + "configuration": { + "load": { + "sourceUris": ["sourceuri"], + "schema": ["schema"], + "destinationTable": { + "projectId": "project", + "datasetId": "dataset", + "tableId": "table" + }, + "createDisposition": "a", + "writeDisposition": "b", + "fieldDelimiter": "c", + "skipLeadingRows": "d", + "encoding": "e", + "quote": "f", + "maxBadRecords": "g", + "allowQuotedNewlines": "h", + "sourceFormat": "CSV", + "allowJaggedRows": "j", + "ignoreUnknownValues": "k" + } + } + } + mock_api.jobs().insert.assert_called_once_with(projectId="project", + body=body) + mock_api.jobs().insert(projectId="project", body=body) \ + .execute.called_once_with() + + def test_json_job_body_constructed_correctly(self): + mock_api = mock.Mock() + bq = client.BigQueryClient(mock_api, "project") + bq.import_data_from_uris(["sourceuri"], "dataset", "table", ["schema"], + job="job", + source_format="JSON") + + body = { + "jobReference": { + "projectId": "project", + "jobId": "job" + }, + "configuration": { + "load": { + "sourceUris": ["sourceuri"], + "schema": ["schema"], + "destinationTable": { + "projectId": "project", + "datasetId": "dataset", + "tableId": "table" + }, + "sourceFormat": "JSON" + } + } + } + + mock_api.jobs().insert.assert_called_once_with(projectId="project", + body=body) + mock_api.jobs().insert(projectId="project", body=body) \ + .execute.called_once_with() + + @raises(Exception) + def test_field_delimiter_exception_if_not_csv(self): + """Raise exception if csv-only parameter is set inappropriately""" + mock_api = mock.Mock() + bq = client.BigQueryClient(mock_api, "project") + bq.import_data_from_uris(["sourceuri"], "dataset", "table", ["schema"], + job="job", + source_format="JSON", + field_delimiter=",") + + @raises(Exception) + def test_allow_jagged_rows_exception_if_not_csv(self): + """Raise exception if csv-only parameter is set inappropriately""" + mock_api = mock.Mock() + bq = client.BigQueryClient(mock_api, "project") + bq.import_data_from_uris(["sourceuri"], "dataset", "table", ["schema"], + job="job", + source_format="JSON", + allow_jagged_rows=True) + + @raises(Exception) + def test_allow_quoted_newlines_exception_if_not_csv(self): + """Raise exception if csv-only parameter is set inappropriately""" + mock_api = mock.Mock() + bq = client.BigQueryClient(mock_api, "project") + bq.import_data_from_uris(["sourceuri"], "dataset", "table", ["schema"], + job="job", + source_format="JSON", + allow_quoted_newlines=True) + + @raises(Exception) + def test_quote_exception_if_not_csv(self): + """Raise exception if csv-only parameter is set inappropriately""" + mock_api = mock.Mock() + bq = client.BigQueryClient(mock_api, "project") + bq.import_data_from_uris(["sourceuri"], "dataset", "table", ["schema"], + job="job", + source_format="JSON", + quote="'") + + @raises(Exception) + def test_skip_leading_rows_exception_if_not_csv(self): + """Raise exception if csv-only parameter is set inappropriately""" + mock_api = mock.Mock() + bq = client.BigQueryClient(mock_api, "project") + bq.import_data_from_uris(["sourceuri"], "dataset", "table", ["schema"], + "job", + source_format="JSON", + skip_leading_rows=10) + + def test_accepts_single_source_uri(self): + """Ensure that a source_uri accepts a non-list""" + mock_api = mock.Mock() + bq = client.BigQueryClient(mock_api, "project") + bq.import_data_from_uris("sourceuri", # not a list! + "dataset", + "table", + schema=["schema"], + job="job") + + body = { + "jobReference": { + "projectId": "project", + "jobId": "job" + }, + "configuration": { + "load": { + "sourceUris": ["sourceuri"], + "schema": ["schema"], + "destinationTable": { + "projectId": "project", + "datasetId": "dataset", + "tableId": "table" + } + } + } + } + + mock_api.jobs().insert.assert_called_once_with(projectId="project", + body=body) + mock_api.jobs().insert(projectId="project", body=body) \ + .execute.called_once_with() + + def test_swallows_exception(self): + """Ensure that exceptions are handled""" + mock_api = mock.Mock() + mock_api.jobs().insert.side_effect = Exception + + bq = client.BigQueryClient(mock_api, "project") + result = bq.import_data_from_uris("sourceuri", # not a list! + "job", + "dataset", + "table", + ["schema"]) + self.assertEqual(result, None) + + +class TestFilterTablesByTime(unittest.TestCase): def test_empty_tables(self): """Ensure we can handle filtering an empty dictionary""" @@ -425,15 +679,15 @@ def test_multi_inside_range(self): bq = client.BigQueryClient(None, 'project') tables = bq._filter_tables_by_time({ - 'Spider-Man': 1370002001, - 'Daenerys Targaryen': 1370001999, - 'Gordon Freeman': 1369999999, - 'William Shatner': 1370001000, - 'Heavy Weapons Guy': 0 - }, 1370002000, 1370000000) + 'Spider-Man': 1370002001, + 'Daenerys Targaryen': 1370001999, + 'Gordon Freeman': 1369999999, + 'William Shatner': 1370001000, + 'Heavy Weapons Guy': 0 + }, 1370002000, 1370000000) self.assertEqual([ - 'Daenerys Targaryen', 'William Shatner', 'Gordon Freeman'], tables) + 'Daenerys Targaryen', 'William Shatner', 'Gordon Freeman'], tables) def test_not_inside_range(self): """Ensure we can correctly filter several application ids outside the @@ -443,11 +697,11 @@ def test_not_inside_range(self): bq = client.BigQueryClient(None, 'project') tables = bq._filter_tables_by_time({ - 'John Snow': 9001, - 'Adam West': 100000000000000, - 'Glados': -1, - 'Potato': 0, - }, 1370002000, 1370000000) + 'John Snow': 9001, + 'Adam West': 100000000000000, + 'Glados': -1, + 'Potato': 0, + }, 1370002000, 1370000000) self.assertEqual([], tables) @@ -530,7 +784,6 @@ def test_not_inside_range(self): @mock.patch('bigquery.client.BigQueryClient._get_query_results') class TestGetQuerySchema(unittest.TestCase): - def test_query_complete(self, get_query_mock): """Ensure that get_query_schema works when a query is complete.""" from bigquery.client import BigQueryClient @@ -564,7 +817,6 @@ def test_query_incomplete(self, get_query_mock): class TestGetTableSchema(unittest.TestCase): - def setUp(self): self.mock_bq_service = mock.Mock() self.mock_tables = mock.Mock() @@ -608,7 +860,6 @@ def test_table_does_not_exist(self): @mock.patch('bigquery.client.BigQueryClient._get_query_results') class TestGetQueryRows(unittest.TestCase): - def test_query_complete(self, get_query_mock): """Ensure that get_query_rows works when a query is complete.""" from bigquery.client import BigQueryClient @@ -664,7 +915,6 @@ def test_query_incomplete(self, get_query_mock): class TestCheckTable(unittest.TestCase): - def setUp(self): self.mock_bq_service = mock.Mock() self.mock_tables = mock.Mock() @@ -705,7 +955,6 @@ def test_table_does_exist(self): class TestCreateTable(unittest.TestCase): - def setUp(self): self.mock_bq_service = mock.Mock() self.mock_tables = mock.Mock() @@ -758,7 +1007,6 @@ def test_table_create_success(self): class TestDeleteTable(unittest.TestCase): - def setUp(self): self.mock_bq_service = mock.Mock() self.mock_tables = mock.Mock() @@ -799,7 +1047,6 @@ def test_delete_table_success(self): class TestParseTableListReponse(unittest.TestCase): - def test_full_parse(self): """Ensures we can parse a full list response.""" @@ -892,7 +1139,6 @@ def test_incorrect_table_formats(self): class TestPushRows(unittest.TestCase): - def setUp(self): self.mock_bq_service = mock.Mock() self.mock_table_data = mock.Mock() @@ -981,7 +1227,6 @@ def test_push_success(self): class TestGetAllTables(unittest.TestCase): - def test_get_tables(self): """Ensure get_all_tables fetches table names from BigQuery.""" @@ -1011,7 +1256,6 @@ def test_get_tables(self): class TestGetTables(unittest.TestCase): - def test_get_tables(self): """Ensure tables falling in the time window are returned.""" @@ -1057,7 +1301,6 @@ def test_get_tables_from_datetimes(self): # Dataset tests # class TestCreateDataset(unittest.TestCase): - def setUp(self): self.mock_bq_service = mock.Mock() self.mock_datasets = mock.Mock() @@ -1092,7 +1335,7 @@ def test_dataset_create_failed(self): self.mock_datasets.insert.assert_called_once_with( projectId=self.project, body=self.body) - self.mock_datasets.insert.return_value.execute.\ + self.mock_datasets.insert.return_value.execute. \ assert_called_once_with() def test_dataset_create_success(self): @@ -1110,12 +1353,11 @@ def test_dataset_create_success(self): self.mock_datasets.insert.assert_called_once_with( projectId=self.project, body=self.body) - self.mock_datasets.insert.return_value.execute.\ + self.mock_datasets.insert.return_value.execute. \ assert_called_once_with() class TestDeleteDataset(unittest.TestCase): - def setUp(self): self.mock_bq_service = mock.Mock() self.mock_datasets = mock.Mock() @@ -1153,7 +1395,7 @@ def test_delete_datasets_success(self): self.mock_datasets.delete.assert_called_once_with( projectId=self.project, datasetId=self.dataset) - self.mock_datasets.delete.return_value.execute.\ + self.mock_datasets.delete.return_value.execute. \ assert_called_once_with() @@ -1227,7 +1469,6 @@ def test_delete_datasets_success(self): class TestGetDatasets(unittest.TestCase): - def test_get_datasets(self): """Ensure datasets are returned.""" @@ -1247,7 +1488,6 @@ def test_get_datasets(self): class TestUpdateDataset(unittest.TestCase): - def setUp(self): self.mock_bq_service = mock.Mock() self.mock_datasets = mock.Mock() @@ -1282,7 +1522,7 @@ def test_dataset_update_failed(self): self.mock_datasets.update.assert_called_once_with( projectId=self.project, datasetId=self.dataset, body=self.body) - self.mock_datasets.update.return_value.execute.\ + self.mock_datasets.update.return_value.execute. \ assert_called_once_with() def test_dataset_update_success(self): @@ -1300,6 +1540,5 @@ def test_dataset_update_success(self): self.mock_datasets.update.assert_called_once_with( projectId=self.project, datasetId=self.dataset, body=self.body) - self.mock_datasets.update.return_value.execute.\ + self.mock_datasets.update.return_value.execute. \ assert_called_once_with() - diff --git a/bigquery/tests/test_query_builder.py b/bigquery/tests/test_query_builder.py index cd3844a..3aaeccb 100644 --- a/bigquery/tests/test_query_builder.py +++ b/bigquery/tests/test_query_builder.py @@ -2,7 +2,6 @@ class TestRenderSelect(unittest.TestCase): - def test_multiple_selects(self): """Ensure that render select can handle multiple selects.""" from bigquery.query_builder import _render_select @@ -30,7 +29,7 @@ def test_casting(self): result = _render_select({ 'start_time': {'alias': 'TimeStamp', 'format': - 'SEC_TO_MICRO-INTEGER-FORMAT_UTC_USEC'} + 'SEC_TO_MICRO-INTEGER-FORMAT_UTC_USEC'} }) self.assertEqual( @@ -47,7 +46,6 @@ def test_no_selects(self): class TestRenderSources(unittest.TestCase): - def test_multi_tables(self): """Ensure that render sources can handle multiple sources.""" from bigquery.query_builder import _render_sources @@ -75,7 +73,6 @@ def test_no_dataset(self): class TestRenderConditions(unittest.TestCase): - def test_single_condition(self): """Ensure that render conditions can handle a single condition.""" from bigquery.query_builder \ @@ -211,7 +208,6 @@ def test_in_comparator(self): class TestRenderOrder(unittest.TestCase): - def test_order(self): """Ensure that render order can work under expected conditions.""" from bigquery.query_builder import _render_order @@ -230,7 +226,6 @@ def test_no_order(self): class TestGroupings(unittest.TestCase): - def test_mutliple_fields(self): """Ensure that render grouping works with multiple fields.""" from bigquery.query_builder \ @@ -251,7 +246,6 @@ def test_no_fields(self): class TestRenderQuery(unittest.TestCase): - def test_full_query(self): """Ensure that all the render query arguments work together.""" from bigquery.query_builder import render_query @@ -335,7 +329,7 @@ def test_incorrect_conditions(self): }, conditions=[ {'asdfasdfasdf': 'start_time', 'ffd': 1371566954, 'comparator': - '<=', 'type': 'INTEGER'}, + '<=', 'type': 'INTEGER'}, {'field': 'start_time', 'value': {'value': 1371556954, 'negate': False}, 'compoorattor': '>=', 'type': 'INTEGER'} @@ -583,7 +577,7 @@ def test_formatting(self): tables=['2013_06_appspot_1'], select={ 'start_time': {'alias': 'timestamp', 'format': - 'INTEGER-FORMAT_UTC_USEC'}, + 'INTEGER-FORMAT_UTC_USEC'}, 'status': {'alias': 'status'}, 'resource': {'alias': 'url'} }, @@ -618,10 +612,10 @@ def test_formatting_duplicate_columns(self): tables=['2013_06_appspot_1'], select={ 'start_time': [{'alias': 'timestamp', 'format': - 'INTEGER-FORMAT_UTC_USEC'}, + 'INTEGER-FORMAT_UTC_USEC'}, {'alias': 'day', 'format': - 'SEC_TO_MICRO-INTEGER-FORMAT_UTC_USEC-LEFT:10'} - ], + 'SEC_TO_MICRO-INTEGER-FORMAT_UTC_USEC-LEFT:10'} + ], 'status': {'alias': 'status'}, 'resource': {'alias': 'url'} }, @@ -658,7 +652,7 @@ def test_sec_to_micro_formatting(self): tables=['2013_06_appspot_1'], select={ 'start_time': {'alias': 'timestamp', 'format': - 'SEC_TO_MICRO-INTEGER-SEC_TO_TIMESTAMP'}, + 'SEC_TO_MICRO-INTEGER-SEC_TO_TIMESTAMP'}, 'status': {'alias': 'status'}, 'resource': {'alias': 'url'} },