diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 39b25e5c56e6..d422fab6776b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,16 +81,22 @@ If you want to run the Google App Engine tests, you will need: $ export GAE_PYTHONPATH= -To run the bigquery tests, you'll need to create a bigquery dataset: +To run the bigquery tests: * Create a dataset in your project named `test_dataset`. -* Create a table named `test_table2`, upload ``tests/resources/data.csv`` and give it the following schema: - Name STRING - Age INTEGER - Weight FLOAT - IsMagic BOOLEAN + gcloud alpha bigquery datasets create test_dataset +* Load sample data into google cloud storage (for import tests): + + gsutil cp tests/resources/data.csv gs://$TEST_BUCKET_NAME/data.csv + +* Load the sample data into a table named `test_table` (for export and streaming tests): + + gcloud alpha bigquery import \ + gs://$TEST_BUCKET_NAME/data.csv \ + test_dataset/test_table \ + --schema-file tests/resources/schema.json ### Test environments diff --git a/appengine/bigquery/main_test.py b/appengine/bigquery/main_test.py index 58e79a6a40fb..7eff8ec469f6 100644 --- a/appengine/bigquery/main_test.py +++ b/appengine/bigquery/main_test.py @@ -28,6 +28,7 @@ class TestAuthSample(tests.AppEngineTestbedCase): def setUp(self): super(TestAuthSample, self).setUp() self.app = webtest.TestApp(main.app) + main.PROJECTID = self.project_id def test_anonymous_get(self): response = self.app.get('/') @@ -55,12 +56,7 @@ def test_oauthed_get(self, *args): {'status': '200'}) with mock.patch.object(main.decorator, 'http', return_value=mock_http): - original_projectid = main.PROJECTID - try: - main.PROJECTID = self.constants['projectId'] - response = self.app.get('/') - finally: - main.PROJECTID = original_projectid + response = self.app.get('/') # Should make the api call self.assertEqual(response.status_int, 200) diff --git a/bigquery/samples/async_query_test.py b/bigquery/samples/async_query_test.py index 5e6dbba2feea..336f9ec57a47 100644 --- a/bigquery/samples/async_query_test.py +++ b/bigquery/samples/async_query_test.py @@ -21,13 +21,17 @@ class TestAsyncQuery(tests.CloudBaseTest): def test_async_query(self): + query = ( + 'SELECT corpus FROM publicdata:samples.shakespeare ' + 'GROUP BY corpus;') + with tests.capture_stdout() as stdout: main( - self.constants['projectId'], - self.constants['query'], - False, - 5, - 5) + project_id=self.project_id, + query_string=query, + batch=False, + num_retries=5, + interval=1) value = stdout.getvalue().strip().split('\n').pop() diff --git a/bigquery/samples/export_data_to_cloud_storage_test.py b/bigquery/samples/export_data_to_cloud_storage_test.py index 6ab4adbb3e89..e93e5d459b96 100644 --- a/bigquery/samples/export_data_to_cloud_storage_test.py +++ b/bigquery/samples/export_data_to_cloud_storage_test.py @@ -19,33 +19,41 @@ class TestExportTableToGCS(CloudBaseTest): + dataset_id = 'test_dataset' + table_id = 'test_table' def test_export_table_csv(self): + cloud_storage_output_uri = \ + 'gs://{}/output.csv'.format(self.bucket_name) main( - self.constants['cloudStorageOutputURI'], - self.constants['projectId'], - self.constants['datasetId'], - self.constants['newTableId'], - 5, - 1, + cloud_storage_output_uri, + self.project_id, + self.dataset_id, + self.table_id, + num_retries=5, + interval=1, export_format="CSV") def test_export_table_json(self): + cloud_storage_output_uri = \ + 'gs://{}/output.json'.format(self.bucket_name) main( - self.constants['cloudStorageOutputURI'], - self.constants['projectId'], - self.constants['datasetId'], - self.constants['newTableId'], - 5, - 1, + cloud_storage_output_uri, + self.project_id, + self.dataset_id, + self.table_id, + num_retries=5, + interval=1, export_format="NEWLINE_DELIMITED_JSON") def test_export_table_avro(self): + cloud_storage_output_uri = \ + 'gs://{}/output.avro'.format(self.bucket_name) main( - self.constants['cloudStorageOutputURI'], - self.constants['projectId'], - self.constants['datasetId'], - self.constants['newTableId'], - 5, - 1, + cloud_storage_output_uri, + self.project_id, + self.dataset_id, + self.table_id, + num_retries=5, + interval=1, export_format="AVRO") diff --git a/bigquery/samples/getting_started_test.py b/bigquery/samples/getting_started_test.py index e2801e377d0b..5eb8242b6304 100644 --- a/bigquery/samples/getting_started_test.py +++ b/bigquery/samples/getting_started_test.py @@ -21,7 +21,7 @@ class TestGettingStarted(tests.CloudBaseTest): def test_main(self): with tests.capture_stdout() as mock_stdout: - main(self.constants['projectId']) + main(self.project_id) stdout = mock_stdout.getvalue() self.assertRegexpMatches(stdout, re.compile( diff --git a/bigquery/samples/list_datasets_projects_test.py b/bigquery/samples/list_datasets_projects_test.py index 51c027db0176..d99acb62f91d 100644 --- a/bigquery/samples/list_datasets_projects_test.py +++ b/bigquery/samples/list_datasets_projects_test.py @@ -22,7 +22,7 @@ class TestListDatasetsProjects(tests.CloudBaseTest): def test_main(self): with tests.capture_stdout() as mock_stdout: - main(self.constants['projectId']) + main(self.project_id) stdout = mock_stdout.getvalue() diff --git a/bigquery/samples/load_data_from_csv_test.py b/bigquery/samples/load_data_from_csv_test.py index 8cd51f696661..1b311a9ab4cc 100644 --- a/bigquery/samples/load_data_from_csv_test.py +++ b/bigquery/samples/load_data_from_csv_test.py @@ -20,12 +20,18 @@ class TestLoadDataFromCSV(CloudBaseTest): + dataset_id = 'test_dataset' + table_id = 'test_import_table' + def test_load_table(self): + cloud_storage_input_uri = 'gs://{}/data.csv'.format(self.bucket_name) + schema_file = os.path.join(self.resource_path, 'schema.json') + main( - self.constants['projectId'], - self.constants['datasetId'], - self.constants['newTableId'], - os.path.join(self.resource_path, 'schema.json'), - self.constants['cloudStorageInputURI'], - 1, - 5) + self.project_id, + self.dataset_id, + self.table_id, + schema_file=schema_file, + data_path=cloud_storage_input_uri, + poll_interval=1, + num_retries=5) diff --git a/bigquery/samples/streaming_test.py b/bigquery/samples/streaming_test.py index b6ad0cc62c5a..1325a85b4abb 100644 --- a/bigquery/samples/streaming_test.py +++ b/bigquery/samples/streaming_test.py @@ -21,6 +21,8 @@ class TestStreaming(CloudBaseTest): + dataset_id = 'test_dataset' + table_id = 'test_table' def test_stream_row_to_bigquery(self): with open( @@ -33,10 +35,10 @@ def test_stream_row_to_bigquery(self): with capture_stdout() as stdout: streaming.main( - self.constants['projectId'], - self.constants['datasetId'], - self.constants['newTableId'], - 5) + self.project_id, + self.dataset_id, + self.table_id, + num_retries=5) results = stdout.getvalue().split('\n') self.assertIsNotNone(json.loads(results[0])) diff --git a/bigquery/samples/sync_query_test.py b/bigquery/samples/sync_query_test.py index 6123614d234a..07ddb1cd07d7 100644 --- a/bigquery/samples/sync_query_test.py +++ b/bigquery/samples/sync_query_test.py @@ -21,12 +21,16 @@ class TestSyncQuery(CloudBaseTest): def test_sync_query(self): + query = ( + 'SELECT corpus FROM publicdata:samples.shakespeare ' + 'GROUP BY corpus;') + with capture_stdout() as stdout: main( - self.constants['projectId'], - self.constants['query'], - 30, - 5) + project_id=self.project_id, + query=query, + timeout=30, + num_retries=5) result = stdout.getvalue().split('\n')[0] self.assertIsNotNone(json.loads(result)) diff --git a/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py b/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py index f3042700deba..722ef624f2d0 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/blog_test.py @@ -20,4 +20,4 @@ class BlogTestCase(CloudBaseTest): """Simple test case that ensures the blog code doesn't throw any errors.""" def test_main(self): - main(self.constants['projectId']) + main(self.project_id) diff --git a/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py b/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py index 19a850e8eb91..aa3aea09676a 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py @@ -20,4 +20,4 @@ class WikiTestCase(CloudBaseTest): """Simple test case that ensures the wiki code doesn't throw any errors.""" def test_main(self): - main(self.constants['projectId']) + main(self.project_id) diff --git a/monitoring/samples/auth_test.py b/monitoring/samples/auth_test.py index 90e6d8865eb2..a0430bb47545 100644 --- a/monitoring/samples/auth_test.py +++ b/monitoring/samples/auth_test.py @@ -11,7 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import re import tests @@ -21,13 +20,9 @@ class TestTimeseriesList(tests.CloudBaseTest): - @classmethod - def setUpClass(cls): - cls.test_project_id = os.environ.get(tests.PROJECT_ID_ENV) - def test_main(self): with tests.capture_stdout() as stdout: - auth.main(self.test_project_id) + auth.main(self.project_id) output = stdout.getvalue().strip() self.assertRegexpMatches( output, re.compile(r'Timeseries.list raw response:\s*' diff --git a/storage/api/compose_objects_test.py b/storage/api/compose_objects_test.py index 695b1fad7ebb..7f37caa87b78 100644 --- a/storage/api/compose_objects_test.py +++ b/storage/api/compose_objects_test.py @@ -22,7 +22,7 @@ class TestComposeObjects(CloudBaseTest): def test_main(self): args = [ 'ignored_command_name', - self.constants['bucketName'], + self.bucket_name, 'dest.txt', os.path.join(self.resource_path, 'file1.txt'), os.path.join(self.resource_path, 'file2.txt'), diff --git a/storage/api/list_objects_test.py b/storage/api/list_objects_test.py index 85b5bfb1ac93..1f990699daf1 100644 --- a/storage/api/list_objects_test.py +++ b/storage/api/list_objects_test.py @@ -20,6 +20,6 @@ class TestListObjects(CloudBaseTest): def test_main(self): args = [ 'ignored_command_name', - self.constants['bucketName'] + self.bucket_name ] main(args) diff --git a/tests/__init__.py b/tests/__init__.py index 84164816c8a2..d6c5b977fb37 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -14,18 +14,14 @@ from .utils import ( AppEngineTestbedCase, - BUCKET_NAME_ENV, capture_stdout, CloudBaseTest, - PROJECT_ID_ENV, RESOURCE_PATH) __all__ = [ 'AppEngineTestbedCase', - 'BUCKET_NAME_ENV', 'capture_stdout', 'CloudBaseTest', - 'PROJECT_ID_ENV', 'RESOURCE_PATH' ] diff --git a/tests/resources/constants.json b/tests/resources/constants.json deleted file mode 100644 index dd76bb905c9e..000000000000 --- a/tests/resources/constants.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "datasetId": "test_dataset", - "currentTableId": "test_table", - "newTableId": "test_table2", - "cloudStorageInputURI": "gs://%s/data.csv", - "cloudStorageOutputURI": "gs://%s/output.csv", - "query": "SELECT corpus FROM publicdata:samples.shakespeare GROUP BY corpus;" -} diff --git a/tests/utils.py b/tests/utils.py index a6aa6e69ebd7..7c8064e58865 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -16,7 +16,6 @@ """ import contextlib -import json import os import sys import tempfile @@ -32,45 +31,30 @@ except ImportError: APPENGINE_AVAILABLE = False -BUCKET_NAME_ENV = 'TEST_BUCKET_NAME' -PROJECT_ID_ENV = 'TEST_PROJECT_ID' + RESOURCE_PATH = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'resources') +PROJECT_ID_ENV_VAR = 'TEST_PROJECT_ID' +BUCKET_NAME_ENV_VAR = 'TEST_BUCKET_NAME' class CloudBaseTest(unittest.TestCase): def setUp(self): self.resource_path = RESOURCE_PATH + self.project_id = os.environ.get(PROJECT_ID_ENV_VAR) - # A hack to prevent get_application_default from going GAE route. - self._server_software_org = os.environ.get('SERVER_SOFTWARE') - os.environ['SERVER_SOFTWARE'] = '' + if not self.project_id: + raise EnvironmentError( + 'You must set the {} environment variable to a valid Google ' + 'Cloud project ID.'.format(PROJECT_ID_ENV_VAR)) - # Constants from environment - test_bucket_name = os.environ.get(BUCKET_NAME_ENV, '') - test_project_id = os.environ.get(PROJECT_ID_ENV, '') - if not test_project_id or not test_bucket_name: - raise Exception('You need to define an env var "%s" and "%s" to ' - 'run the test.' - % (PROJECT_ID_ENV, BUCKET_NAME_ENV)) - - # Constants from resources/constants.json - with open( - os.path.join(RESOURCE_PATH, 'constants.json'), - 'r') as constants_file: - - self.constants = json.load(constants_file) - self.constants['projectId'] = test_project_id - self.constants['bucketName'] = test_bucket_name - self.constants['cloudStorageInputURI'] = ( - self.constants['cloudStorageInputURI'] % test_bucket_name) - self.constants['cloudStorageOutputURI'] = ( - self.constants['cloudStorageOutputURI'] % test_bucket_name) + self.bucket_name = os.environ.get(BUCKET_NAME_ENV_VAR) - def tearDown(self): - if self._server_software_org: - os.environ['SERVER_SOFTWARE'] = self._server_software_org + if not self.bucket_name: + raise EnvironmentError( + 'You must set the {} environment variable to a valid Google ' + 'Cloud Storage bucket.'.format(BUCKET_NAME_ENV_VAR)) class AppEngineTestbedCase(CloudBaseTest): @@ -81,6 +65,10 @@ def setUp(self): if not APPENGINE_AVAILABLE: raise SkipTest() + # A hack to prevent get_application_default from going GAE route. + self._server_software_org = os.environ.get('SERVER_SOFTWARE') + os.environ['SERVER_SOFTWARE'] = '' + # Setup the datastore and memcache stub. # First, create an instance of the Testbed class. self.testbed = testbed.Testbed() @@ -103,6 +91,10 @@ def setUp(self): def tearDown(self): super(AppEngineTestbedCase, self).tearDown() + + if self._server_software_org: + os.environ['SERVER_SOFTWARE'] = self._server_software_org + self.testbed.deactivate() def loginUser(self, email='user@example.com', id='123', is_admin=False): @@ -115,7 +107,7 @@ def loginUser(self, email='user@example.com', id='123', is_admin=False): @contextlib.contextmanager def capture_stdout(): - """Capture stdout.""" + """Capture stdout to a StringIO object.""" fake_stdout = cStringIO() old_stdout = sys.stdout