diff --git a/api/jobs/handlers.py b/api/jobs/handlers.py index 0b9148b71..9c18768a3 100644 --- a/api/jobs/handlers.py +++ b/api/jobs/handlers.py @@ -74,7 +74,7 @@ def suggest(self, _id, cont_name, cid): return suggest_container(gear, cont_name+'s', cid) # Temporary Function - def upload(self): + def upload(self): # pragma: no cover """Upload new gear tarball file""" if not self.user_is_admin: self.abort(403, 'Request requires admin') @@ -88,7 +88,7 @@ def upload(self): return {'_id': str(gear_id)} # Temporary Function - def download(self, **kwargs): + def download(self, **kwargs): # pragma: no cover """Download gear tarball file""" dl_id = kwargs.pop('cid') gear = get_gear(dl_id) @@ -224,7 +224,7 @@ def delete(self, cid, rid): class JobsHandler(base.RequestHandler): """Provide /jobs API routes.""" - def get(self): + def get(self): # pragma: no cover (no route) """List all jobs.""" if not self.superuser_request and not self.user_is_admin: self.abort(403, 'Request requires admin') diff --git a/api/placer.py b/api/placer.py index c16bb04f7..5be15bc4e 100644 --- a/api/placer.py +++ b/api/placer.py @@ -250,8 +250,8 @@ def check(self): ### # Remove when switch to dmv2 is complete across all gears - c_metadata = self.metadata.get(self.container_type, {}) - if self.context.get('job_id') and c_metadata and not c_metadata.get('files', []): + c_metadata = self.metadata.get(self.container_type, {}) # pragma: no cover + if self.context.get('job_id') and c_metadata and not c_metadata.get('files', []): # pragma: no cover job = Job.get(self.context.get('job_id')) input_names = [{'name': v.name} for v in job.inputs.itervalues()] diff --git a/api/resolver.py b/api/resolver.py index 691b13ab2..9964d1a61 100644 --- a/api/resolver.py +++ b/api/resolver.py @@ -26,11 +26,11 @@ class Node(object): @staticmethod def get_children(parent): - raise NotImplementedError() + raise NotImplementedError() # pragma: no cover @staticmethod def filter(children, criterion): - raise NotImplementedError() + raise NotImplementedError() # pragma: no cover def _get_files(table, match): """ @@ -81,7 +81,7 @@ def filter(children, criterion): for x in children: if x['node_type'] == "file" and x.get('name') == criterion: return x, FileNode - raise Exception('No ' + criterion + ' acquisition or file found.') + raise Exception('No ' + criterion + ' file found.') class SessionNode(Node): diff --git a/api/web/start.py b/api/web/start.py index ac6652889..ace553902 100644 --- a/api/web/start.py +++ b/api/web/start.py @@ -8,7 +8,7 @@ # Enable code coverage for testing when API is started # Start coverage before local module loading so their def and imports are counted # http://coverage.readthedocs.io/en/coverage-4.2/faq.html -if os.environ.get("SCITRAN_RUNTIME_COVERAGE") == "true": +if os.environ.get("SCITRAN_RUNTIME_COVERAGE") == "true": # pragma: no cover - oh, the irony def save_coverage(cov): print("Saving coverage") cov.stop() diff --git a/test/integration_tests/python/test_handlers.py b/test/integration_tests/python/test_handlers.py index 2c8a3bb86..4f2015fdf 100644 --- a/test/integration_tests/python/test_handlers.py +++ b/test/integration_tests/python/test_handlers.py @@ -78,3 +78,10 @@ def test_devicehandler(as_user, as_root, as_drone, api_db): # clean up api_db.devices.remove({'_id': 'test_drone'}) + + +def test_config_version(as_user): + # get database schema version + r = as_user.get('/version') + assert r.ok + assert r.text == '' # not set yet diff --git a/test/integration_tests/python/test_jobs.py b/test/integration_tests/python/test_jobs.py index eeb64e447..b92a498d8 100644 --- a/test/integration_tests/python/test_jobs.py +++ b/test/integration_tests/python/test_jobs.py @@ -1,5 +1,7 @@ import copy +import bson + def test_jobs_access(as_user): r = as_user.get('/jobs/next') @@ -18,7 +20,7 @@ def test_jobs_access(as_user): assert r.status_code == 403 -def test_jobs(data_builder, as_user, as_admin, as_root): +def test_jobs(data_builder, as_user, as_admin, as_root, api_db): gear = data_builder.create_gear() invalid_gear = data_builder.create_gear(gear={'custom': {'flywheel': {'invalid': True}}}) acquisition = data_builder.create_acquisition() @@ -70,6 +72,16 @@ def test_jobs(data_builder, as_user, as_admin, as_root): r = as_root.post('/jobs/000000000000000000000000/logs', json=job_logs) assert r.status_code == 404 + # get job log as text w/o logs + r = as_admin.get('/jobs/' + job1_id + '/logs/text') + assert r.ok + assert r.text == 'No logs were found for this job.' + + # get job log as html w/o logs + r = as_admin.get('/jobs/' + job1_id + '/logs/html') + assert r.ok + assert r.text == 'No logs were found for this job.' + # add job log r = as_root.post('/jobs/' + job1_id + '/logs', json=job_logs) assert r.ok @@ -83,19 +95,33 @@ def test_jobs(data_builder, as_user, as_admin, as_root): assert r.ok assert len(r.json()['logs']) == 2 + # add same logs again (for testing text/html logs) + r = as_root.post('/jobs/' + job1_id + '/logs', json=job_logs) + assert r.ok + + # get job log as text + r = as_admin.get('/jobs/' + job1_id + '/logs/text') + assert r.ok + assert r.text == 2 * ''.join(log['msg'] for log in job_logs) + + # get job log as html + r = as_admin.get('/jobs/' + job1_id + '/logs/html') + assert r.ok + assert r.text == 2 * ''.join('{msg}\n'.format(**log) for log in job_logs) + # get job config r = as_root.get('/jobs/' + job1_id + '/config.json') assert r.ok - # try to update job (user may only cancel) - # root = true for as_admin, until thats fixed, using user - r = as_user.put('/jobs/' + job1_id, json={'test': 'invalid'}) - assert r.status_code == 403 - # try to cancel job w/o permission (different user) r = as_user.put('/jobs/' + job1_id, json={'state': 'cancelled'}) assert r.status_code == 403 + # try to update job (user may only cancel) + api_db.jobs.update_one({'_id': bson.ObjectId(job1_id)}, {'$set': {'origin.id': 'user@user.com'}}) + r = as_user.put('/jobs/' + job1_id, json={'test': 'invalid'}) + assert r.status_code == 403 + # add job with implicit destination job2 = copy.deepcopy(job_data) del job2['destination'] diff --git a/test/integration_tests/python/test_resolver.py b/test/integration_tests/python/test_resolver.py index 13ab5a478..2fd7541fe 100644 --- a/test/integration_tests/python/test_resolver.py +++ b/test/integration_tests/python/test_resolver.py @@ -1,4 +1,13 @@ -def test_resolver(data_builder, as_admin, as_public): +def path_in_result(path, result): + return [node.get('_id', node.get('name')) for node in result['path']] == path + + +def child_in_result(child, result): + return sum(all((k in c and c[k]==v) for k,v in child.iteritems()) for c in result['children']) == 1 + + +def test_resolver(data_builder, as_admin, as_user, as_public, file_form): + # ROOT # try accessing resolver w/o logging in r = as_public.post('/resolve', json={'path': []}) assert r.status_code == 403 @@ -7,58 +16,166 @@ def test_resolver(data_builder, as_admin, as_public): r = as_admin.post('/resolve', json={'path': 'test'}) assert r.status_code == 500 - # resolve root + # resolve root (empty) r = as_admin.post('/resolve', json={'path': []}) + result = r.json() assert r.ok - assert r.json()['path'] == [] + assert result['path'] == [] + assert result['children'] == [] - # resolve root w/ group + # resolve root (1 group) group = data_builder.create_group() r = as_admin.post('/resolve', json={'path': []}) result = r.json() assert r.ok assert result['path'] == [] - assert sum(child['_id'] == group for child in result['children']) == 1 - assert all(child['node_type'] == 'group' for child in result['children']) + assert child_in_result({'_id': group, 'node_type': 'group'}, result) - # try to resolve non-existent group id - r = as_admin.post('/resolve', json={'path': ['non-existent-group-id']}) + # try to resolve non-existent root/child + r = as_admin.post('/resolve', json={'path': ['child']}) assert r.status_code == 500 - # resolve group + + # GROUP + # try to resolve root/group as different (and non-root) user + r = as_user.post('/resolve', json={'path': [group]}) + assert r.status_code == 403 + + # resolve root/group (empty) r = as_admin.post('/resolve', json={'path': [group]}) result = r.json() assert r.ok - assert [node['_id'] for node in result['path']] == [group] + assert path_in_result([group], result) assert result['children'] == [] - # resolve group w/ project + # resolve root/group (1 project) project_label = 'test-resolver-project-label' project = data_builder.create_project(label=project_label) r = as_admin.post('/resolve', json={'path': [group]}) result = r.json() assert r.ok - assert [node['_id'] for node in result['path']] == [group] - assert sum(child['_id'] == project for child in result['children']) == 1 - assert all(child['node_type'] == 'project' for child in result['children']) + assert path_in_result([group], result) + assert child_in_result({'_id': project, 'node_type': 'project'}, result) - # try to resolve non-existent project label - r = as_admin.post('/resolve', json={'path': [group, 'non-existent-project-label']}) + # try to resolve non-existent root/group/child + r = as_admin.post('/resolve', json={'path': [group, 'child']}) assert r.status_code == 500 - # resolve project + + # PROJECT + # resolve root/group/project (empty) r = as_admin.post('/resolve', json={'path': [group, project_label]}) result = r.json() assert r.ok - assert [node['_id'] for node in result['path']] == [group, project] + assert path_in_result([group, project], result) assert result['children'] == [] - # resolve project w/ session + # resolve root/group/project (1 file) + project_file = 'project_file' + r = as_admin.post('/projects/' + project + '/files', files=file_form(project_file)) + assert r.ok + r = as_admin.post('/resolve', json={'path': [group, project_label]}) + result = r.json() + assert r.ok + assert path_in_result([group, project], result) + assert child_in_result({'name': project_file, 'node_type': 'file'}, result) + assert len(result['children']) == 1 + + # resolve root/group/project (1 file, 1 session) session_label = 'test-resolver-session-label' session = data_builder.create_session(label=session_label) r = as_admin.post('/resolve', json={'path': [group, project_label]}) result = r.json() assert r.ok - assert [node['_id'] for node in result['path']] == [group, project] - assert sum(child['_id'] == session for child in result['children']) == 1 - assert all(child['node_type'] == 'session' for child in result['children']) + assert path_in_result([group, project], result) + assert child_in_result({'_id': session, 'node_type': 'session'}, result) + assert len(result['children']) == 2 + + # resolve root/group/project/file + r = as_admin.post('/resolve', json={'path': [group, project_label, project_file]}) + result = r.json() + assert r.ok + assert path_in_result([group, project, project_file], result) + assert result['children'] == [] + + # try to resolve non-existent root/group/project/child + r = as_admin.post('/resolve', json={'path': [group, project_label, 'child']}) + assert r.status_code == 500 + + + # SESSION + # resolve root/group/project/session (empty) + r = as_admin.post('/resolve', json={'path': [group, project_label, session_label]}) + result = r.json() + assert r.ok + assert path_in_result([group, project, session], result) + assert result['children'] == [] + + # resolve root/group/project/session (1 file) + session_file = 'session_file' + r = as_admin.post('/sessions/' + session + '/files', files=file_form(session_file)) + assert r.ok + r = as_admin.post('/resolve', json={'path': [group, project_label, session_label]}) + result = r.json() + assert r.ok + assert path_in_result([group, project, session], result) + assert child_in_result({'name': session_file, 'node_type': 'file'}, result) + assert len(result['children']) == 1 + + # resolve root/group/project/session (1 file, 1 acquisition) + acquisition_label = 'test-resolver-acquisition-label' + acquisition = data_builder.create_acquisition(label=acquisition_label) + r = as_admin.post('/resolve', json={'path': [group, project_label, session_label]}) + result = r.json() + assert r.ok + assert path_in_result([group, project, session], result) + assert child_in_result({'_id': acquisition, 'node_type': 'acquisition'}, result) + assert len(result['children']) == 2 + + # resolve root/group/project/session/file + r = as_admin.post('/resolve', json={'path': [group, project_label, session_label, session_file]}) + result = r.json() + assert r.ok + assert path_in_result([group, project, session, session_file], result) + assert result['children'] == [] + + # try to resolve non-existent root/group/project/session/child + r = as_admin.post('/resolve', json={'path': [group, project_label, session_label, 'child']}) + assert r.status_code == 500 + + + # ACQUISITION + # resolve root/group/project/session/acquisition (empty) + r = as_admin.post('/resolve', json={'path': [group, project_label, session_label, acquisition_label]}) + result = r.json() + assert r.ok + assert path_in_result([group, project, session, acquisition], result) + assert result['children'] == [] + + # resolve root/group/project/session/acquisition (1 file) + acquisition_file = 'acquisition_file' + r = as_admin.post('/acquisitions/' + acquisition + '/files', files=file_form(acquisition_file)) + assert r.ok + r = as_admin.post('/resolve', json={'path': [group, project_label, session_label, acquisition_label]}) + result = r.json() + assert r.ok + assert path_in_result([group, project, session, acquisition], result) + assert child_in_result({'name': acquisition_file, 'node_type': 'file'}, result) + assert len(result['children']) == 1 + + # resolve root/group/project/session/acquisition/file + r = as_admin.post('/resolve', json={'path': [group, project_label, session_label, acquisition_label, acquisition_file]}) + result = r.json() + assert r.ok + assert path_in_result([group, project, session, acquisition, acquisition_file], result) + assert result['children'] == [] + + # try to resolve non-existent root/group/project/session/acquisition/child + r = as_admin.post('/resolve', json={'path': [group, project_label, session_label, acquisition_label, 'child']}) + assert r.status_code == 500 + + + # FILE + # try to resolve non-existent (also invalid) root/group/project/session/acquisition/file/child + r = as_admin.post('/resolve', json={'path': [group, project_label, session_label, acquisition_label, acquisition_file, 'child']}) + assert r.status_code == 500 diff --git a/test/integration_tests/requirements-integration-test.txt b/test/integration_tests/requirements-integration-test.txt index 9ee7b735d..b11515b92 100644 --- a/test/integration_tests/requirements-integration-test.txt +++ b/test/integration_tests/requirements-integration-test.txt @@ -3,9 +3,11 @@ coverage==4.0.3 coveralls==1.1 mock==2.0.0 mongomock==3.8.0 +newrelic==2.60.0.46 pdbpp==0.8.3 pylint==1.5.3 pytest-cov==2.2.0 +pytest-mock==1.6.0 pytest-watch==3.8.0 pytest==2.8.5 requests_mock==1.3.0 diff --git a/test/unit_tests/python/test_config.py b/test/unit_tests/python/test_config.py new file mode 100644 index 000000000..9eae69c9a --- /dev/null +++ b/test/unit_tests/python/test_config.py @@ -0,0 +1,51 @@ +import json + +import api.config + + +def test_apply_env_variables(mocker, tmpdir): + auth_file, auth_content = 'auth_config.json', {'auth': {'test': 'test'}} + tmpdir.join(auth_file).write(json.dumps(auth_content)) + mocker.patch('os.environ', { + 'SCITRAN_AUTH_CONFIG_FILE': str(tmpdir.join(auth_file)), + 'SCITRAN_TEST_TRUE': 'true', + 'SCITRAN_TEST_FALSE': 'false', + 'SCITRAN_TEST_NONE': 'none'}) + config = { + 'auth': {}, + 'test': {'true': '', 'false': '', 'none': ''}} + api.config.apply_env_variables(config) + assert config == { + 'auth': {'test': 'test'}, + 'test': {'true': True, 'false': False, 'none': None}} + + +def test_create_or_recreate_ttl_index(mocker): + db = mocker.patch('api.config.db') + collection, index_id, index_name, ttl = 'collection', 'timestamp_1', 'timestamp', 1 + + # create - collection not in collection_names + db.collection_names.return_value = [] + api.config.create_or_recreate_ttl_index(collection, index_name, ttl) + db.collection_names.assert_called_with() + db[collection].create_index.assert_called_with(index_name, expireAfterSeconds=ttl) + db[collection].create_index.reset_mock() + + # create - index doesn't exist + db.collection_names.return_value = [collection] + db[collection].index_information.return_value = {} + api.config.create_or_recreate_ttl_index(collection, index_name, ttl) + db[collection].create_index.assert_called_with(index_name, expireAfterSeconds=ttl) + db[collection].create_index.reset_mock() + + # skip - index exists and matches + db[collection].index_information.return_value = {index_id: {'key': [[index_name]], 'expireAfterSeconds': ttl}} + api.config.create_or_recreate_ttl_index(collection, index_name, ttl) + assert not db[collection].create_index.called + + # recreate - index exists but doesn't match + db[collection].create_index.reset_mock() + db[collection].index_information.return_value = {index_id: {'key': [[index_name]], 'expireAfterSeconds': 10}} + api.config.create_or_recreate_ttl_index(collection, index_name, ttl) + db[collection].drop_index.assert_called_with(index_id) + db[collection].create_index.assert_called_with(index_name, expireAfterSeconds=ttl) diff --git a/test/unit_tests/python/test_web_start.py b/test/unit_tests/python/test_web_start.py new file mode 100644 index 000000000..b0f5f496f --- /dev/null +++ b/test/unit_tests/python/test_web_start.py @@ -0,0 +1,32 @@ +import sys + +import mock +import newrelic.api.exceptions +import pytest + +import api.web.start + + +def test_newrelic(config, mocker): + # set config var to trigger newrelic setup and setup mocks + config['core']['newrelic'] = 'test' + init = mocker.patch('newrelic.agent.initialize') + log = mocker.patch('api.web.start.log') + + # successfully enable monitoring via newrelic + api.web.start.app_factory() + init.assert_called_with('test') + log.info.assert_called_with('New Relic detected and loaded. Monitoring enabled.') + + # newrelic import error => log error and exit + init.side_effect = ImportError + with pytest.raises(SystemExit): + api.web.start.app_factory() + log.critical.assert_called_with('New Relic libraries not found.') + init.side_effect = None + + # newrelic config error => log error and exit + init.side_effect = newrelic.api.exceptions.ConfigurationError + with pytest.raises(SystemExit): + api.web.start.app_factory() + log.critical.assert_called_with('New Relic detected, but configuration invalid.')