diff --git a/test/tests_for_integration/api_to_db_test.py b/test/tests_for_integration/api_to_db_test.py index cfb7ef243..dc46bb4ce 100644 --- a/test/tests_for_integration/api_to_db_test.py +++ b/test/tests_for_integration/api_to_db_test.py @@ -1,7 +1,7 @@ """ Integration tests that cover the entire codebase from API to database. -NOTE 1: These tests are designed to only be runnable after running docker-compose up +NOTE 1: These tests are designed to only be runnable after running docker-compose up. NOTE 2: These tests were set up quickly in order to debug a problem with administration related calls. As such, the auth server was set up to run in test mode locally. If more integrations @@ -9,19 +9,36 @@ If the latter, the test auth and workspace integrations will likely need to be converted to docker containers or exposed to other containers. -NOTE 3: Posting to Slack always fails silently. +NOTE 3: Although this is supposed to be an integration test, the catalog service and htcondor +are still mocked out as bringing them up would take a large amount of effort. Someday... + +NOTE 4: Kafka notes + a) Currently nothing listens to the kafka feed. + b) When running the tests, the kafka producer logs that kafka cannot be reached. However, + this error is silent otherwise. + c) I wasn't able to contact the docker kafka service with the kafka-python client either. + d) As such, Kafka is not tested. Once tests are added, at least one test should check that + something sensible happens if a kafka message cannot be sent. + +NOTE 5: EE2 posting to Slack always fails silently in tests. Currently slack calls are not tested. """ +# TODO add more integration tests, these are not necessarily exhaustive + import os import tempfile import time +import htcondor +from bson import ObjectId from configparser import ConfigParser from threading import Thread from pathlib import Path import pymongo -from pytest import fixture +from pytest import fixture, raises from typing import Dict +from unittest.mock import patch, create_autospec, ANY + from tests_for_integration.auth_controller import AuthController from tests_for_integration.workspace_controller import WorkspaceController from utils_shared.test_utils import ( @@ -34,11 +51,17 @@ create_auth_user, create_auth_role, set_custom_roles, + assert_close_to_now, + assert_exception_correct, ) from execution_engine2.sdk.EE2Constants import ADMIN_READ_ROLE, ADMIN_WRITE_ROLE +from installed_clients.baseclient import ServerError from installed_clients.execution_engine2Client import execution_engine2 as ee2client from installed_clients.WorkspaceClient import Workspace +# in the future remove this +from tests_for_utils.Condor_test import _get_common_sub + KEEP_TEMP_FILES = False TEMP_DIR = Path("test_temp_can_delete") @@ -52,6 +75,22 @@ USER_WRITE_ADMIN = "writeuser" TOKEN_WRITE_ADMIN = None +USER_WS_READ_ADMIN = "wsreadadmin" +TOKEN_WS_READ_ADMIN = None +USER_WS_FULL_ADMIN = "wsfulladmin" +TOKEN_WS_FULL_ADMIN = None +WS_READ_ADMIN = "WS_READ_ADMIN" +WS_FULL_ADMIN = "WS_FULL_ADMIN" + +CAT_GET_MODULE_VERSION = "installed_clients.CatalogClient.Catalog.get_module_version" +CAT_LIST_CLIENT_GROUPS = ( + "installed_clients.CatalogClient.Catalog.list_client_group_configs" +) + +# from test/deploy.cfg +MONGO_EE2_DB = "ee2" +MONGO_EE2_JOBS_COL = "ee2_jobs" + @fixture(scope="module") def config() -> Dict[str, str]: @@ -88,23 +127,41 @@ def _create_db_user(mongo_client, db, db_user, password): mongo_client[db].command("createUser", db_user, pwd=password, roles=["readWrite"]) +def _set_up_auth_user(auth_url, user, display, roles=None): + create_auth_user(auth_url, user, display) + if roles: + set_custom_roles(auth_url, user, roles) + return create_auth_login_token(auth_url, user) + + def _set_up_auth_users(auth_url): create_auth_role(auth_url, ADMIN_READ_ROLE, "ee2 admin read doohickey") create_auth_role(auth_url, ADMIN_WRITE_ROLE, "ee2 admin write thinger") + create_auth_role(auth_url, WS_READ_ADMIN, "wsr") + create_auth_role(auth_url, WS_FULL_ADMIN, "wsf") global TOKEN_READ_ADMIN - create_auth_user(auth_url, USER_READ_ADMIN, "display1") - TOKEN_READ_ADMIN = create_auth_login_token(auth_url, USER_READ_ADMIN) - set_custom_roles(auth_url, USER_READ_ADMIN, [ADMIN_READ_ROLE]) + TOKEN_READ_ADMIN = _set_up_auth_user( + auth_url, USER_READ_ADMIN, "display1", [ADMIN_READ_ROLE] + ) global TOKEN_NO_ADMIN - create_auth_user(auth_url, USER_NO_ADMIN, "display2") - TOKEN_NO_ADMIN = create_auth_login_token(auth_url, USER_NO_ADMIN) + TOKEN_NO_ADMIN = _set_up_auth_user(auth_url, USER_NO_ADMIN, "display2") global TOKEN_WRITE_ADMIN - create_auth_user(auth_url, USER_WRITE_ADMIN, "display3") - TOKEN_WRITE_ADMIN = create_auth_login_token(auth_url, USER_WRITE_ADMIN) - set_custom_roles(auth_url, USER_WRITE_ADMIN, [ADMIN_WRITE_ROLE]) + TOKEN_WRITE_ADMIN = _set_up_auth_user( + auth_url, USER_WRITE_ADMIN, "display3", [ADMIN_WRITE_ROLE] + ) + + global TOKEN_WS_READ_ADMIN + TOKEN_WS_READ_ADMIN = _set_up_auth_user( + auth_url, USER_WS_READ_ADMIN, "wsra", [WS_READ_ADMIN] + ) + + global TOKEN_WS_FULL_ADMIN + TOKEN_WS_FULL_ADMIN = _set_up_auth_user( + auth_url, USER_WS_FULL_ADMIN, "wsrf", [WS_FULL_ADMIN] + ) @fixture(scope="module") @@ -143,6 +200,27 @@ def auth_url(config, mongo_client): _clean_db(mongo_client, auth_db, auth_mongo_user) +def _add_ws_types(ws_controller): + wsc = Workspace(f"http://localhost:{ws_controller.port}", token=TOKEN_WS_FULL_ADMIN) + wsc.request_module_ownership("Trivial") + wsc.administer({"command": "approveModRequest", "module": "Trivial"}) + wsc.register_typespec( + { + "spec": """ + module Trivial { + /* @optional dontusethisfieldorifyoudomakesureitsastring */ + typedef structure { + string dontusethisfieldorifyoudomakesureitsastring; + } Object; + }; + """, + "dryrun": 0, + "new_types": ["Object"], + } + ) + wsc.release_module("Trivial") + + @fixture(scope="module") def ws_controller(config, mongo_client, auth_url): ws_db = "api_to_db_ws_test" @@ -170,6 +248,8 @@ def ws_controller(config, mongo_client, auth_url): f"Started KBase Workspace {ws.version} on port {ws.port} " + f"in dir {ws.temp_dir} in {ws.startup_count}s" ) + _add_ws_types(ws) + yield ws print(f"shutting down workspace, KEEP_TEMP_FILES={KEEP_TEMP_FILES}") @@ -287,7 +367,238 @@ def test_get_admin_permission_success(ee2_port): assert ee2cli_write.get_admin_permission() == {"permission": "w"} -def test_temporary_check_ws(ee2_port, ws_controller): +######## run_job tests ######## + + +def _get_htc_mocks(): + sub = create_autospec(htcondor.Submit, spec_set=True, instance=True) + schedd = create_autospec(htcondor.Schedd, spec_set=True, instance=True) + txn = create_autospec(htcondor.Transaction, spec_set=True, instance=True) + return sub, schedd, txn + + +def _finish_htc_mocks(sub_init, schedd_init, sub, schedd, txn): + sub_init.return_value = sub + schedd_init.return_value = schedd + # mock context manager ops + schedd.transaction.return_value = txn + txn.__enter__.return_value = txn + return sub, schedd, txn + + +def _check_htc_calls(sub_init, sub, schedd_init, schedd, txn, expected_sub): + sub_init.assert_called_once_with(expected_sub) + schedd_init.assert_called_once_with() + schedd.transaction.assert_called_once_with() + sub.queue.assert_called_once_with(txn, 1) + + +def test_run_job(ee2_port, ws_controller, mongo_client): + """ + A test of the run_job method. + """ + # Set up workspace and objects + wsc = Workspace(ws_controller.get_url(), token=TOKEN_NO_ADMIN) + wsc.create_workspace({"workspace": "foo"}) + wsc.save_objects( + { + "id": 1, + "objects": [ + {"name": "one", "type": "Trivial.Object-1.0", "data": {}}, + {"name": "two", "type": "Trivial.Object-1.0", "data": {}}, + ], + } + ) + + # need to get the mock objects first so spec_set can do its magic before we mock out + # the classes in the context manager + sub, schedd, txn = _get_htc_mocks() + # seriously black you're killing me here. This is readable? + with patch("htcondor.Submit", spec_set=True, autospec=True) as sub_init, patch( + "htcondor.Schedd", spec_set=True, autospec=True + ) as schedd_init, patch( + CAT_LIST_CLIENT_GROUPS, spec_set=True, autospec=True + ) as list_cgroups, patch( + CAT_GET_MODULE_VERSION, spec_set=True, autospec=True + ) as get_mod_ver: + # set up the rest of the mocks + _finish_htc_mocks(sub_init, schedd_init, sub, schedd, txn) + sub.queue.return_value = 123 + list_cgroups.return_value = [ + {"client_groups": ['{"request_cpus":8,"request_memory":5}']} + ] + get_mod_ver.return_value = {"git_commit_hash": "somehash"} + + # run the method + ee2 = ee2client(f"http://localhost:{ee2_port}", token=TOKEN_NO_ADMIN) + job_id = ee2.run_job( + { + "method": "mod.meth", + "app_id": "mod/app", + "wsid": 1, + "source_ws_objects": ["1/1/1", "1/2/1"], + "params": [{"foo": "bar"}, 42], + "service_ver": "beta", + "parent_job_id": "totallywrongid", + "meta": { + "run_id": "rid", + "token_id": "tid", + "tag": "yourit", + "cell_id": "cid", + "status": "totally wasted bro", + "thiskey": "getssilentlydropped", + }, + } + ) + + # check that mocks were called correctly + # Since these are class methods, the first argument is self, which we ignore + get_mod_ver.assert_called_once_with( + ANY, {"module_name": "mod", "version": "beta"} + ) + list_cgroups.assert_called_once_with( + ANY, {"module_name": "mod", "function_name": "meth"} + ) + + expected_sub = _get_common_sub(job_id) + expected_sub.update( + { + "JobBatchName": job_id, + "arguments": f"{job_id} https://ci.kbase.us/services/ee2", + "+KB_PARENT_JOB_ID": '"totallywrongid"', + "+KB_MODULE_NAME": '"mod"', + "+KB_FUNCTION_NAME": '"meth"', + "+KB_APP_ID": '"mod/app"', + "+KB_APP_MODULE_NAME": '"mod"', + "+KB_WSID": '"1"', + "+KB_SOURCE_WS_OBJECTS": '"1/1/1,1/2/1"', + "request_cpus": "8", + "request_memory": "5MB", + "request_disk": "30GB", + "requirements": 'regexp("njs",CLIENTGROUP)', + "+KB_CLIENTGROUP": '"njs"', + "Concurrency_Limits": f"{USER_NO_ADMIN}", + "+AccountingGroup": f'"{USER_NO_ADMIN}"', + "environment": ( + '"DOCKER_JOB_TIMEOUT=604805 KB_ADMIN_AUTH_TOKEN=test_auth_token ' + + f"KB_AUTH_TOKEN={TOKEN_NO_ADMIN} CLIENTGROUP=njs JOB_ID={job_id} " + + "CONDOR_ID=$(Cluster).$(Process) PYTHON_EXECUTABLE=/miniconda/bin/python " + + 'DEBUG_MODE=False PARENT_JOB_ID=totallywrongid "' + ), + "leavejobinqueue": "true", + "initial_dir": "../scripts/", + "+Owner": '"condor_pool"', + "executable": "../scripts//../scripts/execute_runner.sh", + "transfer_input_files": "../scripts/JobRunner.tgz", + } + ) + + _check_htc_calls(sub_init, sub, schedd_init, schedd, txn, expected_sub) + + # check the mongo record is correct + job = mongo_client[MONGO_EE2_DB][MONGO_EE2_JOBS_COL].find_one( + {"_id": ObjectId(job_id)} + ) + assert_close_to_now(job.pop("updated")) + assert_close_to_now(job.pop("queued")) + expected_job = { + "_id": ObjectId(job_id), + "user": USER_NO_ADMIN, + "authstrat": "kbaseworkspace", + "wsid": 1, + "status": "queued", + "job_input": { + "wsid": 1, + "method": "mod.meth", + "params": [{"foo": "bar"}, 42], + "service_ver": "somehash", + "app_id": "mod/app", + "source_ws_objects": ["1/1/1", "1/2/1"], + "parent_job_id": "totallywrongid", + "requirements": { + "clientgroup": "njs", + "cpu": 8, + "memory": 5, + "disk": 30, + }, + "narrative_cell_info": { + "run_id": "rid", + "token_id": "tid", + "tag": "yourit", + "cell_id": "cid", + "status": "totally wasted bro", + }, + }, + "child_jobs": [], + "batch_job": False, + "scheduler_id": "123", + "scheduler_type": "condor", + } + assert job == expected_job + + +def test_run_job_fail_no_workspace_access(ee2_port): + params = {"method": "mod.meth", "app_id": "mod/app", "wsid": 1} + # this error could probably use some cleanup + err = ( + "('An error occurred while fetching user permissions from the Workspace', " + + "ServerError('No workspace with id 1 exists'))" + ) + _run_job_fail(ee2_port, TOKEN_NO_ADMIN, params, err) + + +def test_run_job_fail_bad_method(ee2_port): + params = {"method": "mod.meth.moke", "app_id": "mod/app"} + # TODO the Server.py file is quoting strings for some reason it seems + # see https://github.com/kbase/sample_service/blob/master/lib/SampleService/SampleServiceServer.py#L119-L127 + err = "\"Unrecognized method: 'mod.meth.moke'. Please input module_name.function_name\"" + _run_job_fail(ee2_port, TOKEN_NO_ADMIN, params, err) + + +def test_run_job_fail_bad_app(ee2_port): + params = {"method": "mod.meth", "app_id": "mod.app"} + # TODO the Server.py file is quoting strings for some reason it seems + # see https://github.com/kbase/sample_service/blob/master/lib/SampleService/SampleServiceServer.py#L119-L127 + err = "\"Application ID 'mod.app' contains a '.'\"" + _run_job_fail(ee2_port, TOKEN_NO_ADMIN, params, err) + + +def test_run_job_fail_bad_upa(ee2_port): + params = { + "method": "mod.meth", + "app_id": "mod/app", + "source_ws_objects": ["ws/obj/1"], + } + # TODO the Server.py file is quoting strings for some reason it seems + # see https://github.com/kbase/sample_service/blob/master/lib/SampleService/SampleServiceServer.py#L119-L127 + err = "\"source_ws_objects index 0, 'ws/obj/1', is not a valid Unique Permanent Address\"" + _run_job_fail(ee2_port, TOKEN_NO_ADMIN, params, err) + + +def test_run_job_fail_no_such_object(ee2_port, ws_controller): + # Set up workspace and objects wsc = Workspace(ws_controller.get_url(), token=TOKEN_NO_ADMIN) - ws = wsc.create_workspace({"workspace": "foo"}) - assert ws[1] == "foo" + wsc.create_workspace({"workspace": "foo"}) + wsc.save_objects( + { + "id": 1, + "objects": [ + {"name": "one", "type": "Trivial.Object-1.0", "data": {}}, + ], + } + ) + params = {"method": "mod.meth", "app_id": "mod/app", "source_ws_objects": ["1/2/1"]} + # TODO the Server.py file is quoting strings for some reason it seems + # see https://github.com/kbase/sample_service/blob/master/lib/SampleService/SampleServiceServer.py#L119-L127 + err = "'Some workspace object is inaccessible'" + _run_job_fail(ee2_port, TOKEN_NO_ADMIN, params, err) + + +def _run_job_fail(ee2_port, token, params, expected, throw_exception=False): + client = ee2client(f"http://localhost:{ee2_port}", token=token) + if throw_exception: + client.run_job(params) + else: + with raises(ServerError) as got: + client.run_job(params) + assert_exception_correct(got.value, ServerError("name", 1, expected)) diff --git a/test/utils_shared/test_utils.py b/test/utils_shared/test_utils.py index e37bb7063..47d5d75e6 100644 --- a/test/utils_shared/test_utils.py +++ b/test/utils_shared/test_utils.py @@ -3,6 +3,7 @@ import uuid import logging import socket +import time from configparser import ConfigParser from contextlib import closing from datetime import datetime @@ -393,6 +394,15 @@ def assert_exception_correct(got: Exception, expected: Exception): assert type(got) == type(expected) +def assert_close_to_now(time_): + """ + Checks that a timestamp in seconds since the epoch is within a second of the current time. + """ + now_ms = time.time() + assert now_ms + 1 > time_ + assert now_ms - 1 < time_ + + def find_free_port() -> int: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(("", 0))