From dc8ebc4852ea2ad31ee8292642ac05c6e3d54f6e Mon Sep 17 00:00:00 2001 From: MrCreosote Date: Sun, 26 Oct 2025 16:05:21 -0700 Subject: [PATCH] Add base client call_method function and supporting code. Not worth the effort to set up an auth service etc. locally, just use a token. Still to do: * Dynamic service support * Async job via callback server support --- .github/workflows/test.yml | 7 +- .gitignore | 1 + README.md | 2 + pyproject.toml | 1 + src/kbase/sdk_baseclient.py | 158 ++++++++++++++++++++++++++- test.cfg.example | 5 + test/test_sdk_baseclient.py | 207 ++++++++++++++++++++++++++++++++++++ uv.lock | 11 ++ 8 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 test.cfg.example diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2cf6c05..7ff2386 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,7 +43,12 @@ jobs: - name: Run tests shell: bash - run: PYTHONPATH=src pytest --cov=src --cov-report=xml test + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_CI_TOKEN }} + run: | + cp test.cfg.example test.cfg + sed -ie "s/^test_token=.*$/&$KBASE_TEST_TOKEN/g" test.cfg + PYTHONPATH=src pytest --cov=src --cov-report=xml test - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 diff --git a/.gitignore b/.gitignore index fc23657..5e8bfa1 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ /.pytest_cache/ __pycache__ /.venv/ +/test.cfg diff --git a/README.md b/README.md index 6db4063..1eb0c91 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ code and tests to determine proper useage. ### Testing +Copy `test.cfg.example` to `test.cfg` and fill it in appropriately. + ``` uv sync --dev # only required on first run or when the uv.lock file changes PYTHONPATH=src uv run pytest test diff --git a/pyproject.toml b/pyproject.toml index c858a16..8baa82f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dev = [ "ipython==9.6.0", "pytest==8.4.2", "pytest-cov==7.0.0", + "semver>=3.0.4", ] [project.urls] diff --git a/src/kbase/sdk_baseclient.py b/src/kbase/sdk_baseclient.py index ab89348..fc7138c 100644 --- a/src/kbase/sdk_baseclient.py +++ b/src/kbase/sdk_baseclient.py @@ -2,9 +2,165 @@ The base client for all SDK clients. """ +import json as _json +import random as _random +import requests as _requests +import os as _os +from urllib.parse import urlparse as _urlparse +from typing import Any + + +# The first version is a pretty basic port from the old baseclient, removing some no longer +# relevant cruft. + __version__ = "0.1.0" +_CT = "content-type" +_AJ = "application/json" +_URL_SCHEME = frozenset(["http", "https"]) +_CHECK_JOB_RETRIES = 3 + + +class ServerError(Exception): + + def __init__(self, name, code, message, data=None, error=None): + super(Exception, self).__init__(message) + self.name = name + self.code = code + # Ew. Leave it for backwards compatibility + self.message = "" if message is None else message + # Not really worth setting up a mock for the error case + # data = JSON RPC 2.0, error = 1.1 + self.data = data or error or "" + + def __str__(self): + return self.name + ": " + str(self.code) + ". " + self.message + \ + "\n" + self.data + + +class _JSONObjectEncoder(_json.JSONEncoder): + + def default(self, obj): + if isinstance(obj, set): + return list(obj) + if isinstance(obj, frozenset): + return list(obj) + return _json.JSONEncoder.default(self, obj) + + class SDKBaseClient: - """ The base client. """ + """ + The KBase base client. + + url - the url of the the service to contact: + For SDK methods: the url of the callback service. + For SDK dynamic services: the url of the Service Wizard. + For other services: the url of the service. + timeout - methods will fail if they take longer than this value in seconds. + Default 1800. + token - a KBase authentication token. + trust_all_ssl_certificates - set to True to trust self-signed certificates. + If you don't understand the implications, leave as the default, False. + lookup_url - set to true when contacting KBase dynamic services. + async_job_check_time_ms - the wait time between checking job state for + asynchronous jobs run with the run_job method. + async_job_check_time_scale_percent - the percentage increase in wait time between async job + check attempts. + async_job_check_max_time_ms - the maximum time to wait for a job check attempt before + failing. + """ + def __init__( + self, + url: str, + *, + timeout: int = 30 * 60, + token: str = None, + trust_all_ssl_certificates: bool = False, # Too much of a pain to test + lookup_url: bool = False, + async_job_check_time_ms: int = 100, + async_job_check_time_scale_percent: int = 150, + async_job_check_max_time_ms: int = 300000 + ): + if url is None: + raise ValueError("A url is required") + scheme, _, _, _, _, _ = _urlparse(url) + if scheme not in _URL_SCHEME: + raise ValueError(url + " isn't a valid http url") + self.url = url + self.timeout = int(timeout) + self._headers = {} + self.trust_all_ssl_certificates = trust_all_ssl_certificates + self.lookup_url = lookup_url + self.async_job_check_time = async_job_check_time_ms / 1000.0 + self.async_job_check_time_scale_percent = async_job_check_time_scale_percent + self.async_job_check_max_time = async_job_check_max_time_ms / 1000.0 + self.token = None + if token is not None: + self.token = token + # Not a fan of magic env vars but this is too baked in to remove + elif "KB_AUTH_TOKEN" in _os.environ: + self.token = _os.environ.get("KB_AUTH_TOKEN") + if self.token: + self._headers["AUTHORIZATION"] = self.token + if self.timeout < 1: + raise ValueError("Timeout value must be at least 1 second") + + def _call(self, url: str, method: str, params: list[Any], context: dict[str, Any] | None): + arg_hash = {"method": method, + "params": params, + "version": "1.1", + "id": str(_random.random())[2:], + } + if context: + arg_hash["context"] = context + + body = _json.dumps(arg_hash, cls=_JSONObjectEncoder) + ret = _requests.post( + url, + data=body, + headers=self._headers, + timeout=self.timeout, + verify=not self.trust_all_ssl_certificates + ) + ret.encoding = "utf-8" + if ret.status_code == 500: + if ret.headers.get(_CT) == _AJ: + err = ret.json() + if "error" in err: + raise ServerError(**err["error"]) + else: + raise ServerError( + "Unknown", 0, f"The server returned unexpected error JSON: {ret.text}" + ) + else: + raise ServerError( + "Unknown", 0, f"The server returned a non-JSON response: {ret.text}" + ) + if not ret.ok: + ret.raise_for_status() + resp = ret.json() + if "result" not in resp: + raise ServerError("Unknown", 0, "An unknown server error occurred") + if not resp["result"]: + return None + if len(resp["result"]) == 1: + return resp["result"][0] + return resp["result"] + + def call_method(self, service_method: str, args: list[Any], *, service_ver: str | None = None): + """ + Call a standard or dynamic service synchronously. + Required arguments: + service_method - the service and method to run, e.g. myserv.mymeth. + args - a list of arguments to the method. + Optional arguments: + service_ver - the version of the service to run, e.g. a git hash + or dev/beta/release. + """ + # TDOO NEXT implement dynamic methods + #url = self._get_service_url(service_method, service_ver) + #context = self._set_up_context(service_ver) + url = self.url + return self._call(url, service_method, args, None) diff --git a/test.cfg.example b/test.cfg.example new file mode 100644 index 0000000..451b670 --- /dev/null +++ b/test.cfg.example @@ -0,0 +1,5 @@ +[kbase_sdk_baseclient_tests] +# The url for the environment to test against. +test_url=https://ci.kbase.us/ +# A valid token for the environment. +test_token= diff --git a/test/test_sdk_baseclient.py b/test/test_sdk_baseclient.py index ece585b..5c5e7bd 100644 --- a/test/test_sdk_baseclient.py +++ b/test/test_sdk_baseclient.py @@ -1,8 +1,215 @@ +from configparser import ConfigParser +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import os +import pytest +import re +from requests.exceptions import HTTPError, ReadTimeout +import semver +import threading +import time + from kbase import sdk_baseclient _VERSION = "0.1.0" +_MOCKSERVER_PORT = 31590 # should be fine, find an empty port otherwise + + +@pytest.fixture(scope="module") +def url_and_token(): + config = ConfigParser() + config.read("test.cfg") + sec = config["kbase_sdk_baseclient_tests"] + return sec["test_url"], sec["test_token"] + + +class MockHandler(BaseHTTPRequestHandler): + def do_POST(self): + if self.path == "/not-json": + self.send_response(500) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"Wrong server pal") + elif self.path == "/missing-error": + self.send_response(500) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"oops": "no error key"}).encode("utf-8")) + else: + self.send_response(500) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b"Don't call this endpoint chum") + + +@pytest.fixture(scope="module") +def mockserver(): + server = HTTPServer(("localhost", _MOCKSERVER_PORT), MockHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + yield f"http://localhost:{_MOCKSERVER_PORT}" + + server.shutdown() def test_version(): assert sdk_baseclient.__version__ == _VERSION + + +def test_construct_fail(): + _test_construct_fail(None, 1, "A url is required") + _test_construct_fail("ftp://foo.com/bar", 1, "ftp://foo.com/bar isn't a valid http url") + for t in [.999999, 0, -1, -1000]: + _test_construct_fail("http://example.com", t, "Timeout value must be at least 1 second") + + +def _test_construct_fail(url: str, timeout: int, expected: str): + with pytest.raises(ValueError, match=expected): + sdk_baseclient.SDKBaseClient(url, timeout=timeout) + + +def test_tokenless(url_and_token): + bc = sdk_baseclient.SDKBaseClient(url_and_token[0] + "/services/ws") + res = bc.call_method("Workspace.ver", []) + semver.Version.parse(res) + + +def test_call_method_basic_passed_token(url_and_token): + # Tests returning a single value + _test_call_method_basic(url_and_token[0] + "/services/ws", url_and_token[1]) + + +def test_call_method_basic_env_token(url_and_token): + os.environ["KB_AUTH_TOKEN"] = url_and_token[1] + try: + _test_call_method_basic(url_and_token[0] + "/services/ws", None) + finally: + del os.environ["KB_AUTH_TOKEN"] + + +def _test_call_method_basic(url: str, token: str | None): + # Also tests a null result with delete_workspace + ws_name = f"sdk_baseclient_test_{time.time()}" + bc = sdk_baseclient.SDKBaseClient(url, token=token) + try: + res = bc.call_method("Workspace.create_workspace", [{"workspace": ws_name}]) + assert len(res) == 9 + assert res[1] == ws_name + assert res[4:] == [0, "a", "n", "unlocked", {}] + finally: + res = bc.call_method("Workspace.delete_workspace", [{"workspace": ws_name}]) + assert res is None + + +def test_serialize_sets_and_list_return(url_and_token): + """ + Tests + * Serializing set and frozenset + * Methods that return a list vs. a single value (save_objects). + """ + bc = sdk_baseclient.SDKBaseClient(url_and_token[0] + "/services/ws", token=url_and_token[1]) + ws_name = f"sdk_baseclient_test_{time.time()}" + try: + res = bc.call_method("Workspace.create_workspace", [{"workspace": ws_name}]) + wsid = res[0] + res = bc.call_method("Workspace.save_objects", [{ + "id": wsid, + "objects": [{ + "type": "Empty.AType", # basically no restrictions + "name": "foo", + "data": {}, + "provenance": [{ + "method_params": set(["a"]), + "intermediate_outgoing": frozenset(["b"]) + }] + }] + }]) + assert len(res) == 1 + res = res[0] + assert res[0] == 1 + assert res[1] == "foo" + assert res[2].startswith("Empty.AType") + assert res[4] == 1 + assert res[7:] == [ws_name, "99914b932bd37a50b983c5e7c90ae93b", 2, {}] + res = bc.call_method("Workspace.get_objects2", [{"objects": [{"ref": f"{wsid}/1/1"}]}]) + assert set(res.keys()) == {"data"} + objs = res["data"] + assert len(objs) == 1 + assert objs[0]["provenance"] == [{ + "method_params": ["a"], + "input_ws_objects": [], + "resolved_ws_objects": [], + "intermediate_incoming": [], + "intermediate_outgoing": ["b"], + "external_data": [], + "subactions": [], + "custom": {} + }] + finally: + res = bc.call_method("Workspace.delete_workspace", [{"workspace": ws_name}]) + + +def test_call_method_error(url_and_token): + bc = sdk_baseclient.SDKBaseClient(url_and_token[0] + "/services/ws", token=url_and_token[1]) + with pytest.raises(sdk_baseclient.ServerError) as got: + bc.call_method("Workspace.get_workspace_info", [{"id": 100000000000000}]) + assert got.value.name == "JSONRPCError" + assert got.value.message == "No workspace with id 100000000000000 exists" + assert got.value.code == -32500 + assert got.value.data.startswith( + "us.kbase.workspace.database.exceptions.NoSuchWorkspaceException: " + + "No workspace with id 100000000000000 exists" + ) + assert str(got.value).startswith( + "JSONRPCError: -32500. No workspace with id 100000000000000 exists\n" + + "us.kbase.workspace.database.exceptions.NoSuchWorkspaceException") + + +def test_error_non_500(url_and_token): + bc = sdk_baseclient.SDKBaseClient(url_and_token[0] + "/services/wsfake") + err = "404 Client Error: Not Found for url: https://ci.kbase.us//services/wsfake" + with pytest.raises(HTTPError, match=err): + bc.call_method("Workspace.ver", []) + + +def test_timeout(): + bc = sdk_baseclient.SDKBaseClient("https://httpbin.org/delay/10", timeout=1) + err = re.escape( + "HTTPSConnectionPool(host='httpbin.org', port=443): Read timed out. (read timeout=1)" + ) + with pytest.raises(ReadTimeout, match=err): + bc.call_method("Workspace.ver", []) + + +def test_missing_result_key(): + bc = sdk_baseclient.SDKBaseClient("https://httpbin.org/delay/0") + with pytest.raises(sdk_baseclient.ServerError) as got: + bc.call_method("Workspace.ver", []) + assert got.value.name == "Unknown" + assert got.value.message == "An unknown server error occurred" + assert got.value.code == 0 + assert got.value.data == "" + + +def test_not_application_json(mockserver): + bc = sdk_baseclient.SDKBaseClient(mockserver + "/not-json") + with pytest.raises(sdk_baseclient.ServerError) as got: + bc.call_method("Workspace.ver", []) + assert got.value.name == "Unknown" + assert got.value.message == "The server returned a non-JSON response: Wrong server pal" + assert got.value.code == 0 + assert got.value.data == "" + + +def test_missing_error_key(mockserver): + bc = sdk_baseclient.SDKBaseClient(mockserver + "/missing-error") + with pytest.raises(sdk_baseclient.ServerError) as got: + bc.call_method("Workspace.ver", []) + assert got.value.name == "Unknown" + assert got.value.message == ( + 'The server returned unexpected error JSON: {"oops": "no error key"}' + ) + assert got.value.code == 0 + assert got.value.data == "" diff --git a/uv.lock b/uv.lock index c5b61e0..226eac6 100644 --- a/uv.lock +++ b/uv.lock @@ -254,6 +254,7 @@ dev = [ { name = "ipython" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "semver" }, ] [package.metadata] @@ -264,6 +265,7 @@ dev = [ { name = "ipython", specifier = "==9.6.0" }, { name = "pytest", specifier = "==8.4.2" }, { name = "pytest-cov", specifier = "==7.0.0" }, + { name = "semver", specifier = ">=3.0.4" }, ] [[package]] @@ -401,6 +403,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[[package]] +name = "semver" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/d1/d3159231aec234a59dd7d601e9dd9fe96f3afff15efd33c1070019b26132/semver-3.0.4.tar.gz", hash = "sha256:afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602", size = 269730, upload-time = "2025-01-24T13:19:27.617Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912, upload-time = "2025-01-24T13:19:24.949Z" }, +] + [[package]] name = "stack-data" version = "0.6.3"