From 2383bf42c4f407a0d96c04f9f0fb99e09c4b0e1b Mon Sep 17 00:00:00 2001 From: Caleigh Runge-Hottman Date: Fri, 29 Jan 2021 22:29:26 -0500 Subject: [PATCH 1/2] Create cdn_definitions.api module [RHELDST-4862] The cdn_definitions.api module allows clients to consume the data.json or the schema.json files via a convenient API. This commit also introduces an optional "source" argument to the data_loads method. The source argument allows the user to specify the source of the cdn_definitions data.json file. The source could be a URL or a local path on a directory tree. If a source is not specified, the location may be overriden via the CDN_DEFINITIONS_PATH environment variable, which also may either be in the form of a URL or a local path. --- requirements.txt | 1 + src/cdn_definitions/__init__.py | 4 +-- src/cdn_definitions/_impl/__init__.py | 39 ++++++++++++++++----------- src/cdn_definitions/api/__init__.py | 14 ++++++++++ test-requirements.txt | 1 + tests/test_api.py | 26 ++++++++++++++++++ tests/test_load.py | 35 ++++++++++++++++++++++++ tox.ini | 7 ++++- 8 files changed, 109 insertions(+), 18 deletions(-) create mode 100644 requirements.txt create mode 100644 src/cdn_definitions/api/__init__.py create mode 100644 tests/test_api.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f229360 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/src/cdn_definitions/__init__.py b/src/cdn_definitions/__init__.py index b1d66f3..560850b 100644 --- a/src/cdn_definitions/__init__.py +++ b/src/cdn_definitions/__init__.py @@ -1,3 +1,3 @@ -from ._impl import PathAlias, origin_aliases, rhui_aliases +from ._impl import PathAlias, origin_aliases, rhui_aliases, load_data -__all__ = ["PathAlias", "origin_aliases", "rhui_aliases"] +__all__ = ["PathAlias", "origin_aliases", "rhui_aliases", "load_data"] diff --git a/src/cdn_definitions/_impl/__init__.py b/src/cdn_definitions/_impl/__init__.py index d197367..9c07038 100644 --- a/src/cdn_definitions/_impl/__init__.py +++ b/src/cdn_definitions/_impl/__init__.py @@ -1,20 +1,29 @@ import json import os - - -def load_data(): - # Load data from first existing of these - candidate_paths = [ - os.path.join(os.path.dirname(os.path.dirname(__file__)), "data.json"), - "/usr/share/cdn-definitions/data.json", - ] - - # If env var is set, it takes highest precedence - if "CDN_DEFINITIONS_PATH" in os.environ: - candidate_paths.insert(0, os.environ["CDN_DEFINITIONS_PATH"]) - - existing_paths = [p for p in candidate_paths if os.path.exists(p)] - return json.load(open(existing_paths[0])) +import requests + + +def load_data(source=None): + if source is None: + # Load data from first existing of these + candidate_paths = [ + os.path.join(os.path.dirname(os.path.dirname(__file__)), "data.json"), + "/usr/share/cdn-definitions/data.json", + ] + + # If env var is set, it takes highest precedence + if "CDN_DEFINITIONS_PATH" in os.environ: + cdn_definitions_path = os.environ["CDN_DEFINITIONS_PATH"] + candidate_paths.insert(0, cdn_definitions_path) + if cdn_definitions_path.startswith("http"): + return requests.get(cdn_definitions_path) + existing_paths = [p for p in candidate_paths if os.path.exists(p)] + return json.load(open(existing_paths[0])) + if os.path.exists(source): + return json.load(open(source)) + if source.startswith("http"): + return requests.get(source) + raise RuntimeError("Could not load data") DATA = load_data() diff --git a/src/cdn_definitions/api/__init__.py b/src/cdn_definitions/api/__init__.py new file mode 100644 index 0000000..ee62dc0 --- /dev/null +++ b/src/cdn_definitions/api/__init__.py @@ -0,0 +1,14 @@ +import os +import json +from .. import load_data + + +def schema(): + with open( + os.path.join(os.path.dirname(os.path.dirname(__file__)), "schema.json") + ) as f: + return json.load(f) + + +def data(source=None): + return load_data(source) diff --git a/test-requirements.txt b/test-requirements.txt index 0ad3076..6ce7680 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,4 @@ pytest jsonschema PyYAML +mock diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..67183a5 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,26 @@ +import json +import os + +from cdn_definitions import api + + +def test_load_schema_api(): + schema = api.schema() + with open( + os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "src", + "cdn_definitions", + "schema.json", + ) + ) as f: + local_schema = json.load(f) + assert schema == local_schema + + +def test_load_data_api(tmpdir): + json_file = tmpdir.join("myfile.json") + json_file.write('{"hello": "world"}') + data = api.data(source=str(json_file)) + assert data == {"hello": "world"} + diff --git a/tests/test_load.py b/tests/test_load.py index bf84c66..9852a2a 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -1,4 +1,7 @@ from cdn_definitions._impl import load_data +from mock import patch +import pytest +import requests def test_can_load_custom_data(monkeypatch, tmpdir): @@ -11,3 +14,35 @@ def test_can_load_custom_data(monkeypatch, tmpdir): data = load_data() assert data == {"hello": "world"} + + +def test_can_load_custom_data_from_source(monkeypatch, tmpdir): + json_file = tmpdir.join("myfile.json") + json_file.write('{"hello": "world"}') + + # If we set an env var pointing at the above JSON file, the library + # should load it instead of the bundled data + data = load_data(source=str(json_file)) + + assert data == {"hello": "world"} + + +def test_can_load_url_from_env_var(monkeypatch): + with patch("cdn_definitions._impl.requests.get") as r: + r.return_value = {"hello": "world"} + monkeypatch.setenv("CDN_DEFINITIONS_PATH", "https://example") + data = load_data() + assert data == {"hello": "world"} + + +def test_can_load_url_from_source_arg(monkeypatch): + with patch("cdn_definitions._impl.requests.get") as r: + r.return_value = {"hello": "world"} + data = load_data("https://example.com") + + assert data == {"hello": "world"} + + +def test_invalid_source(monkeypatch, tmpdir): + with pytest.raises(RuntimeError, match="Could not load data"): + data = load_data(source="test") diff --git a/tox.ini b/tox.ini index a08a8be..2655cef 100644 --- a/tox.ini +++ b/tox.ini @@ -2,12 +2,15 @@ envlist = py27,py38,static,docs [testenv] -deps=-rtest-requirements.txt +deps= + -rrequirements.txt + -rtest-requirements.txt commands=pytest -v {posargs} whitelist_externals=sh [testenv:static] deps= + -rrequirements.txt -rtest-requirements.txt black pylint @@ -17,6 +20,7 @@ commands= [testenv:cov] deps= + -rrequirements.txt -rtest-requirements.txt pytest-cov usedevelop=true @@ -26,6 +30,7 @@ commands= [testenv:cov-travis] passenv = TRAVIS TRAVIS_* deps= + -rrequirements.txt -rtest-requirements.txt pytest-cov coveralls From b112aebfc6af972d354c35e15549d6ba936daaf6 Mon Sep 17 00:00:00 2001 From: Caleigh Runge-Hottman Date: Fri, 29 Jan 2021 22:29:26 -0500 Subject: [PATCH 2/2] Create wrapper class so data.json is only loaded once [RHELDST-4863] This commit includes the creation of a CdnData class, which holds methods related to the Python implementation of the data. The CdnData class allows for the data set to be loaded once. --- src/cdn_definitions/__init__.py | 5 +++-- src/cdn_definitions/_impl/__init__.py | 19 ------------------- src/cdn_definitions/_impl/cdn_data.py | 26 ++++++++++++++++++++++++++ tests/test_alias.py | 8 +++++--- tests/test_api.py | 1 - 5 files changed, 34 insertions(+), 25 deletions(-) create mode 100644 src/cdn_definitions/_impl/cdn_data.py diff --git a/src/cdn_definitions/__init__.py b/src/cdn_definitions/__init__.py index 560850b..8d9d9b7 100644 --- a/src/cdn_definitions/__init__.py +++ b/src/cdn_definitions/__init__.py @@ -1,3 +1,4 @@ -from ._impl import PathAlias, origin_aliases, rhui_aliases, load_data +from ._impl import PathAlias, load_data +from ._impl.cdn_data import CdnData -__all__ = ["PathAlias", "origin_aliases", "rhui_aliases", "load_data"] +__all__ = ["PathAlias", "load_data", "CdnData"] diff --git a/src/cdn_definitions/_impl/__init__.py b/src/cdn_definitions/_impl/__init__.py index 9c07038..1421c6b 100644 --- a/src/cdn_definitions/_impl/__init__.py +++ b/src/cdn_definitions/_impl/__init__.py @@ -26,9 +26,6 @@ def load_data(source=None): raise RuntimeError("Could not load data") -DATA = load_data() - - class PathAlias(object): """Represents an alias between one CDN path and another, generally used to make two directory trees on CDN serve identical content.""" @@ -48,19 +45,3 @@ def __init__(self, **kwargs): # Equal src/dest is nonsensical if self.src == self.dest: raise ValueError("%s cannot alias itself!" % self.src) - - -def rhui_aliases(): - """Returns: - list[:class:`~PathAlias`] - A list of aliases relating to RHUI paths. - """ - return [PathAlias(**elem) for elem in DATA["rhui_alias"]] - - -def origin_aliases(): - """Returns: - list[:class:`~PathAlias`] - A list of aliases relating to origin paths. - """ - return [PathAlias(**elem) for elem in DATA["origin_alias"]] diff --git a/src/cdn_definitions/_impl/cdn_data.py b/src/cdn_definitions/_impl/cdn_data.py new file mode 100644 index 0000000..578fb00 --- /dev/null +++ b/src/cdn_definitions/_impl/cdn_data.py @@ -0,0 +1,26 @@ +from . import load_data, PathAlias + + +class CdnData(object): + def __init__(self, source=None): + self._source = source + self._data = None + + def data(self): + if not self._data: + self._data = load_data(self._source) + return self._data + + def rhui_aliases(self): + """Returns: + list[:class:`~PathAlias`] + A list of aliases relating to RHUI paths. + """ + return [PathAlias(**elem) for elem in self.data()["rhui_alias"]] + + def origin_aliases(self): + """Returns: + list[:class:`~PathAlias`] + A list of aliases relating to origin paths. + """ + return [PathAlias(**elem) for elem in self.data()["origin_alias"]] diff --git a/tests/test_alias.py b/tests/test_alias.py index cbb3446..53aaaef 100644 --- a/tests/test_alias.py +++ b/tests/test_alias.py @@ -1,11 +1,12 @@ import pytest -from cdn_definitions import PathAlias, origin_aliases, rhui_aliases +from cdn_definitions import PathAlias, CdnData def test_rhui_sanity(): """rhui_aliases must exist with expected properties""" - paths = rhui_aliases() + data = CdnData() + paths = data.rhui_aliases() # There should be some items assert paths @@ -18,7 +19,8 @@ def test_rhui_sanity(): def test_origin_sanity(): """origin_aliases must exist with expected properties""" - paths = origin_aliases() + data = CdnData() + paths = data.origin_aliases() # There should be some items assert paths diff --git a/tests/test_api.py b/tests/test_api.py index 67183a5..134f317 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -23,4 +23,3 @@ def test_load_data_api(tmpdir): json_file.write('{"hello": "world"}') data = api.data(source=str(json_file)) assert data == {"hello": "world"} -