Enhancement/251 cache validation

.gitignore

@@ -22,3 +22,4 @@ venv36
 coverage.xml
 docs/source/_*
 __pycache__
+__open_alchemy_*_cache__

CHANGELOG.md

@@ -9,7 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
-- Removed unnecessary imports in `__init__.py` files.
+- Removed unnecessary imports in `__init__.py` files. [#255]
+
+### Added
+
+- Caching validation results to speed up startup. [#251]
 
 ## [v2.1.0] - 2020-12-20
 
@@ -496,3 +500,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [#201]: https://github.com/jdkandersson/OpenAlchemy/issues/201
 [#202]: https://github.com/jdkandersson/OpenAlchemy/issues/202
 [#236]: https://github.com/jdkandersson/OpenAlchemy/issues/236
+[#251]: https://github.com/jdkandersson/OpenAlchemy/issues/251
+[#255]: https://github.com/jdkandersson/OpenAlchemy/issues/255

open_alchemy/__init__.py

@@ -64,7 +64,7 @@ def init_model_factory(
     schemas = components.get("schemas", {})
 
     # Pre-processing schemas
-    _schemas_module.process(schemas=schemas)
+    _schemas_module.process(schemas=schemas, spec_filename=spec_path)
 
     # Getting artifacts
     schemas_artifacts = _schemas_artifacts.get_from_schemas(

open_alchemy/build/MANIFEST.j2

@@ -1,3 +1,3 @@
-recursive-include {{ name }} *.json
+recursive-include {{ name }} *.json __open_alchemy_*_cache__
 remove .*
 

open_alchemy/build/__init__.py

@@ -10,6 +10,7 @@
 
 import jinja2
 
+from .. import cache
 from .. import exceptions
 from .. import models_file as models_file_module
 from .. import schemas as schemas_module
@@ -336,7 +337,9 @@ def dump(
         # Write files in the package directory.
         package = directory / name
         package.mkdir(parents=True, exist_ok=True)
-        (package / "spec.json").write_text(spec_str)
+        spec_file = package / "spec.json"
+        spec_file.write_text(spec_str)
+        cache.schemas_are_valid(str(spec_file))
         (package / "__init__.py").write_text(init)
     except OSError as exc:
         raise exceptions.BuildError(str(exc)) from exc

open_alchemy/build/setup.j2

@@ -4,7 +4,7 @@ setuptools.setup(
     name="{{ name }}",
     version="{{ version }}",
     packages=setuptools.find_packages(),
-    python_requires=">=3.6",
+    python_requires=">=3.7",
     install_requires=[
         "OpenAlchemy",
     ],

open_alchemy/cache.py

@@ -0,0 +1,173 @@
+"""
+Cache for OpenAlchemy.
+
+The name of the file is:
+__open_alchemy_<sha256 of spec filename>_cache__
+
+The structure of the file is:
+
+{
+    "hash": "<sha256 hash of the file contents>",
+    "data": {
+        "schemas": {
+            "valid": true/false
+        }
+    }
+}
+"""
+
+import hashlib
+import json
+import pathlib
+import shutil
+
+from . import exceptions
+
+
+def calculate_hash(value: str) -> str:
+    """Create hash of a value."""
+    sha256 = hashlib.sha256()
+    sha256.update(value.encode())
+    return sha256.hexdigest()
+
+
+def calculate_cache_path(path: pathlib.Path) -> pathlib.Path:
+    """
+    Calculate the name of the cache file.
+
+    Args:
+        path: The path to the spec file.
+
+    Returns:
+        The path to the cache file.
+
+    """
+    return path.parent / f"__open_alchemy_{calculate_hash(path.name)}_cache__"
+
+
+_HASH_KEY = "hash"
+_DATA_KEY = "data"
+_DATA_SCHEMAS_KEY = "schemas"
+_DATA_SCHEMAS_VALID_KEY = "valid"
+
+
+def schemas_valid(filename: str) -> bool:
+    """
+    Calculate whether the cache indicates that the schemas in the file are valid.
+
+    Algorithm:
+    1. If the file does not exist, return False.
+    2. If the file is actually a folder, return False.
+    3. If the spec file is actually a folder, return False.
+    4. If the spec file does not exist, return False.
+    5. Calculate the hash of the spec file contents.
+    6. Try to load the cache, if it fails or it is not a dictionary, return False.
+    7. Try to retrieve the hash key, if it does not exist, return False.
+    8. If the value of the hash key is different to the hash of the file, return False.
+    9. Look for the data.schemas.valid key, if it does not exist, return False.
+    12. If the value of data.schemas.valid is True return True, otherwise return False.
+
+    Args:
+        filename: The name of the OpenAPI specification file.
+
+    Returns:
+        Whether the cache indicates that the schemas in the file are valid.
+
+    """
+    path = pathlib.Path(filename)
+    cache_path = calculate_cache_path(path)
+
+    # Check that both file and cache exists and are files
+    if (
+        not path.exists()
+        or not path.is_file()
+        or not cache_path.exists()
+        or not cache_path.is_file()
+    ):
+        return False
+
+    file_hash = calculate_hash(path.read_text())
+
+    try:
+        cache = json.loads(cache_path.read_text())
+    except json.JSONDecodeError:
+        return False
+
+    cache_valid = (
+        isinstance(cache, dict)
+        and _HASH_KEY in cache
+        and _DATA_KEY in cache
+        and isinstance(cache[_DATA_KEY], dict)
+        and _DATA_SCHEMAS_KEY in cache[_DATA_KEY]
+        and isinstance(cache[_DATA_KEY][_DATA_SCHEMAS_KEY], dict)
+        and _DATA_SCHEMAS_VALID_KEY in cache[_DATA_KEY][_DATA_SCHEMAS_KEY]
+    )
+    if not cache_valid:
+        return False
+
+    cache_file_hash = cache[_HASH_KEY]
+    if file_hash != cache_file_hash:
+        return False
+
+    return cache[_DATA_KEY][_DATA_SCHEMAS_KEY][_DATA_SCHEMAS_VALID_KEY] is True
+
+
+def schemas_are_valid(filename: str) -> None:
+    """
+    Update the cache to indicate that the filename is valid.
+
+    Algorithm:
+    1. If the spec filename is actually a folder, raise a CacheError.
+    2. If the spec filename does not exist, raise a CacheError.
+    3. Calculate the hash of the spec file contents.
+    4. If the chache is actually a folder, delete the folder.
+    5. If the cache does not exist, create the cache.
+    6. Read the contents of the cache. If it is not a dictionary, throw the contents
+        away and create an empty dictionary.
+    7. Create or update the hash key in the cache dictionary to be the calculated value.
+    8. Look for the data key in the cache dictionary. If it does not exist or is not a
+        dictionary, make it an empty dictionary.
+    9. Look for the schemas key under data in the cache dictionary. If it does not exist
+        or is not a dictionary, set it to be an empty dictionary.
+    10. Create or update the valid key under data.schemas and set it to True.
+    11. Write the dictionary to the file as JSON.
+
+    Args:
+        filename: The name of the spec file.
+
+    """
+    path = pathlib.Path(filename)
+    if not path.exists():
+        raise exceptions.CacheError(
+            f"the spec file does not exists, filename={filename}"
+        )
+    if not path.is_file():
+        raise exceptions.CacheError(f"the spec file is not a file, filename={filename}")
+    file_hash = calculate_hash(path.read_text())
+
+    cache_path = calculate_cache_path(path)
+    if cache_path.exists() and not cache_path.is_file():
+        shutil.rmtree(cache_path)
+    if not cache_path.exists():
+        cache_path.write_text("", encoding="utf-8")
+
+    try:
+        cache = json.loads(cache_path.read_text())
+    except json.JSONDecodeError:
+        cache = {}
+    if not isinstance(cache, dict):
+        cache = {}
+
+    cache[_HASH_KEY] = file_hash
+
+    if _DATA_KEY not in cache or not isinstance(cache[_DATA_KEY], dict):
+        cache[_DATA_KEY] = {}
+    cache_data = cache[_DATA_KEY]
+    if _DATA_SCHEMAS_KEY not in cache_data or not isinstance(
+        cache_data[_DATA_SCHEMAS_KEY], dict
+    ):
+        cache_data[_DATA_SCHEMAS_KEY] = {}
+    cache_data_schemas = cache_data[_DATA_SCHEMAS_KEY]
+    cache_data_schemas[_DATA_SCHEMAS_VALID_KEY] = True
+
+    cache_path.write_text(json.dumps(cache), encoding="utf-8")

open_alchemy/exceptions.py

@@ -74,3 +74,7 @@ class BuildError(BaseError):
 
 class CLIError(BaseError):
     """Raised when an error occurs when the CLI is used."""
+
+
+class CacheError(BaseError):
+    """Raised when an error occurs when the cache is used."""

open_alchemy/schemas/__init__.py

@@ -1,13 +1,17 @@
 """Performs operations on the schemas to prepare them for further processing."""
 
+import typing
+
 from .. import types as _types
 from . import association
 from . import backref
 from . import foreign_key
 from . import validation
 
 
-def process(*, schemas: _types.Schemas) -> None:
+def process(
+    *, schemas: _types.Schemas, spec_filename: typing.Optional[str] = None
+) -> None:
     """
     Pre-process schemas.
 
@@ -18,7 +22,7 @@ def process(*, schemas: _types.Schemas) -> None:
         schemas: The schemas to pre-process in place.
 
     """
-    validation.process(schemas=schemas)
+    validation.process(schemas=schemas, spec_filename=spec_filename)
     backref.process(schemas=schemas)
     foreign_key.process(schemas=schemas)
     association.process(schemas=schemas)

open_alchemy/schemas/validation/__init__.py

@@ -2,6 +2,7 @@
 
 import typing
 
+from ... import cache
 from ... import exceptions as _exceptions
 from ... import types as _oa_types
 from ..helpers import iterate
@@ -87,14 +88,21 @@ def _other_schemas_checks(*, schemas: _oa_types.Schemas) -> types.Result:
     return types.Result(valid=True, reason=None)
 
 
-def process(*, schemas: _oa_types.Schemas) -> None:
+def process(
+    *, schemas: _oa_types.Schemas, spec_filename: typing.Optional[str] = None
+) -> None:
     """
     Validate schemas.
 
     Args:
         schemas: The schemas to validate.
+        spec_filename: The filename of the spec, used to cache the result.
 
     """
+    if spec_filename is not None:
+        if cache.schemas_valid(spec_filename):
+            return
+
     schemas_result = schemas_validation.check(schemas=schemas)
     if not schemas_result.valid:
         raise _exceptions.MalformedSchemaError(schemas_result.reason)
@@ -121,6 +129,9 @@ def process(*, schemas: _oa_types.Schemas) -> None:
     if not other_results_result.valid:
         raise _exceptions.MalformedSchemaError(other_results_result.reason)
 
+    if spec_filename is not None:
+        cache.schemas_are_valid(spec_filename)
+
 
 def check_one_model(*, schemas: _oa_types.Schemas) -> types.Result:
     """

setup.cfg

@@ -27,6 +27,7 @@ markers =
     utility_base
     validation
     validate
+    cache
 python_functions = test_*
 mocked-sessions = examples.app.database.db.session
 flake8-max-line-length = 88

tests/open_alchemy/integration/test_init.py

@@ -8,6 +8,7 @@
 import yaml
 
 import open_alchemy
+from open_alchemy import cache
 from open_alchemy.facades.sqlalchemy import types as sqlalchemy_types
 
 
@@ -236,6 +237,9 @@ def test_init_json(engine, sessionmaker, tmp_path):
     queried_model = session.query(model).first()
     assert queried_model.column == value
 
+    # Checking for cache
+    assert cache.schemas_valid(str(spec_file)) is True
+
 
 @pytest.mark.integration
 def test_init_json_remote(engine, sessionmaker, tmp_path, _clean_remote_schemas_store):
@@ -315,6 +319,9 @@ def test_init_yaml(engine, sessionmaker, tmp_path):
     queried_model = session.query(model).first()
     assert queried_model.column == value
 
+    # Checking for cache
+    assert cache.schemas_valid(str(spec_file)) is True
+
 
 @pytest.mark.integration
 def test_init_yaml_remote(engine, sessionmaker, tmp_path, _clean_remote_schemas_store):

tests/open_alchemy/schemas/validation/test_validation.py

@@ -1,5 +1,7 @@
 """Tests for validation rules."""
 
+import pathlib
+
 import pytest
 
 from open_alchemy import exceptions
@@ -259,6 +261,27 @@ def test_process(schemas, raises):
         validation.process(schemas=schemas)
 
 
+def test_process_cache(tmpdir):
+    """
+    GIVEN spec filename
+    WHEN process is called with the schemas and filename twice
+    THEN the schemas are not checked on the second run.
+    """
+    tmpdir_path = pathlib.Path(tmpdir)
+    spec_file = tmpdir_path / "spec.json"
+    spec_file.write_text("spec 1", encoding="utf-8")
+    schemas = {
+        "Schema1": {
+            "type": "object",
+            "x-tablename": "schema_1",
+            "properties": {"prop_1": {"type": "integer"}},
+        }
+    }
+
+    validation.process(schemas=schemas, spec_filename=str(spec_file))
+    validation.process(schemas={}, spec_filename=str(spec_file))
+
+
 CHECK_TESTS = [
     pytest.param(
         True,