Add support for combining multiple schemas.

requirements/base.in

@@ -1,4 +1,5 @@
 click
+deepmerge
 jsonschema
 pygithub
 pyyaml

requirements/base.txt

@@ -1,4 +1,4 @@
-# SHA1:4f2317e083f71fd04cb136c69d83dbad45556565
+# SHA1:66b10f5a2376464e542de525145ef76f588af355
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -20,6 +20,9 @@ chardet==3.0.4 \
 click==7.0 \
     --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
     --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7
+deepmerge==0.1.0 \
+    --hash=sha256:3d37f739e74e8a284ee0bd683daaef88acc8438ba048545aefb87ade695a2a34 \
+    --hash=sha256:ae23dd76d3c0d22d33a3fd3980c92d3f0773e4affb48d9b341847d0b0a24e8f8
 deprecated==1.2.6 \
     --hash=sha256:a515c4cf75061552e0284d123c3066fbbe398952c87333a92b8fc3dd8e4f9cc1 \
     --hash=sha256:b07b414c8aac88f60c1d837d21def7e83ba711052e03b3cbaff27972567a8f8d \
@@ -39,8 +42,8 @@ more-itertools==7.2.0 \
     --hash=sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832 \
     --hash=sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4 \
     # via zipp
-pygithub==1.44 \
-    --hash=sha256:fd10fc9006fd54080b190c5c863384381905160c8ea8e830c4a3d8219f23193d
+pygithub==1.44.1 \
+    --hash=sha256:453896a1c3d46eb6724598daa21cf7ae9a83c6012126e840e3f7c665142fb04f
 pyjwt==1.7.1 \
     --hash=sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e \
     --hash=sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96 \

src/dirschema/cli.py

@@ -8,15 +8,15 @@
 
 
 @click.command()
-@click.argument("schema")
 @click.argument("project_dir_or_repo", nargs=-1)
+@click.option("-s", "--schema", multiple=True)
 @click.option("-v", "--verbose", is_flag=True, help="Enables verbose mode")
 @click.option(
     "--access-token",
     default=os.environ.get("GITHUB_ACCESS_TOKEN"),
     help="Github access token, if checking a repository",
 )
-def dirschema(schema, project_dir_or_repo, verbose, access_token):
+def dirschema(project_dir_or_repo, schema, verbose, access_token):
     # TODO: why isn't logging.config.dictConfig working when we
     # set config for a "dirschema" logger
     if verbose:
@@ -27,9 +27,10 @@ def dirschema(schema, project_dir_or_repo, verbose, access_token):
         logging.basicConfig(level=logging.WARNING)
 
     from .checks import check_github_structure, check_ondisk_structure
-    from .schema import load_schema
+    from .schema import load_schemas
 
-    schema = load_schema(open(schema).read())
+    logging.debug(schema)
+    loaded_schema = load_schemas(*[open(s).read() for s in schema])
 
     project_errors = {}
 
@@ -38,9 +39,9 @@ def dirschema(schema, project_dir_or_repo, verbose, access_token):
         click.echo(f"Checking {tocheck}…")
         if "://" in tocheck:
             repo = urlparse(tocheck).path[1:]
-            project_errors[tocheck] = check_github_structure(schema, repo, access_token)
+            project_errors[tocheck] = check_github_structure(loaded_schema, repo, access_token)
         else:
-            project_errors[tocheck] = check_ondisk_structure(schema, tocheck)
+            project_errors[tocheck] = check_ondisk_structure(loaded_schema, tocheck)
 
     click.echo()
     click.echo("Results")

src/dirschema/schema.py

@@ -1,13 +1,53 @@
+import logging
 from pathlib import Path
 
 import jsonschema
 import yaml
+from deepmerge import STRATEGY_END, Merger
+from deepmerge.exception import InvalidMerge
+
+logger = logging.getLogger("dirschema")
 
 SCHEMA_SCHEMA = Path(__file__).parent / "schemas" / "dirschema-v1.yaml"
 
 
-def load_schema(schema):
-    loaded = yaml.safe_load(schema)
+def fail_if_file_is_absent_and_present(config, path, base, nxt):
+    for f in base.get("files", {}).keys():
+        if f in nxt.get("files", {}):
+            if base["files"][f].get("absent") != nxt["files"][f].get("absent"):
+                raise InvalidMerge(
+                    f"Cannot merge {base} and {nxt} because {f}"
+                    "is specified as both absent and present"
+                )
+
+    return STRATEGY_END
+
+
+def allow_same_simple_values(config, path, base, nxt):
+    if base != nxt:
+        raise InvalidMerge(f"Cannot merge {path} because values differ ({base}, {nxt})")
+
+    return nxt
+
+
+schema_merger = Merger(
+    [(list, ["append"]), (dict, [fail_if_file_is_absent_and_present, "merge"])],
+    [allow_same_simple_values],
+    [],
+)
+
+
+def combine_schemas(schemas):
+    combined = schemas[0]
+
+    for s in schemas[1:]:
+        schema_merger.merge(combined, s)
+
+    return combined
+
+
+def load_schemas(*schemas):
+    loaded = combine_schemas([yaml.safe_load(schema) for schema in schemas])
     schema_schema = yaml.safe_load(open(SCHEMA_SCHEMA).read())
     jsonschema.validate(loaded, schema_schema)
     return loaded

tests/on-disk-schema-secondary.yaml

@@ -0,0 +1,5 @@
+---
+files:
+  foo:
+    contains:
+      - "foo"

tests/test_cli.py

@@ -8,18 +8,28 @@
 from dirschema.cli import dirschema
 
 YAML_SCHEMA_PATH = Path(__file__).parent / "on-disk-schema.yaml"
+SECONDARY_SCHEMA_PATH = Path(__file__).parent / "on-disk-schema-secondary.yaml"
 JSON_SCHEMA_PATH = Path(__file__).parent / "on-disk-schema.json"
-GOOD_PROJECT = {"files": {"foo": "", "bar": ""}, "dirs": {"dir1": {"files": {"f1": "", "f2": ""}}}}
+GOOD_PROJECT = {
+    "files": {"foo": "foo", "bar": ""},
+    "dirs": {"dir1": {"files": {"f1": "", "f2": ""}}},
+}
 BAD_PROJECT = deepcopy(GOOD_PROJECT)
 BAD_PROJECT["files"] = {}
 
 
-@pytest.mark.parametrize("schema", (YAML_SCHEMA_PATH, JSON_SCHEMA_PATH))
-def test_cli_good_project(make_project, tmp_path, schema):
+@pytest.mark.parametrize(
+    "schemas", ((YAML_SCHEMA_PATH,), (YAML_SCHEMA_PATH, SECONDARY_SCHEMA_PATH), (JSON_SCHEMA_PATH,))
+)
+def test_cli_good_project(make_project, tmp_path, schemas):
     make_project(tmp_path, GOOD_PROJECT)
 
     runner = CliRunner()
-    result = runner.invoke(dirschema, [str(schema), str(tmp_path)])
+    args = []
+    for schema in schemas:
+        args.extend(["-s", str(schema)])
+    args.append(str(tmp_path))
+    result = runner.invoke(dirschema, args)
 
     if result.exit_code != 0:
         assert False, print_exception(*result.exc_info[:3])
@@ -31,7 +41,7 @@ def test_cli_bad_project(make_project, tmp_path):
     make_project(tmp_path, BAD_PROJECT)
 
     runner = CliRunner()
-    result = runner.invoke(dirschema, [str(YAML_SCHEMA_PATH), str(tmp_path)])
+    result = runner.invoke(dirschema, ["-s", str(YAML_SCHEMA_PATH), str(tmp_path)])
 
     if result.exit_code != 1:
         assert False, print_exception(*result.exc_info[:3])
@@ -46,7 +56,7 @@ def test_cli_multiple_good_projects(make_project, tmp_path_factory):
         make_project(d, GOOD_PROJECT)
 
     runner = CliRunner()
-    result = runner.invoke(dirschema, [str(YAML_SCHEMA_PATH), *[str(d) for d in dirs]])
+    result = runner.invoke(dirschema, ["-s", str(YAML_SCHEMA_PATH), *[str(d) for d in dirs]])
 
     if result.exit_code != 0:
         assert False, print_exception(*result.exc_info[:3])
@@ -61,7 +71,7 @@ def test_cli_multiple_projects_one_bad(make_project, tmp_path_factory):
     make_project(bad_dir, BAD_PROJECT)
 
     runner = CliRunner()
-    result = runner.invoke(dirschema, [str(YAML_SCHEMA_PATH), str(good_dir), str(bad_dir)])
+    result = runner.invoke(dirschema, ["-s", str(YAML_SCHEMA_PATH), str(good_dir), str(bad_dir)])
 
     if result.exit_code != 1:
         assert False, print_exception(*result.exc_info[:3])

tests/test_schema.py

@@ -1,38 +1,87 @@
 import pytest
+from deepmerge.exception import InvalidMerge
 from jsonschema import ValidationError
 from yaml import YAMLError
 
-from dirschema.schema import load_schema
+from dirschema.schema import combine_schemas, load_schemas
 
 
 @pytest.mark.parametrize(
-    "schema",
+    "schemas",
     (
-        '{"files": {}, "dirs": {}, "allow_extra_files": false, "allow_extra_dirs": false}',
-        '{"files": {"foo": {"absent": True}}, "dirs": {},'
-        '"allow_extra_files": false, "allow_extra_dirs": false}',
-        '{"files": {}, "dirs": {"dir1": {"absent": True}},'
-        '"allow_extra_files": false, "allow_extra_dirs": false}',
+        ('{"files": {}, "dirs": {}, "allow_extra_files": false, "allow_extra_dirs": false}',),
+        (
+            '{"files": {"foo": {"absent": True}}, "dirs": {},'
+            '"allow_extra_files": false, "allow_extra_dirs": false}',
+        ),
+        (
+            '{"files": {}, "dirs": {"dir1": {"absent": True}},'
+            '"allow_extra_files": false, "allow_extra_dirs": false}',
+        ),
     ),
 )
-def test_load_schema_good(schema):
-    load_schema(schema)
+def test_load_schemas_good(schemas):
+    load_schemas(*schemas)
     # No errors!
 
 
 @pytest.mark.parametrize(
-    "schema,expected",
+    "schemas,expected",
     (
-        ('{"files": {}, "dirs": {}}', ValidationError),
-        ('{"files": {"foo": {"contains": ["blah"], "absent": true}}}', ValidationError),
-        ('{"dirs": {"foo": {"allow_extra_files": true, "absent": true}}}', ValidationError),
-        ('{"files": {"foo": {"absent": false}}, "dirs": {}}', ValidationError),
-        ("][:badyaml:", YAMLError),
+        (('{"files": {}, "dirs": {}}',), ValidationError),
+        (('{"files": {"foo": {"contains": ["blah"], "absent": true}}}',), ValidationError),
+        (('{"dirs": {"foo": {"allow_extra_files": true, "absent": true}}}',), ValidationError),
+        (('{"files": {"foo": {"absent": false}}, "dirs": {}}',), ValidationError),
+        (("][:badyaml:",), YAMLError),
     ),
 )
-def test_load_schema_invalid(schema, expected):
+def test_load_schemas_invalid(schemas, expected):
     try:
-        load_schema(schema)
+        load_schemas(*schemas)
         assert False, "Shouldn't have successfully loaded schema"
     except Exception as e:
         assert isinstance(e, expected)
+
+
+@pytest.mark.parametrize(
+    "schemas,expected",
+    (
+        (
+            ({"files": {"foo": {}}}, {"dirs": {"bar": {}}}),
+            {"files": {"foo": {}}, "dirs": {"bar": {}}},
+        ),
+        (
+            ({"files": {"foo": {}, "bar": {}}}, {"files": {"foo": {"contains": ["foo"]}}}),
+            {"files": {"foo": {"contains": ["foo"]}, "bar": {}}},
+        ),
+        (
+            ({"files": {"foo": {"contains": ["oof"]}}}, {"files": {"foo": {"contains": ["foo"]}}}),
+            {"files": {"foo": {"contains": ["oof", "foo"]}}},
+        ),
+    ),
+)
+def test_can_combine(schemas, expected):
+    assert combine_schemas(schemas) == expected
+
+
+@pytest.mark.parametrize(
+    "schemas,expected",
+    (
+        (({"files": {"foo": {}}}, {"files": {"foo": {"absent": True}}}), "foo"),
+        (
+            (
+                {"dirs": {"foo": {"allow_empty_dirs": True}}},
+                {"dirs": {"foo": {"allow_empty_dirs": False}}},
+            ),
+            "allow_empty_dirs",
+        ),
+    ),
+)
+def test_cannot_combine(schemas, expected):
+    try:
+        print(schemas)
+        combine_schemas(schemas)
+        assert False, "Shouldn't have successfully combined schemas"
+    except Exception as e:
+        assert isinstance(e, InvalidMerge)
+        assert any([expected in a for a in e.args])