docgenerator: Added support for JSON, instead of XML, as a schema's f…

…ormat.

A schema that's documented by this system can now be in JSON
format. Previously this system assumed every documented format
was XML.

The new models JSONObject and JSONObjectRelationship contain
the meat of the metadata about all of the objects and keys
within a JSON format. These models are roughly analogous to
the existing XMLElement and XMLRelationship models. I originally
tried to shoehorn JSON into the existing XMLElement model, but
that ended up being very confusing -- hence the totally separate
models.

There are two new Django views, json_object_list and json_object_detail.
These work similarly to the existing element_list and element_detail.

Example pages now properly support JSON formats, inserting the
appropriate deep links to doc pages for each element key/value.

The json_object_detail page also has an 'Examples' section, linking to
each example document in which that particular JSON object is used.

Note that this commit is purely the infrastructure for the docs system
and doesn't include the actual changes to MNX (to convert it from XML
into a JSON format). That will come in a subsequent commit.

docgenerator/docgenerator/urls.py

@@ -12,6 +12,8 @@
     path('<slug:schema_slug>-reference/element-tree/', views.element_tree, name='element_tree'),
     path('<slug:schema_slug>-reference/examples/', views.example_list, name='example_list'),
     path('<slug:schema_slug>-reference/examples/<slug:slug>/', views.example_detail, name='example_detail'),
+    path('<slug:schema_slug>-reference/objects/', views.json_object_list, name='json_object_list'),
+    path('<slug:schema_slug>-reference/objects/<slug:slug>/', views.json_object_detail, name='json_object_detail'),
     path('concepts/', views.concept_list, name='concept_list'),
     path('concepts/<slug:slug>/', views.concept_detail, name='concept_detail'),
     path('comparisons/<slug:slug>/', views.format_comparison_detail, name='format_comparison_detail'),

docgenerator/spec/admin.py

@@ -23,6 +23,11 @@ class ChildElementsInline(admin.TabularInline):
     extra = 0
     fk_name = 'parent'
 
+class JSONChildElementsInline(admin.TabularInline):
+    model = JSONObjectRelationship
+    extra = 0
+    fk_name = 'parent'
+
 class ElementConceptInline(admin.TabularInline):
     model = ElementConcept
     extra = 0
@@ -52,6 +57,12 @@ class DataTypeAdmin(admin.ModelAdmin):
     list_display = ['name', 'base_type', 'xsd_name', 'is_featured']
     ordering = ['name']
 
+class JSONObjectAdmin(admin.ModelAdmin):
+    inlines = [JSONChildElementsInline]
+    list_display = ['name', 'slug', 'pretty_object_type']
+    ordering = ['name']
+    search_fields = ['name', 'slug']
+
 class DocumentFormatAdmin(admin.ModelAdmin):
     model = DocumentFormat
     list_display = ['name', 'slug']
@@ -85,6 +96,7 @@ class StaticPageAdmin(admin.ModelAdmin):
 admin.site.register(XMLElement, XMLElementAdmin)
 admin.site.register(XMLAttributeGroup, XMLAttributeGroupAdmin)
 admin.site.register(DataType, DataTypeAdmin)
+admin.site.register(JSONObject, JSONObjectAdmin)
 admin.site.register(DocumentFormat, DocumentFormatAdmin)
 admin.site.register(ExampleDocument, ExampleDocumentAdmin)
 admin.site.register(Concept, ConceptAdmin)

docgenerator/spec/migrations/0031_json_support.py

@@ -0,0 +1,67 @@
+# Generated by Django 4.1.5 on 2023-03-14 11:19
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('spec', '0030_auto_20211105_1029'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='xmlschema',
+            name='is_json',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='datatype',
+            name='union_types',
+            field=models.ManyToManyField(blank=True, help_text='If this data type is a union of multiple other types, list them here.', related_name='+', to='spec.datatype'),
+        ),
+        migrations.CreateModel(
+            name='JSONObject',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=80)),
+                ('slug', models.CharField(max_length=80)),
+                ('object_type', models.SmallIntegerField(choices=[(1, 'Dictionary'), (2, 'Array'), (3, 'String'), (4, 'Number'), (5, 'Boolean'), (6, 'Literal string')])),
+                ('schema', models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, to='spec.xmlschema')),
+            ],
+            options={
+                'verbose_name': 'JSON object',
+                'verbose_name_plural': 'JSON objects',
+                'db_table': 'json_objects',
+                'unique_together': {('schema', 'slug')},
+            },
+        ),
+        migrations.CreateModel(
+            name='ExampleDocumentObject',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('example', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='spec.exampledocument')),
+                ('json_object', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='spec.jsonobject')),
+            ],
+            options={
+                'db_table': 'example_objects',
+            },
+        ),
+        migrations.CreateModel(
+            name='JSONObjectRelationship',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('child_key', models.CharField(max_length=80)),
+                ('is_required', models.BooleanField(default=False)),
+                ('child', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='child_rel', to='spec.jsonobject')),
+                ('parent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='parent_rel', to='spec.jsonobject')),
+            ],
+            options={
+                'verbose_name': 'JSON object relationship',
+                'verbose_name_plural': 'JSON object relationships',
+                'db_table': 'json_object_relationships',
+                'unique_together': {('parent', 'child_key')},
+            },
+        ),
+    ]

docgenerator/spec/models.py

@@ -27,6 +27,7 @@ def __str__(self):
 class XMLSchema(models.Model):
     name = models.CharField(max_length=100)
     slug = models.CharField(max_length=100, unique=True)
+    is_json = models.BooleanField(default=False)
 
     class Meta:
         db_table = 'xml_schemas'
@@ -51,6 +52,9 @@ def elements_url(self):
     def element_tree_url(self):
         return reverse('element_tree', args=(self.slug,))
 
+    def json_objects_url(self):
+        return reverse('json_object_list', args=(self.slug,))
+
 class DataType(models.Model):
     name = models.CharField(max_length=80)
     slug = models.CharField(max_length=80, unique=True)
@@ -369,6 +373,114 @@ def pretty_amount(self):
             return '(One or more times)'
         return f'({min_amount} to {max_amount} times)'
 
+class JSONObject(models.Model):
+    OBJECT_TYPE_DICT = 1
+    OBJECT_TYPE_ARRAY = 2
+    OBJECT_TYPE_STRING = 3
+    OBJECT_TYPE_NUMBER = 4
+    OBJECT_TYPE_BOOLEAN = 5
+    OBJECT_TYPE_LITERAL_STRING = 6
+    OBJECT_TYPE_CHOICES = (
+        (OBJECT_TYPE_DICT, 'Dictionary'),
+        (OBJECT_TYPE_ARRAY, 'Array'),
+        (OBJECT_TYPE_STRING, 'String'),
+        (OBJECT_TYPE_NUMBER, 'Number'),
+        (OBJECT_TYPE_BOOLEAN, 'Boolean'),
+        (OBJECT_TYPE_LITERAL_STRING, 'Literal string'),
+    )
+    ROOT_OBJECT_NAME = '__root__'
+
+    name = models.CharField(max_length=80)
+    slug = models.CharField(max_length=80)
+    schema = models.ForeignKey(XMLSchema, on_delete=models.CASCADE, default=1)
+    object_type = models.SmallIntegerField(choices=OBJECT_TYPE_CHOICES)
+
+    class Meta:
+        db_table = 'json_objects'
+        verbose_name = 'JSON object'
+        verbose_name_plural = 'JSON objects'
+        unique_together = (
+            ('schema', 'slug'),
+        )
+
+    def __str__(self):
+        return self.name
+
+    def get_absolute_url(self):
+        return reverse('json_object_detail', args=(self.schema.slug, self.slug))
+
+    def is_array(self):
+        return self.object_type == JSONObject.OBJECT_TYPE_ARRAY
+
+    def get_child_relationships(self):
+        return list(JSONObjectRelationship.objects.filter(parent=self).order_by('child_key'))
+
+    def pretty_object_type(self):
+        return {
+            JSONObject.OBJECT_TYPE_DICT: 'Dictionary',
+            JSONObject.OBJECT_TYPE_ARRAY: 'Array',
+            JSONObject.OBJECT_TYPE_STRING: 'String',
+            JSONObject.OBJECT_TYPE_NUMBER: 'Number',
+            JSONObject.OBJECT_TYPE_BOOLEAN: 'Boolean',
+            JSONObject.OBJECT_TYPE_LITERAL_STRING: 'Literal string',
+        }[self.object_type]
+
+    def matches_json(self, json_data):
+        """
+        Given a JSON object, returns True if the object appears to be
+        described by this JSONObject definition.
+
+        This only searches one level deep.
+        """
+        object_type = self.object_type
+        if object_type == JSONObject.OBJECT_TYPE_DICT:
+            child_rels = {r.child_key: r for r in self.get_child_relationships()}
+            for k in json_data.keys():
+                if k not in child_rels:
+                    return False
+            return True
+        elif object_type == JSONObject.OBJECT_TYPE_ARRAY:
+            return isinstance(json_data, list)
+        elif object_type in {JSONObject.OBJECT_TYPE_STRING, JSONObject.OBJECT_TYPE_LITERAL_STRING}:
+            return isinstance(json_data, str)
+        elif object_type == JSONObject.OBJECT_TYPE_NUMBER:
+            return isinstance(json_data, (int, float))
+        else:
+            raise NotImplementedError()
+
+    @staticmethod
+    def get_jsonobject_for_data(json_data, object_def_list):
+        """
+        Given JSON data and a list of potential JSONObjects that describe it,
+        returns the JSONObject that describes it.
+        """
+        if len(object_def_list) == 1:
+            return object_def_list[0] # Common case.
+        for object_def in object_def_list:
+            if object_def.matches_json(json_data):
+                return object_def
+
+        # By now, one of the given object_def_list should have matched.
+        # If not, raise an exception to bring attention to the wonky data.
+        raise ValueError()
+
+class JSONObjectRelationship(models.Model):
+    parent = models.ForeignKey(JSONObject, on_delete=models.CASCADE, related_name='parent_rel')
+    child_key = models.CharField(max_length=80)
+    child = models.ForeignKey(JSONObject, on_delete=models.CASCADE, related_name='child_rel')
+    is_required = models.BooleanField(default=False)
+
+    class Meta:
+        db_table = 'json_object_relationships'
+        verbose_name = 'JSON object relationship'
+        verbose_name_plural = 'JSON object relationships'
+        unique_together = (
+            ('parent', 'child_key'),
+        )
+
+    def __repr__(self):
+        return f'<JSONObjectRelationship parent="{self.parent.name}" child="{self.child.name}">'
+
 class ExampleDocument(models.Model):
     name = models.CharField(max_length=300)
     slug = models.CharField(max_length=100)
@@ -432,6 +544,15 @@ class ExampleDocumentElement(models.Model):
     class Meta:
         db_table = 'example_elements'
 
+class ExampleDocumentObject(models.Model):
+    # This is a cache of each JSONObject used in each
+    # ExampleDocument. It's updated via ExampleDocument.save().
+    example = models.ForeignKey(ExampleDocument, on_delete=models.CASCADE)
+    json_object = models.ForeignKey(JSONObject, null=True, on_delete=models.CASCADE)
+
+    class Meta:
+        db_table = 'example_objects'
+
 class ElementConcept(models.Model):
     element = models.ForeignKey(XMLElement, on_delete=models.CASCADE)
     concept = models.ForeignKey(Concept, on_delete=models.CASCADE)

docgenerator/spec/templates/json_object_detail.html

@@ -0,0 +1,61 @@
+{% extends "base.html" %}
+
+{% block title %}The {{ object.name }} object{% endblock %}
+
+{% block content %}
+<p class="breadcrumb">
+    <a href="{% relative_url 'homepage' %}">{{ SITE_OPTIONS.site_name }}</a> &gt;
+    <a href="{% relative_url_string object.schema.reference_url %}">{{ object.schema.name }} reference</a> &gt;
+    <a href="{% relative_url_string object.schema.json_objects_url %}">Objects</a> &gt;
+    {{ object.name }}
+</p>
+
+<h1>The {{ object.name }} object</h1>
+
+<p><b>Type:</b> {{ object.pretty_object_type }}</p>
+
+{% if child_relationships %}
+<h2>Keys:</h2>
+
+<table>
+<thead>
+<tr>
+    <th>Name</th>
+    <th>Type</th>
+    <th>Required?</th>
+    <th>Description</th>
+</tr>
+</thead>
+
+{% for rel in child_relationships %}
+<tr>
+    <td>"{{ rel.child_key }}"</td>
+    <td>
+        {% if rel.child.is_array %}
+            An array of
+            {% for childrel in rel.child.get_child_relationships %}
+            <a href="{% relative_url_string childrel.child.get_absolute_url %}">{{ childrel.child.name }} objects</a>{% if not forloop.last %}, {% endif %}
+            {% endfor %}
+        {% else %}
+        <a href="{% relative_url_string rel.child.get_absolute_url %}">{{ rel.child.name }} object</a>
+        {% endif %}
+    </td>
+    <td>{{ rel.is_required|yesno|title }}</td>
+    <td></td>
+</tr>
+{% endfor %}
+
+</table>
+{% endif %}
+
+{% if examples %}
+<h2 id="examples">Examples</h2>
+
+<p>This element is used in the following examples:</p>
+<p>
+    {% for example in examples %}<nobr><a href="{% relative_url_string example.example.get_absolute_url %}">{{ example.example.name }}</a></nobr>{% if not forloop.last %}, {% endif %}{% endfor %}
+</p>
+{% endif %}
+
+{% endblock %}
+

docgenerator/spec/templates/json_object_list.html

@@ -0,0 +1,21 @@
+{% extends "base.html" %}
+
+{% block title %}Objects used in {{ schema.name }}{% endblock %}
+
+{% block content %}
+<p class="breadcrumb">
+    <a href="{% relative_url 'homepage' %}">{{ SITE_OPTIONS.site_name }}</a> &gt;
+    <a href="{% relative_url_string schema.reference_url %}">{{ schema.name }} reference</a> &gt;
+    Objects
+</p>
+
+<h1>Objects used in {{ schema.name }}</h1>
+
+<p><a href="{% relative_url_string schema.element_tree_url %}">Tree view</a> | <b>Alphabetical list</b></p>
+
+<ul>
+{% for object in objects %}
+<li><a href="{% relative_url_string object.get_absolute_url %}">{{ object.name }}</a></li>
+{% endfor %}
+</ul>
+{% endblock %}

docgenerator/spec/utils/datautils.py

@@ -1,5 +1,6 @@
-from spec.models import XMLElement, ExampleDocumentElement
+from spec.models import XMLElement, JSONObject, ExampleDocumentElement, ExampleDocumentObject
 import xml.sax
+import json
 
 ELEMENTS_TO_IGNORE = {'metadiff'}
 
@@ -49,6 +50,47 @@ def update_example_elements(example):
     in the given ExampleDocument (in which case they're
     deleted).
     """
+    if example.schema.is_json:
+        update_example_elements_json(example)
+    else:
+        update_example_elements_xml(example)
+
+def accumulate_used_json_objects(json_data, object_def):
+    """
+    Given JSON data and a JSONObject that describes what level of the
+    document tree we're in, returns a set of all JSONObjects used
+    within (recursively).
+    """
+    result = set()
+    if object_def.is_array():
+        child_object_defs = [c.child for c in object_def.get_child_relationships()]
+        for child_obj in json_data:
+            child_object_def = JSONObject.get_jsonobject_for_data(child_obj, child_object_defs)
+            result.update(accumulate_used_json_objects(child_obj, child_object_def))
+    else:
+        result.add(object_def)
+        for rel in object_def.get_child_relationships():
+            if rel.child_key in json_data:
+                result.update(accumulate_used_json_objects(json_data[rel.child_key], rel.child))
+    return result
+
+def update_example_elements_json(example):
+    schema = example.schema
+    root_object = JSONObject.objects.get(schema=schema, name=JSONObject.ROOT_OBJECT_NAME)
+    example_doc = json.loads(example.document)
+    seen_objects = accumulate_used_json_objects(example_doc, root_object)
+    for existing in ExampleDocumentObject.objects.filter(example=example):
+        if existing.json_object in seen_objects:
+            seen_objects.remove(existing.json_object)
+        else:
+            existing.delete()
+    for obj in seen_objects:
+        ExampleDocumentObject.objects.create(
+            example=example,
+            json_object=obj,
+        )
+
+def update_example_elements_xml(example):
     reader = xml.sax.make_parser()
     handler = ElementCollector(example.schema)
     xml.sax.parseString(example.document, handler)