Merge pull request #566 from ckolumbus/feat/yaml_frontmatter_format

YAML frontmatter format

docs/reference/document.md

@@ -18,6 +18,7 @@ Then open `REQ/.doorstop.yml` in any text editor:
 settings:
   digits: 3
   prefix: REQ
+  itemformat: yaml
   sep: ''
 ```
 
@@ -49,6 +50,12 @@ The default value is the empty string.  You have to set it manually before an
 item is created.  Afterwards, it should be considered as read-only.  This
 document setting is mandatory.
 
+### `itemformat`
+
+Requirement items can be stored in different file formats. The two types
+currently supported are `yaml` and `markdown`. See the [item](item.md)
+documentation for more details. The default format is `yaml`.
+
 # Extended Document Attributes
 
 In addition to the standard attributes, Doorstop will allow any number of custom

docs/reference/item.md

@@ -10,7 +10,24 @@ default, the number is automatically assigned by Doorstop. Optionally, a user
 can specify a name for the UID during item creation. The name must not contain
 separator characters.
 
-Example item:
+Items can be stored in different formats. Two types are currently supported and
+can be configured via [itemformat](document.md#itemformat) in the
+`.doorstop.yml` configuration file:
+
+The default `yaml` format stores the requirement items completely as yaml.
+Advantage is the standardized and easily parsable format in case the files
+need to be processed furher. The disadvantage is that there is little editor
+support when writing longer requirement texts, as the text need to adhere to the
+yaml formatting rules (e.g. indentation).
+
+The alternative `markdown` format stores requirement items as markdown text with
+yaml frontmatter. This concept is used by many other tools as well (e.g. 
+[jekyll](https://jekyllrb.com/docs/front-matter/)). With this format, the
+`header` attribute is automatically derived from the first *level 1* header 
+line within the text, the remaining content of the markdown text section 
+is used as `text` attribute.
+
+Example item in `yaml` format: `REQ_000.yml`
 
 ```yaml
 active: true
@@ -27,6 +44,32 @@ text: |
   sections of text.
 ```
 
+The equivalent item in `markdown` format: `REQ_000.md`
+
+```text
+---
+active: true
+derived: false
+level: 2.1
+links: []
+normative: true
+ref: ''
+reviewed: 9TcFUzsQWUHhoh5wsqnhL7VRtSqMaIhrCXg7mfIkxKM=
+---
+
+# Identifiers
+
+Doorstop **shall** provide unique and permanent identifiers to linkable 
+sections of text.
+```
+
+Regarding the documentation below both formats are equivalent, besides the
+specific `header` and `text` handling as shown above. The decision can be based
+on whether the better text editing support of the `markdown` format or the
+consistent yaml structure is more important. The formats can also be freely 
+mixed within a [document tree](tree.md), the decision can be made and configured
+per document.
+
 # Standard Item Attributes
 
 ## `active`
@@ -398,7 +441,7 @@ reviewed attributes to the document configuration does not change the
 fingerprint of existing items of the document, if they do not have them,
 otherwise the fingerprint changes.  Removing a reviewed attribute from the
 document configuration changes the fingerprint of all items of the document
-with such an attribute.  The type of the values, empty strings, lists, and
+with such an attribute. The type of the values, empty strings, lists, and
 dictionaries affect the fingerprint.
 
 ## Publishing extended attributes

doorstop/cli/commands.py

@@ -127,6 +127,7 @@ def run_create(args, cwd, _, catch=True):
             parent=args.parent,
             digits=args.digits,
             sep=args.separator,
+            itemformat=args.itemformat,
         )
 
     if not success:

doorstop/cli/main.py

@@ -197,6 +197,13 @@ def _create(subs, shared):
     sub.add_argument("prefix", help="document prefix for new item UIDs")
     sub.add_argument("path", help="path to a directory for item files")
     sub.add_argument("-p", "--parent", help="prefix of parent document")
+    sub.add_argument(
+        "-i",
+        "--itemformat",
+        choices=["yaml", "markdown"],
+        default="yaml",
+        help="item file format",
+    )
     sub.add_argument(
         "-d",
         "--digits",

doorstop/common.py

@@ -6,10 +6,13 @@
 import codecs
 import csv
 import glob
+import io
 import logging
 import os
+import re
 import shutil
 
+import frontmatter
 import yaml
 
 verbosity = 0  # global verbosity setting for controlling string formatting
@@ -143,6 +146,33 @@ def load_yaml(text, path, loader=yaml.SafeLoader):
     return data
 
 
+def load_markdown(text, path, textattributekeys):
+    """Parse a dictionary from Markdown file with YAML frontmatter.
+
+    :param text: string containing markdown data with yaml frontmatter
+    :param path: file path for error messages
+
+    :return: dictionary
+
+    """
+    # Load YAML-frontmatter data from text
+    try:
+        data, content = frontmatter.parse(text, handler=frontmatter.YAMLHandler())
+    except yaml.error.YAMLError as exc:
+        msg = "invalid yaml contents: {}:\n{}".format(path, exc)
+        raise DoorstopError(msg) from None
+    # Ensure data is a dictionary
+    if not isinstance(data, dict):
+        msg = "invalid contents: {}".format(path)
+        raise DoorstopError(msg)
+
+    # parse content and update data dictionariy accordingly
+    update_data_from_markdown_content(data, content, textattributekeys)
+
+    # Return the parsed data
+    return data
+
+
 def write_lines(lines, path, end="\n", encoding="utf-8", *, executable=False):
     """Write lines of text to a file.
 
@@ -258,3 +288,59 @@ def delete_contents(dirname):
                     "Two assets folders have files or directories " "with the same name"
                 )
                 raise
+
+
+def update_data_from_markdown_content(data, content, textattributekeys):
+    """Update data dictionary based on text content and attribute keys to look for within the content."""
+    h1 = re.compile(r"^#{1}\s+(.*)")
+    # for line based iteration
+    s = io.StringIO(content)
+    # final text
+    header = None
+    text = ""
+
+    if "header" in textattributekeys:
+        # search for first content line and check
+        # if it is a h1 header
+        for l in s:
+            # skip empty lines
+            if len(l.strip()) == 0:
+                continue
+            # check if first found line is a header
+            m = h1.match(l.strip())
+            if m:
+                # header found
+                header = m.group(1)
+            else:
+                # no header found, add to normal text
+                text += l
+            break
+
+        # if header was found, skip empty lines before main text
+        if header:
+            for l in s:
+                if len(l.strip()) != 0:
+                    text += l
+                    break
+
+    # remaining content is normal text
+    for l in s:
+        text += l
+
+    if "header" in textattributekeys and header:
+        data["header"] = header
+
+    if "text" in textattributekeys:
+        data["text"] = text
+
+
+def dump_markdown(data, textattr):
+    content = ""
+    if "header" in textattr and textattr["header"].strip() != "":
+        content += "# {}\n".format(textattr["header"].strip())
+        content += "\n"
+
+    content += textattr["text"]
+
+    text = frontmatter.dumps(frontmatter.Post(content, **data))
+    return text

doorstop/core/document.py

@@ -65,11 +65,15 @@ def __init__(self, path, root=os.getcwd(), **kwargs):
         self._data["sep"] = Document.DEFAULT_SEP
         self._data["digits"] = Document.DEFAULT_DIGITS  # type: ignore
         self._data["parent"] = None  # type: ignore
+        self._data["itemformat"] = kwargs.get("itemformat")  # type: ignore
         self._extended_reviewed: List[str] = []
         self._items: List[Item] = []
         self._itered = False
         self.children: List[Document] = []
 
+        if not self._data["itemformat"]:
+            self._data["itemformat"] = Item.DEFAULT_ITEMFORMAT
+
     def __repr__(self):
         return "Document('{}')".format(self.path)
 
@@ -92,7 +96,15 @@ def __bool__(self):
     @staticmethod
     @add_document
     def new(
-        tree, path, root, prefix, sep=None, digits=None, parent=None, auto=None
+        tree,
+        path,
+        root,
+        prefix,
+        sep=None,
+        digits=None,
+        parent=None,
+        auto=None,
+        itemformat=None,
     ):  # pylint: disable=R0913,C0301
         """Create a new document.
 
@@ -107,6 +119,8 @@ def new(
         :param parent: parent UID for the new document
         :param auto: automatically save the document
 
+        :param itemformat: file format for storing items
+
         :raises: :class:`~doorstop.common.DoorstopError` if the document
             already exists
 
@@ -127,7 +141,9 @@ def new(
         Document._create(config, name="document")
 
         # Initialize the document
-        document = Document(path, root=root, tree=tree, auto=False)
+        document = Document(
+            path, root=root, tree=tree, auto=False, itemformat=itemformat
+        )
         document.prefix = (  # type: ignore
             prefix if prefix is not None else document.prefix
         )
@@ -186,6 +202,8 @@ def load(self, reload=False):
                     self._data[key] = value.strip()
                 elif key == "digits":
                     self._data[key] = int(value)  # type: ignore
+                elif key == "itemformat":
+                    self._data[key] = value.strip()
                 else:
                     msg = "unexpected document setting '{}' in: {}".format(
                         key, self.config
@@ -267,7 +285,13 @@ def _iter(self, reload=False):
             for filename in filenames:
                 path = os.path.join(dirpath, filename)
                 try:
-                    item = Item(self, path, root=self.root, tree=self.tree)
+                    item = Item(
+                        self,
+                        path,
+                        root=self.root,
+                        tree=self.tree,
+                        itemformat=self.itemformat,
+                    )
                 except DoorstopError:
                     pass  # skip non-item files
                 else:
@@ -378,6 +402,11 @@ def parent(self, value):
         """Set the document's parent document prefix."""
         self._data["parent"] = str(value) if value else ""
 
+    @property
+    def itemformat(self):
+        """Get storage format for item files."""
+        return self._data["itemformat"]
+
     @property
     def items(self):
         """Get an ordered list of active items in the document."""