Merge 0f5c355 into 28eb8c1

docs/reference/item.md

@@ -177,7 +177,40 @@ item of the reviewed attribute, not the parent item of the link).  The
 fingerprint of the link is does **not** contribute to the fingerprint of the
 item.
 
-## `ref`
+## `references` (new array behavior)
+
+An array of external references. An item may reference a number of external
+files. The external references are displayed in a published document.
+
+Doorstop will search in the project root for a file matching the specified
+reference. If multiple matching files exist, the first found will be used.
+
+The value of this attribute contributes to the [fingerprint](item.md#reviewed)
+of the item.
+
+### Example: Reference files
+
+```yaml
+references:
+- path: tests/test1.cpp
+  type: file
+- path: tests/test2.cpp
+  type: file
+```
+
+### Note: new behavior vs old behavior
+
+Note: Newer `references` attribute is a new array behavior compared to the 
+original `ref` attribute's string behavior. The old behavior supports referencing
+only one file via file name or referencing a file that contains a given
+keyword.
+
+The new behavior of `references` attribute allows referencing many files. It
+discards referencing files via keywords and only supports referencing files.
+
+## `ref` (deprecated behavior, see 'references')
+
+Please check the "`references` (new array behavior)" section before reading further.
 
 External reference. An item may reference an external file or a line in an
 external file. An external reference is displayed in a published document.

doorstop/core/exporter.py

@@ -131,20 +131,31 @@ def _tabulate(obj, sep=LIST_SEP, auto=False):
     :return: iterator of rows of data
 
     """
-    yield_header = True
+    # yield_header = True
+    header = ['level', 'text', 'ref', 'links']
 
+    at_least_one_ref = False
     for item in iter_items(obj):
-
         data = item.data
+        ref_value = data.get('ref')
+        if ref_value:
+            at_least_one_ref = True
+
+        for value in sorted(data.keys()):
+            if value not in header:
+                header.append(value)
+    try:
+        reference_index = header.index('references')
+        header.insert(3, header.pop(reference_index))
+        if not at_least_one_ref:
+            header.remove('ref')
+    except ValueError:
+        pass
 
-        # Yield header
-        if yield_header:
-            header = ['level', 'text', 'ref', 'links']
-            for value in sorted(data.keys()):
-                if value not in header:
-                    header.append(value)
-            yield ['uid'] + header
-            yield_header = False
+    yield ['uid'] + header
+
+    for item in iter_items(obj):
+        data = item.data
 
         # Yield row
         row = [item.uid]
@@ -156,6 +167,23 @@ def _tabulate(obj, sep=LIST_SEP, auto=False):
             elif key == 'links':
                 # separate identifiers with a delimiter
                 value = sep.join(uid.string for uid in item.links)
+            elif key == 'references':
+                if value is None:
+                    value = ''
+                else:
+                    ref_strings = []
+                    for ref_item in value:
+                        ref_type = ref_item['type']
+                        ref_path = ref_item['path']
+
+                        ref_string = "type:{},path:{}".format(ref_type, ref_path)
+
+                        if 'keyword' in ref_item:
+                            keyword = ref_item['keyword']
+                            ref_string += ",keyword:{}".format(keyword)
+
+                        ref_strings.append(ref_string)
+                    value = '\n'.join(ref_string for ref_string in ref_strings)
             elif isinstance(value, str) and key not in ('reviewed',):
                 # remove sentence boundaries and line wrapping
                 value = item.get(key)

doorstop/core/importer.py

@@ -254,6 +254,29 @@ def _itemize(header, data, document, mapping=None):
             elif key == 'links':
                 # split links into a list
                 attrs[key] = _split_list(value)
+
+            elif key == 'references' and (value is not None):
+                ref_items = value.split('\n')
+                if ref_items[0] != '':
+                    ref = []
+                    for ref_item in ref_items:
+                        ref_item_components = ref_item.split(',')
+                        ref_type = ref_item_components[0].split(':')[1]
+                        ref_path = ref_item_components[1].split(':')[1]
+
+                        if len(ref_item_components) == 2:
+                            ref.append({'type': ref_type, 'path': ref_path})
+                        else:
+                            ref_keyword = ref_item_components[2].split(':')[1]
+                            ref.append(
+                                {
+                                    'type': ref_type,
+                                    'path': ref_path,
+                                    'keyword': ref_keyword,
+                                }
+                            )
+
+                    attrs[key] = ref
             elif key == 'active':
                 # require explicit disabling
                 attrs['active'] = value is not False

doorstop/core/item.py

@@ -4,7 +4,6 @@
 
 import functools
 import os
-import re
 from typing import Any, List
 
 import pyficache
@@ -20,7 +19,9 @@
     delete_item,
     edit_item,
 )
+from doorstop.core.reference_finder import ReferenceFinder
 from doorstop.core.types import UID, Level, Prefix, Stamp, Text, to_bool
+from doorstop.core.yaml_validator import YamlValidator
 
 log = common.logger(__name__)
 
@@ -104,10 +105,12 @@ def __init__(self, document, path, root=os.getcwd(), **kwargs):
             raise DoorstopError(msg)
         # Initialize the item
         self.path = path
-        self.root = root
+        self.root: str = root
         self.document = document
         self.tree = kwargs.get('tree')
         self.auto = kwargs.get('auto', Item.auto)
+        self.reference_finder = ReferenceFinder()
+        self.yaml_validator = YamlValidator()
         # Set default values
         self._data['level'] = Item.DEFAULT_LEVEL
         self._data['active'] = Item.DEFAULT_ACTIVE
@@ -116,6 +119,7 @@ def __init__(self, document, path, root=os.getcwd(), **kwargs):
         self._data['reviewed'] = Item.DEFAULT_REVIEWED
         self._data['text'] = Item.DEFAULT_TEXT
         self._data['ref'] = Item.DEFAULT_REF
+        self._data['references'] = None
         self._data['links'] = set()
         if settings.ENABLE_HEADERS:
             self._data['header'] = Item.DEFAULT_HEADER
@@ -174,6 +178,7 @@ def new(
 
     def _set_attributes(self, attributes):
         """Set the item's attributes."""
+        self.yaml_validator.validate_item_yaml(attributes)
         for key, value in attributes.items():
             if key == 'level':
                 value = Level(value)
@@ -189,6 +194,23 @@ def _set_attributes(self, attributes):
                 value = Text(value)
             elif key == 'ref':
                 value = value.strip()
+            elif key == 'references':
+                if value is None:
+                    continue
+
+                stripped_value = []
+                for ref_dict in value:
+                    ref_type = ref_dict['type']
+                    ref_path = ref_dict['path']
+
+                    stripped_ref_dict = {"type": ref_type, "path": ref_path.strip()}
+                    if 'keyword' in ref_dict:
+                        ref_keyword = ref_dict['keyword']
+                        stripped_ref_dict['keyword'] = ref_keyword
+
+                    stripped_value.append(stripped_ref_dict)
+
+                value = stripped_value
             elif key == 'links':
                 value = set(UID(part) for part in value)
             elif key == 'header':
@@ -241,6 +263,25 @@ def _yaml_data(self):
                     value = ''
             elif key == 'ref':
                 value = value.strip()
+            elif key == 'references':
+                if value is None:
+                    continue
+                stripped_value = []
+                for el in value:
+                    if 'keyword' in el:
+                        stripped_value.append(
+                            {
+                                "type": "file",
+                                "path": el["path"].strip(),
+                                "keyword": el["keyword"].strip(),
+                            }
+                        )
+                    else:
+                        stripped_value.append(
+                            {"type": "file", "path": el["path"].strip()}
+                        )
+
+                value = stripped_value
             elif key == 'links':
                 value = [{str(i): i.stamp.yaml} for i in sorted(value)]
             elif key == 'reviewed':
@@ -439,6 +480,25 @@ def ref(self, value):
         """Set the item's external file reference."""
         self._data['ref'] = str(value) if value else ""
 
+    @property  # type: ignore
+    @auto_load
+    def references(self):
+        """Get the item's external file references.
+
+        An external reference can be part of a line in a text file or
+        the filename of any type of file.
+
+        """
+        return self._data['references']
+
+    @references.setter  # type: ignore
+    @auto_save
+    def references(self, value):
+        """Set the item's external file reference."""
+        if value is not None:
+            assert isinstance(value, list)
+        self._data['references'] = value
+
     @property  # type: ignore
     @auto_load
     def links(self):
@@ -576,31 +636,47 @@ def find_ref(self):
         if not settings.CACHE_PATHS:
             pyficache.clear_file_cache()
         # Search for the external reference
-        log.debug("seraching for ref '{}'...".format(self.ref))
-        pattern = r"(\b|\W){}(\b|\W)".format(re.escape(self.ref))
-        log.trace("regex: {}".format(pattern))  # type: ignore
-        regex = re.compile(pattern)
+        return self.reference_finder.find_ref(self.ref, self.tree, self.path)
+
+    @requires_tree
+    def find_references(self):
+        """Get the array of references. Check each references before returning.
+
+        :raises: :class:`~doorstop.common.DoorstopError` when no
+            reference is found
+
+        :return: relative path to file or None (when no reference
+            set),
+            line number (when found in file) or None (when found as
+            filename) or None (when no reference set)
+
+        """
+        if not self.references:
+            log.debug("no external reference to search for")
+            return []
+        if not settings.CACHE_PATHS:
+            pyficache.clear_file_cache()
+        for ref_item in self.references:
+            path = ref_item["path"]
+            keyword = ref_item["keyword"] if "keyword" in ref_item else None
+
+            self.reference_finder.check_file_reference(
+                path, self.root, self.tree, path, keyword
+            )
+        return self.references
+
+    def _find_external_file_ref(self, ref_path):
+        log.debug("searching for ref '{}'...".format(ref_path))
+        ref_full_path = os.path.join(self.root, ref_path)
+
         for path, filename, relpath in self.tree.vcs.paths:
             # Skip the item's file while searching
             if path == self.path:
                 continue
-            # Check for a matching filename
-            if filename == self.ref:
-                return relpath, None
-            # Skip extensions that should not be considered text
-            if os.path.splitext(filename)[-1] in settings.SKIP_EXTS:
-                continue
-            # Search for the reference in the file
-            lines = pyficache.getlines(path)
-            if lines is None:
-                log.trace("unable to read lines from: {}".format(path))  # type: ignore
-                continue
-            for lineno, line in enumerate(lines, start=1):
-                if regex.search(line):
-                    log.debug("found ref: {}".format(relpath))
-                    return relpath, lineno
+            if path == ref_full_path:
+                return True
 
-        msg = "external reference not found: {}".format(self.ref)
+        msg = "external reference not found: {}".format(ref_path)
         raise DoorstopError(msg)
 
     def find_child_links(self, find_all=True):
@@ -690,6 +766,10 @@ def find_child_items_and_documents(self, document=None, tree=None, find_all=True
     def stamp(self, links=False):
         """Hash the item's key content for later comparison."""
         values = [self.uid, self.text, self.ref]
+
+        if self.references:
+            values.append(self.references)
+
         if links:
             values.extend(self.links)
         for key in self.document.extended_reviewed: