Merge pull request #485 from stanislaw/refs-utf8-keyword

Item#references: support keywords with UTF8 symbols

doorstop/core/item.py

@@ -3,11 +3,10 @@
 """Representation of an item in a document."""
 
 import functools
+import linecache
 import os
 from typing import Any, List
 
-import pyficache
-
 from doorstop import common, settings
 from doorstop.common import DoorstopError
 from doorstop.core import editor
@@ -633,7 +632,7 @@ def find_ref(self):
             return None, None
         # Update the cache
         if not settings.CACHE_PATHS:
-            pyficache.clear_file_cache()
+            linecache.clearcache()
         # Search for the external reference
         return self.reference_finder.find_ref(self.ref, self.tree, self.path)
 
@@ -657,7 +656,7 @@ def find_references(self):
             log.debug("no external reference to search for")
             return []
         if not settings.CACHE_PATHS:
-            pyficache.clear_file_cache()
+            linecache.clearcache()
 
         references = []
         for ref_item in self.references:

doorstop/core/reference_finder.py

@@ -2,11 +2,10 @@
 
 """Finding external references."""
 
+import linecache
 import os
 import re
 
-import pyficache
-
 from doorstop import common, settings
 from doorstop.common import DoorstopError
 
@@ -46,8 +45,9 @@ def find_ref(ref, tree, item_path):
             if os.path.splitext(filename)[-1] in settings.SKIP_EXTS:
                 continue
             # Search for the reference in the file
-            lines = pyficache.getlines(path)
-            if lines is None:
+            try:
+                lines = linecache.getlines(path)
+            except SyntaxError:
                 log.trace("unable to read lines from: {}".format(path))  # type: ignore
                 continue
             for lineno, line in enumerate(lines, start=1):
@@ -81,11 +81,10 @@ def find_file_reference(ref_path, root, tree, item_path, keyword=None):
                     return relpath, None
 
                 # Search for the reference in the file
-                lines = pyficache.getlines(path)
-                if lines is None:
-                    log.trace(  # type: ignore
-                        "unable to read lines from: {}".format(path)
-                    )  # type: ignore
+                try:
+                    lines = linecache.getlines(path)
+                except SyntaxError:
+                    log.trace("unable to read lines from: {}".format(path))  # type: ignore
                     continue
 
                 log.debug("searching for ref '{}'...".format(keyword))

doorstop/core/tests/test_fixtures/001-item-references-utf8-keyword/REQ-UTF8.yml

@@ -0,0 +1,15 @@
+active: true
+derived: false
+header: ''
+level: 1.5
+links:
+- REQ001: 35ed54323e3054c33ae5545fffdbbbf5
+normative: true
+ref: ''
+references:
+- keyword: français
+  path: NOT-RELEVANT-FOR-THIS-TEST.txt
+  type: file
+reviewed: c442316131ca0225595ae257f3b4583d
+text: |
+  Hello, world!

doorstop/core/tests/test_fixtures/001-item-references-utf8-keyword/files/text-utf8.txt

@@ -0,0 +1 @@
+français

doorstop/core/tests/test_reference_finder.py

@@ -56,6 +56,26 @@ def test_find_file_reference_with_keyword(self):
         self.assertEqual(path, os.path.join('files', 'REQ006.yml'))
         self.assertEqual(line, 10)
 
+    def test_find_file_reference_with_utf8_keyword(self):
+        reference_path = "test_fixtures/001-item-references-utf8-keyword/REQ-UTF8.yml"
+        root = TESTS_ROOT
+        tree = Mock()
+        tree.vcs = WorkingCopy(TESTS_ROOT)
+        item_path = "NOT-RELEVANT-FOR-THE-TEST"
+
+        reference_finder = ReferenceFinder()
+        path, line = reference_finder.find_file_reference(
+            reference_path, root, tree, item_path, 'français'
+        )
+
+        self.assertEqual(
+            path,
+            os.path.join(
+                'test_fixtures', '001-item-references-utf8-keyword', 'REQ-UTF8.yml'
+            ),
+        )
+        self.assertEqual(line, 10)
+
     def test_find_file_reference_should_skip_item_path(self):
         root = TESTS_ROOT
         tree = Mock()