Merge pull request #1949 from Omikhleia/fix-french-u00a0-nbsp

languages/fr.lua

@@ -176,6 +176,7 @@ function SILE.nodeMakers.fr:mustRemove (i, items)
     if i < #items then
       local next = items[i+1].text
       if self:isSpace(next)
+          or self:isNonBreakingSpace(next)
           or self:isHighPunctuation(next)
           or self:isColonPunctuation(next)
           or self:isClosingQuote(next) then
@@ -223,9 +224,13 @@ function SILE.nodeMakers.fr:iterator (items)
   local removed = 0
   for k = 1, #items do
     if self:mustRemove(k, items) then
-      removed = removed + 1
+      -- the index is actually a character position in the byte stream.
+      -- So we need to take its actual byte length into account.
+      -- For instance, U+00A0 NBSP is 2 bytes long (0xC2 0xA0) in UTF-8.
+      removed = removed + string.len(items[k].text)
     else
-      items[k].index = items[k].index - removed -- index has changed due to removals
+      -- index has changed due to removals
+      items[k].index = items[k].index - removed
       table.insert(cleanItems, items[k])
     end
   end

tests/feat-unicode-nbsp.expected

@@ -155,19 +155,13 @@ T	41 85 68 81 111 68 76 86 w=34.3799	(Français)
 Mx 	50.5831
 T	34 w=4.3311	(?)
 Mx 	57.5711
-T	41 85 68 81 111 68 76 w=30.5176	(Françai)
-Mx 	88.0887
-T	86 w=3.8623	(s)
+T	41 85 68 81 111 68 76 86 w=34.3799	(Français)
 Mx 	94.6080
 T	29 w=2.2900	(:)
 Mx 	99.5550
 T	169 w=4.8535	(«)
 Mx 	106.5269
-T	41 85 68 81 111 w=23.2178	(Franç)
-Mx 	129.7447
-T	68 w=4.5898	(a)
-Mx 	134.3345
-T	76 86 w=6.5723	(is)
+T	41 85 68 81 111 68 76 86 w=34.3799	(Français)
 Mx 	142.2281
 T	4 w=2.7197	(!)
 Mx 	147.0662