l3build: Improve normalisation for chars 0/127/28-255

For ^^@ and ^^?, LuaTeX needs some work to get into line with
other engines.

For the top-half of the 8-bit range, for pdfTeX/XeTeX/LuaTeX
it makes sense to normalise to ^^<n> format, done using .tcx
file for pdfTeX and a search for UTF-8 chars for XeTeX/LuaTeX.
(This will be useful for LaTeX2e testing.) For (u)pTeX we seem
to be out of luck.

.gitattributes

@@ -1,7 +1,6 @@
 * text=auto !eol
 l3experimental/xcoffins/elementare-typographie-title.jpg -text svneol=unset#image/jpeg
 l3kernel/expl3-datatype-analysis.xlsx -text
-l3kernel/testfiles/m3char001.luatex.tlg -text
 xpackages/galley/make.bat -text
 xpackages/make.bat -text
 xpackages/xcontents/Makefile -text

l3build/l3build.dtx

@@ -654,6 +654,16 @@
 %   \item Conversion of low chars ($1$ to $31$) to |^^| notation.
 % \end{itemize}
 %
+% When making comparisons between 8-bit and Unicode engines it is useful to
+% format the top half of the 8-bit range such that it appears in the log as
+% |^^|\texttt{\meta{char}} (the exact nature of the 8-bit output is otherwise
+% dependent on the active code page). This may be controlled using the
+% |asciiengines| option. Any engines named here will use a |.tcx| file to
+% produce only ASCII chars in the log output, whilst for other engines
+% normalisation is carried out from UTF-8 to ASCII. If the option is set to
+% an empty table the latter process is skipped: suitable for cases where only
+% Unicode engines are in use.
+%
 % \section{Writing test files}
 % \label{sec:writing-tests}
 %

l3build/l3build.lua

@@ -146,6 +146,7 @@ makeindexexe  = makeindexexe  or "makeindex"
 makeindexopts = makeindexopts or ""
 
 -- Other required settings
+asciiengines = asciiengines or {"etex", "pdftex"}
 checkruns    = checkruns    or 1
 packtdszip   = packtdszip   or false -- Not actually needed but clearer
 scriptname   = scriptname   or "build.lua" -- Script used in each directory
@@ -373,6 +374,7 @@ end
 -- both Windows and Unix cases: more complex situations are handled inside
 -- the support functions
 if os.type == "windows" then
+  os_ascii    = "@echo."
   os_concat   = "&"
   os_diffext  = os.getenv("diffext") or ".fc"
   os_diffexe  = os.getenv("diffexe") or "fc /n"
@@ -384,6 +386,7 @@ if os.type == "windows" then
   os_windows  = true
   os_yes      = "for /l %I in (1,1,200) do @echo y"
 else
+  os_ascii    = "echo \"\""
   os_concat   = ";"
   os_diffext  = os.getenv("diffext") or ".diff"
   os_diffexe  = os.getenv("diffexe") or "diff -c --strip-trailing-cr"
@@ -598,6 +601,7 @@ function checkinit()
   for _,i in ipairs(checksuppfiles) do
     cp(i, supportdir, testdir)
   end
+  os.execute(os_ascii .. ">" .. testdir .. "/ascii.tcx")
 end
 
 -- Copy files to the main CTAN release directory
@@ -736,7 +740,7 @@ function formatlog(logfile, newfile, engine)
     -- Zap "on line <num>" and replace with "on line ..."
     line = string.gsub(line, "on line %d*", "on line ...")
     -- Tidy up to ^^ notation
-    for i = 1, 31, 1 do
+    for i = 0, 31 do
       line = string.gsub(line, string.char(i), "^^" .. string.char(64 + i))
     end
     -- Zap line numbers from \show, \showbox, \box_show and the like
@@ -748,11 +752,15 @@ function formatlog(logfile, newfile, engine)
     line = string.gsub(line, "^%s+", "")
     -- Remove 'normal' direction information on boxes with (u)pTeX
     line = string.gsub(line, ",? yoko direction,?", "")
+    -- A tidy-up to keep LuaTeX and other engines in sync
+    local utf8_char = unicode.utf8.char
+    line = string.gsub(line, utf8_char(127), "^^?")
     -- Unicode engines display chars in the upper half of the 8-bit range:
-    -- tidy up to match pdfTeX
-    local utf8 = unicode.utf8
-    for i = 128, 255, 1 do
-      line = string.gsub(line, utf8.char(i), "^^" .. string.format("%02x", i))
+    -- tidy up to match pdfTeX if an ASCII engine is in use
+    if next(asciiengines) then
+      for i = 128, 255 do
+        line = string.gsub(line, utf8_char(i), "^^" .. string.format("%02x", i))
+      end
     end
     return line
   end
@@ -998,14 +1006,22 @@ function runtest(name, engine, hide, ext)
   end
   local logfile = testdir .. "/" .. name .. logext
   local newfile = testdir .. "/" .. name .. "." .. engine .. logext
+  local asciiopt = ""
+  for _,i in ipairs(asciiengines) do
+    if realengine == i then
+      asciiopt = "-translate-file ./ascii.tcx "
+      break
+    end
+  end
   for i = 1, checkruns do
     run(
         testdir,
         -- No use of localdir here as the files get copied to testdir:
         -- avoids any paths in the logs
         os_setenv .. " TEXINPUTS=." .. (checksearch and os_pathsep or "")
           .. os_concat ..
-        realengine ..  format .. " " .. checkopts .. " " .. lvtfile
+        realengine ..  format .. " " 
+          .. checkopts .. " " .. asciiopt .. lvtfile
           .. (hide and (" > " .. os_null) or "")
       )
   end