Merge cac1bc0 into e119c09

core/sile.lua

@@ -276,9 +276,9 @@ SILE.process = function (ast)
       content()
     elseif SILE.Commands[content.command] then
       SILE.call(content.command, content.options, content)
-    elseif content.id == "texlike_stuff"
+    elseif content.id == "content"
       or (not content.command and not content.id) then
-      local pId = SILE.traceStack:pushContent(content, "texlike_stuff")
+      local pId = SILE.traceStack:pushContent(content, "content")
       SILE.process(content)
       SILE.traceStack:pop(pId)
     elseif type(content) ~= "nil" then

core/utilities/init.lua

@@ -153,7 +153,7 @@ utilities.debugAST = function (ast, level)
             return out .. "\\" .. content.command .. " " .. pl.pretty.write(content.options, "")
           end)
           if (#content>=1) then utilities.debugAST(content, level+1) end
-        elseif content.id == "texlike_stuff" or (not content.command and not content.id) then
+        elseif content.id == "content" or (not content.command and not content.id) then
           utilities.debugAST(content, level+1)
         else
           SU.debug("ast", function ()

documentation/c03-input.sil

@@ -186,7 +186,7 @@ The \code{\\begin} command at the start of the document is an example of this.%
 \end{raw}
 
 The parameters to a command are enclosed in square brackets and take the form \code{\em{key}=\em{value}};
-    multiple parameters are separated by commas or semicolons, as in \code{[key1=value1,key2=value2,…]}.
+    multiple parameters are separated by commas, as in \code{[key1=value1,key2=value2,…]}.
 Spaces around the keys are not significant;
     we could equally write that as \code{[key1 = value1; key2 = value2; …]}.
 If you need to include a comma or semicolon within the value to a parameter, you can enclose the value in quotes:

inputters/sil.lua

@@ -62,7 +62,7 @@ function inputter._grammar (_ENV)
     return str:gsub('\\([{}%%\\])', '%1')
   end
   local myID = C(bits.silidentifier) / 1
-  local cmdID = myID - P"beign" - P"end"
+  local cmdID = myID - P"begin" - P"end"
   local wrapper = function (a) return type(a)=="table" and a or {} end
   local parameters = (P"[" * bits.parameters * P"]")^-1 / wrapper
   local comment = (
@@ -72,34 +72,34 @@ function inputter._grammar (_ENV)
     ) / ""
 
   START "document"
-  document = V"texlike_stuff" * EOF"Unexpected character at end of input"
-  texlike_stuff = Cg(
+  document = V"content" * EOF"Unexpected character at end of input"
+  content = Cg(
       V"environment" +
       comment +
-      V"texlike_text" +
-      V"texlike_braced_stuff" +
-      V"texlike_command"
+      V"text" +
+      V"braced_content" +
+      V"command"
     )^0
-  passthrough_stuff = C(Cg(
+  passthrough_content = C(Cg(
       V"passthrough_text" +
-      V"passthrough_debraced_stuff"
+      V"debraced_passthrough_text"
     )^0)
-  passthrough_env_stuff = Cg(
-      V"passthrough_env_text"
+  env_passthrough_content = Cg(
+      V"env_passthrough_text"
     )^0
-  texlike_text = C((1 - specials + escaped_specials)^1) / unescapeSpecials
+  text = C((1 - specials + escaped_specials)^1) / unescapeSpecials
   passthrough_text = C((1-S("{}"))^1)
-  passthrough_env_text = C((1 - (P"\\end{" * Cmt(cmdID * Cb"command", isMatchingEndEnv) * P"}"))^1)
-  texlike_braced_stuff = P"{" * V"texlike_stuff" * ( P"}" + E("} expected") )
-  passthrough_braced_stuff = P"{" * V"passthrough_stuff" * ( P"}" + E("} expected") )
-  passthrough_debraced_stuff = C(V"passthrough_braced_stuff")
-  texlike_command = (
+  env_passthrough_text = C((1 - (P"\\end{" * Cmt(cmdID * Cb"command", isMatchingEndEnv) * P"}"))^1)
+  braced_content = P"{" * V"content" * ( P"}" + E("} expected") )
+  braced_passthrough_content = P"{" * V"passthrough_content" * ( P"}" + E("} expected") )
+  debraced_passthrough_text = C(V"braced_passthrough_content")
+  command = (
       P"\\" *
       Cg(cmdID, "command") *
       Cg(parameters, "options") *
       (
-        (Cmt(Cb"command", isPassthrough) * V"passthrough_braced_stuff") +
-        (Cmt(Cb"command", isNotPassthrough) * V"texlike_braced_stuff")
+        (Cmt(Cb"command", isPassthrough) * V"braced_passthrough_content") +
+        (Cmt(Cb"command", isNotPassthrough) * V"braced_content")
       )^0
     )
   local notpass_end =
@@ -117,8 +117,8 @@ function inputter._grammar (_ENV)
     Cg(cmdID, "command") *
     P"}" *
     (
-      (Cmt(Cb"command", isPassthrough) * V"passthrough_env_stuff" * pass_end) +
-      (Cmt(Cb"command", isNotPassthrough) * V"texlike_stuff" * notpass_end)
+      (Cmt(Cb"command", isPassthrough) * V"env_passthrough_content" * pass_end) +
+      (Cmt(Cb"command", isNotPassthrough) * V"content" * notpass_end)
     )
 end
 -- luacheck: pop
@@ -168,21 +168,21 @@ local function massage_ast (tree, doc)
     tree.lno, tree.col = getline(doc, tree.pos)
   end
   if tree.id == "document"
-      or tree.id == "texlike_braced_stuff"
-      or tree.id == "passthrough_stuff"
-      or tree.id == "passthrough_braced_stuff"
-      or tree.id == "passthrough_env_stuff"
+      or tree.id == "braced_content"
+      or tree.id == "passthrough_content"
+      or tree.id == "braced_passthrough_content"
+      or tree.id == "env_passthrough_content"
     then
       return massage_ast(tree[1], doc)
   end
-  if tree.id == "texlike_text"
+  if tree.id == "text"
     or tree.id == "passthrough_text"
-    or tree.id == "passthrough_env_text"
+    or tree.id == "env_passthrough_text"
     then
       return tree[1]
   end
   for key, val in ipairs(tree) do
-    if val.id == "texlike_stuff" then
+    if val.id == "content" then
       SU.splice(tree, key, key, massage_ast(val, doc))
     else
       tree[key] = massage_ast(val, doc)

packages/autodoc/init.lua

@@ -89,10 +89,10 @@ local function typesetAST (options, content)
       else
         seenCommandWithoutArg = true
       end
-    elseif ast.id == "texlike_stuff" or (not ast.command and not ast.id) then
+    elseif ast.id == "content" or (not ast.command and not ast.id) then
       -- Due to the way it is implemented, the SILE-inputter may generate such
       -- nodes in the AST. It's poorly documented, so it's not clear why they
-      -- are even kept there (esp. the "texlike_stuff" nodes), but anyhow, as
+      -- are even kept there (esp. the "content" nodes), but anyhow, as
       -- far as autodoc is concerned for presentation purposes, just
       -- recurse into them.
       typesetAST(options, ast)

packages/math/texlike.lua

@@ -86,8 +86,8 @@ local mathGrammar = function (_ENV)
         return pl.utils.unpack(t)
       end
 
-  START "texlike_math"
-  texlike_math = V"mathlist" * EOF"Unexpected character at end of math code"
+  START "math"
+  math = V"mathlist" * EOF"Unexpected character at end of math code"
   mathlist = (comment + (WS * _) + element)^0
   supsub = element_no_infix * _ * P"^" * _ * element_no_infix * _ *
     P"_" * _ * element_no_infix
@@ -254,7 +254,7 @@ local function compileToMathML_aux (_, arg_env, tree)
     return accumulator
   end
   tree = fold_pairs(compile_and_insert, tree)
-  if tree.id == "texlike_math" then
+  if tree.id == "math" then
     tree.command = "math"
     -- If the outermost `mrow` contains only other `mrow`s, remove it
     -- (allowing vertical stacking).

sil.abnf

@@ -0,0 +1,71 @@
+; Formal grammar definition for SIL (SILE Input Language) files
+;
+; Based on RFC 5234 (Augmented BNF for Syntax Specifications: ABNF)
+; Uses RFC 7405 (Case-Sensitive String Support in ABNF)
+
+; A master document can only have one top level content item, but we allow
+; loading of fragments as well which can have any number of top level content
+; items, hence valid grammar can be any number of content items.
+document = *content
+
+; Top level content can be any sequence of these things
+content =  environment
+content =/ comment
+content =/ text
+content =/ braced_content
+content =/ command
+
+; Environments come in two flavors, raw passthrough and regular. The difference
+; is what is allowed to terminate them and what escapes are needed for the
+; content in the middle.
+environment =  %s"\begin" [ options ] "{" command-id "}" content %s"\end{" command-id "}"
+environment =/ %s"\begin" [ options ] "{" passthrough-command-id "}" passthrough-content %s"\end{" passthrough-command-id "}"
+
+; Nothing to see here.
+comment = "%" utf8-octets CRLF
+
+; Input strings that are not wrapped in some SILE command.
+text = *text-char
+
+; Input content wrapped in braces can be attatched to a command or just used to
+; create a settings wrapper.
+braced_content = "{" content "}"
+
+; As with environments, can be raw passthrough or regular content depending on the command. With raw variants we require contents.
+command =  "\" command-id [ options ] [ braced_content ]
+command =/ "\" passthrough-command-id [ options ] braced-passthrough-content
+
+; Building blocks
+options = "[" parameter *( "," parameter ) "]"
+parameter = *1( sil-identifier "=" value)
+value =
+quoted-value = DQUOTE *( *SP / value / "," ) DQUOTE
+s = *WS
+any = %x20-7F / valid-non-ascii / tab
+
+escaped-char = "\\" / "\%" / "\{" / "\}"
+
+; non-ascii-char = %x80-D7FF / %xE000-10FFFF
+text-char =  %x0-24  ; omit %
+text-char =  %x26-5B ; omit \
+text-char =/ %x5D-7A ; omit {
+text-char =/ %x7C    ; omit }
+text-char =/ %x7E-7F ; end of utf8-1
+text-char =/ utf8-2
+text-char =/ utf8-3
+text-char =/ utf8-4
+
+letters = ALPHA / "_"
+identifier = letters *( letters / DIGIT )
+sil-identifier = 1*( identifier / ":" / "-" )
+command-id = sil-identifier - %s"begin" - %s"end" - passthrough-command-id
+passthrough-command-id = %s"ftl" / %s"lua" / %s"math" / %s"raw" / %s"script" / %s"sil" / %s"use" / %s"xml"
+
+; ASCII isn't good enough for us.
+utf8-octets = *utf8-char
+utf8-char   = utf8-1 / utf8-2 / utf8-3 / utf8-4
+utf8-1      = %x00-7F
+utf8-2      = %xC2-DF utf8-tail
+utf8-3      = %xE0 %xA0-BF utf8-tail / %xE1-EC 2utf8-tail / %xED %x80-9F utf8-tail / %xEE-EF 2utf8-tail
+utf8-4      = %xF0 %x90-BF 2utf8-tail / %xF1-F3 3utf8-tail / %xF4 %x80-8F 2utf8-tail
+utf8-tail   = %x80-BF