Add support for XML export

When export is enabled, e.g., using \setupbackend[export=yes] then \definevimtyping[CODE][syntax=..., alternative=colorscheme] \startCODE ... \stopextCODE is exported as <vimtyping detail="colorscheme"> ... </vimtyping> And, \inlineCODE{...} is exported as <inlinevimtyping detail="colorscheme">...</inlinevimtyping> Each line is wrapped inside a `<verbatimline>...</verbatimline>` and each syntax group is wrapped inside `<syntaxgroup detail="vim....">...</syntaxgroup>`. See the updated documentation for details. The module ships with a default CSS file which can be included as \setupexport[cssfile=\vimtypingcssfile] This CSS file tries to mimic the default PDF output. If you want to tweak the output then it is recommended that you copy the default CSS file and modify it as appropriate.
adityam · Jun 30, 2020 · c113e55 · c113e55
1 parent e65625c
commit c113e55
Show file tree

Hide file tree

Showing 13 changed files with 806 additions and 40 deletions.
diff --git a/README.md b/README.md
@@ -52,6 +52,7 @@ Table of Contents
 * [Processing remote files](#processing-remote-files)
 * [Processing existing buffers](#processing-existing-buffers)
 * [Prepend and append text](#prepend-and-append-text)
+* [XML export](#xml-export)
 * [Special use case:  \write18 with caching](#special-use-case--write18-with-caching)
 * [Dealing with expansion](#dealing-with-expansion)
 * [Limitations](#limitations)
@@ -754,6 +755,43 @@ Then you can use
 
 to get a chess board.
 
+XML export
+----------
+
+The filter module provides a basic support for XML export. If the
+user-document contains
+
+    \setupbackend[export=yes]
+
+or other valid options to `export` such as `export=xml`, then the filter
+environment is exported as well. For example, both
+
+    \startmarkdown
+      ...
+    \stopmarkdown
+
+and
+    \processmarkdownfile{...}
+
+are exported (in `\jobname-export/\jobname.xml`) as
+
+    <externalfilter detail="markdown">
+      ...
+    </externalfilter>
+
+Moreover, 
+
+    \inlinemarkdown{...}
+
+is exported as 
+
+    <inlineexternalfilter detail="markdown">....</inlineexternalfilter>
+
+
+Note that it is the responsibility of the filter to generate properly tagged
+content. For example, `pandoc` used in the `markdown` examples in this
+document converts `**bold**` to `{\bf bold}`, which is not exported. 
+
 Special use case:  `\write18` with caching
 ------------------------------------------
 

diff --git a/Rakefile b/Rakefile
@@ -1,15 +1,6 @@
 require 'fileutils'
 require 'rake/clean'
 
-def make_pandoc
-  File.open('t-pandoc.tex', 'w') do |file|
-    file.puts "\\startmodule [pandoc]"
-    file.puts "\\let\\strong\\bold"
-    file.puts "\\let\\emph\\italic"
-    file.puts "\\stopmodule"
-  end
-end
-
 def make_tds(name, files, doc)
   tex_dir = "#{name}/tex/context/third/#{name}"
   doc_dir = "#{name}/doc/context/third/#{name}"
@@ -30,10 +21,9 @@ def make_zip name
 end
 
 def run_tests tests, engine
-  make_pandoc
   FileUtils.mkdir_p "output"
   tests.each do |file|
-    sh "context --#{engine} --usemodule=pandoc --color --mode=vim-dev --noconsole --purgeall --purgeresult #{file}"
+    sh "context --#{engine} --color --mode=vim-dev --noconsole --purgeall --purgeresult #{file}"
   end
   sh "context --purgeall"
 end
@@ -42,7 +32,7 @@ FILTER_TEX  = %W[t-filter.mkii t-filter.mkiv t-module-catcodes.mkii t-module-cat
 FILTER_DOC  = "README.md"
 FILTER_TEST = FileList['tests/[0-9][0-9]-*.tex']
 
-VIM_TEX  = %W[t-vim.tex t-syntax-groups.tex t-syntax-highlight.mkii t-syntax-highlight.mkiv 2context.vim]
+VIM_TEX  = %W[t-vim.tex t-syntax-groups.mkii t-syntax-groups.mkiv t-syntax-highlight.mkii t-syntax-highlight.mkiv 2context.vim vimtyping-default.css]
 VIM_DOC  = "vim-README.md"
 VIM_TEST = FileList['tests/vim/[0-9][0-9]-*.tex']
 

diff --git a/t-filter.mkiv b/t-filter.mkiv
@@ -1,6 +1,6 @@
 %D \module
 %D   [     file=t-filter,
-%D      version=2020.04.25,
+%D      version=2020.06.29,
 %D        title=\CONTEXT\ User Module,
 %D     subtitle=Filter,
 %D       author=Aditya Mahajan,
@@ -9,7 +9,7 @@
 %D        email=adityam <at> ieee <dot> org,
 %D      license=Simplified BSD License]
 
-\writestatus{loading}{Filter (ver: 2020.04.25)}
+\writestatus{loading}{Filter (ver: 2020.06.29)}
 
 \startmodule [filter]
 \usemodule   [module-catcodes]
@@ -35,6 +35,7 @@
     \setinterfaceconstant {write}            {write}
     \setinterfaceconstant {readcommand}      {readcommand}
     \setinterfaceconstant {taglabel}         {taglabel}
+    \setinterfaceconstant {tagdetail}        {tagdetail}
 \stopinterface
 
 
@@ -66,6 +67,10 @@
     \setuevalue{process\currentexternalfilter file}{\externalfilter@process_file[\currentexternalfilter]}%
     \setuevalue{process\currentexternalfilter buffer}{\externalfilter@process_buffer[\currentexternalfilter]}%
     \setuevalue{inline\currentexternalfilter}{\externalfilter@inline[\currentexternalfilter]}%
+    \setelementbackendtag[\externalfilterparameter\c!taglabel]
+    \setelementnature    [\externalfilterparameter\c!taglabel][display]
+    \setelementbackendtag[inline\externalfilterparameter\c!taglabel]
+    \setelementnature    [inline\externalfilterparameter\c!taglabel][inline]
 \to \everydefineexternalfilter
 
 % For backward compatibility
@@ -479,16 +484,17 @@
    \doifelse{\externalfilterparameter\c!location}\v!paragraph
      {\blank[\externalfilterparameter\c!spacebefore]
       \usealignparameter\externalfilterparameter
-      \externalfilterparameter\c!before}
-     {\externalfilterparameter\c!left}
+      \externalfilterparameter\c!before
+      \dostarttagged{\externalfilterparameter\c!taglabel}{\externalfilterparameter\c!tagdetail}}
+     {\externalfilterparameter\c!left
+      \dostarttagged{inline\externalfilterparameter\c!taglabel}{\externalfilterparameter\c!tagdetail}}
 
    \begingroup
-   \dostarttagged{\externalfilterparameter\c!taglabel}\currentexternalfilter
    \useexternalfilterstyleandcolor\c!style\c!color
    \processcommacommand[\externalfilterparameter\c!setups]\directsetup
    \externalfilterparameter\c!readcommand\externalfilter@output_file
-   \dostoptagged
    \endgroup
+   \dostoptagged
 
    \doifelse{\externalfilterparameter\c!location}\v!paragraph
      {\externalfilterparameter\c!after
@@ -526,6 +532,7 @@
    \c!buffer\c!before=,
    \c!buffer\c!after=,
    \c!taglabel=\externalfilter@id,
+   \c!tagdetail=\currentexternalfilter,
  ]
 
 \def\externalfilterbasefile  {\externalfilter@base_file}

diff --git a/t-syntax-groups.tex → t-syntax-groups.mkii b/t-syntax-groups.tex → t-syntax-groups.mkii
@@ -32,8 +32,6 @@
 
 \installparameterhandler  \syntaxgroup@namespace \syntaxgroup@id
 \installsetuphandler      \syntaxgroup@namespace \syntaxgroup@id
-\doifmode{\s!mkiv}
-  {\installstyleandcolorhandler      \syntaxgroup@namespace \syntaxgroup@id}
 
 \def\m!syntaxgroup{t-syntax-group}
 
@@ -99,14 +97,12 @@
       \iftracesyntaxgroups
           \syntaxgroup@show_values
       \fi
-      \doifmode{\s!mkiv}{\dostarttagged\t!syntaxgroup{#1}}
       \syntaxgroupparameter\c!command
       {
         \externalfilter@attributes_start\syntaxgroup@id\c!style\c!color
             #2
         \externalfilter@attributes_stop
       }
-      \doifmode{\s!mkiv}{\dostoptagged}
       \syntaxgroupparameter\c!after
       \endgroup
 \stoptexdefinition

diff --git a/t-syntax-groups.mkiv b/t-syntax-groups.mkiv
@@ -0,0 +1,126 @@
+%D \module
+%D   [     file=t-syntax-groups,
+%D      version=2020.06.29,
+%D        title=\CONTEXT\ User Module,
+%D     subtitle=Syntax highlighting support,
+%D       author=Aditya Mahajan,
+%D         date=\currentdate,
+%D    copyright=Aditya Mahajan,
+%D        email=adityam <at> ieee <dot> org,
+%D      license=Simplified BSD License]
+
+\writestatus{loading}{Syntax highlighting groups (ver: 2020.06.29)}
+
+\startmodule [syntax-group]
+\usemodule   [module-catcodes]
+
+\unprotectmodulecatcodes
+
+\def\colorscheme@name {}
+
+\def\syntaxgroup@id          {syntaxgroup}
+\def\syntaxgroup@namespace   {@@@@\syntaxgroup@id}
+\def\syntaxgroup@name        {}
+
+\edef\t!vimsyntaxgroup  {\syntaxgroup@id}
+\setelementbackendtag[\t!vimsyntaxgroup]
+\setelementnature    [\t!vimsyntaxgroup][inline]
+
+
+\installparameterhandler     \syntaxgroup@namespace \syntaxgroup@id
+\installsetuphandler         \syntaxgroup@namespace \syntaxgroup@id
+\installstyleandcolorhandler \syntaxgroup@namespace \syntaxgroup@id
+
+\def\m!syntaxgroup{t-syntax-group}
+
+\def\definesyntaxgroup
+    {\dodoubleargument\syntaxgroup@define}
+
+\starttexdefinition syntaxgroup@define [#1][#2]
+  % #1 list name
+  % #2 options
+  \doifassignmentelse{#2}
+  {
+    \def\syntaxgroup@get_parameters##1%
+    {
+         \edef\syntaxgroup@name {\colorscheme@name##1}
+         \setupsyntaxgroup[\syntaxgroup@name]
+                          [\s!parent=\syntaxgroup@namespace,\c!color=,\c!style=,\c!command=,#2]
+         \doifsomething{\syntaxgroupparameter\c!color}
+          {
+             \expanded{\definecolor[\syntaxgroup@namespace-##1-color]
+                                   [\syntaxgroupparameter\c!color]}
+             \setupsyntaxgroup[\syntaxgroup@name][\s!parent=\syntaxgroup@namespace,\c!color=\syntaxgroup@namespace-##1-color]
+          }
+    }
+  }{
+    \def\syntaxgroup@get_parameters##1%
+    {
+        \edef\syntaxgroup@name {\colorscheme@name##1}
+        \expanded{\setupsyntaxgroup[\syntaxgroup@name]
+                         [\c!color=\namedsyntaxgroupparameter{\colorscheme@name#2}\c!color,
+                          \c!style=\namedsyntaxgroupparameter{\colorscheme@name#2}\c!style,
+                        \c!command=\namedsyntaxgroupparameter{\colorscheme@name#2}\c!command,
+                         ]}
+
+        % In MkII, \expanded messes up the definition of \currentsyntaxgroup
+        \def\currentsyntaxgroup   {\syntaxgroup@name}
+    }
+  }
+
+  \processcommalist[#1]\syntaxgroup@get_parameters
+\stoptexdefinition
+
+\def\startcolorscheme%
+    {\dosingleargument\colorscheme@start}
+
+\starttexdefinition colorscheme@start [#1]
+     \pushmacro\colorscheme@name
+     \setcolorscheme{#1}
+     %\setupsyntaxgroup[\c!color=,\c!style=,\c!command=]
+\stoptexdefinition
+
+\def\stopcolorscheme
+    {\popmacro\colorscheme@name}
+
+\def\setcolorscheme#1%
+    {\edef\colorscheme@name{#1}}
+
+\starttexdefinition syntaxgroup [#1]#2
+  % #1 = style
+  % #2 = content
+      \begingroup
+      \edef\syntaxgroup@name{\colorscheme@name#1}%
+      \syntaxgroupparameter\c!before
+      \iftracesyntaxgroups
+          \syntaxgroup@show_values
+      \fi
+      \dostarttagged\t!vimsyntaxgroup{vim#1}
+      \syntaxgroupparameter\c!command
+      {
+        \externalfilter@attributes_start\syntaxgroup@id\c!style\c!color
+            #2
+        \externalfilter@attributes_stop
+      }
+      \dostoptagged
+      \syntaxgroupparameter\c!after
+      \endgroup
+\stoptexdefinition
+
+%D Tracing macros
+
+\newif\iftracesyntaxgroups
+\let\tracesyntaxgroups\tracesyntaxgroupstrue
+
+\def\currentsyntaxgroup   {\syntaxgroup@name}
+
+\starttexdefinition syntaxgroup@show_values
+  \writestatus\m!syntaxgroup{color scheme : \colorscheme@name}
+  \writestatus\m!syntaxgroup{current group: \syntaxgroup@name}
+  \writestatus\m!syntaxgroup{command      : \syntaxgroupparameter\c!command}
+  \writestatus\m!syntaxgroup{style        : \syntaxgroupparameter\c!style}
+  \writestatus\m!syntaxgroup{color        : \syntaxgroupparameter\c!color}
+\stoptexdefinition
+
+\protectmodulecatcodes
+\stopmodule
diff --git a/t-syntax-highlight.mkiv b/t-syntax-highlight.mkiv
@@ -1,6 +1,6 @@
 %D \module
 %D   [     file=t-syntax-highlight,
-%D      version=2018.08.10,
+%D      version=2020.06.29,
 %D        title=\CONTEXT\ User Module,
 %D     subtitle=Code syntax highlighting,
 %D       author=Aditya Mahajan,
@@ -9,7 +9,7 @@
 %D        email=adityam <at> ieee <dot> org,
 %D      license=Simplified BSD License]
 
-\writestatus{loading}{Code syntax highlighting (ver: 2018.08.10)}
+\writestatus{loading}{Code syntax highlighting (ver: 2020.06.29)}
 
 \startmodule    [syntax-highlight]
 \usemodule      [syntax-groups]
@@ -36,7 +36,10 @@
 
 \appendtoks
     \defineexternalfilter[\currentsyntaxhighlighting]
-                         [\s!parent=\????syntaxhighlighting\currentexternalfilter,\c!taglabel=\vimtyping@id]%
+                         [\s!parent=\????syntaxhighlighting\currentexternalfilter,
+                          \c!taglabel=\vimtyping@id,
+                          \c!tagdetail={\externalfilterparameter\c!alternative},
+                        ]%
     \normalexpanded{\definelinenumbering [\currentsyntaxhighlighting]}%
     \setevalue{type\currentsyntaxhighlighting file}{\getvalue{process\currentsyntaxhighlighting file}}%
 \to\everydefinesyntaxhighlighting
@@ -64,7 +67,6 @@
 \to\everysetupsyntaxhighlighting
 
 \def\syntaxhighlighting@id {syntaxhighlighting}
-\edef\t!syntaxhighlighting {\syntaxhighlighting@id}
 
 %D Helper macro
 
@@ -114,7 +116,7 @@
 \stoptexdefinition
 
 \starttexdefinition syntaxhighlighting@linenumbering_start
-   \doifinset{\externalfilterparameter\c!numbering}\syntaxhighlighting@yes
+   \doifinsetelse{\externalfilterparameter\c!numbering}\syntaxhighlighting@yes
        {\let\SYNBOL=\syntaxhighlighting_begin_number_lines
         \let\SYNEOL=\syntaxhighlighting_end_number_lines
         \startlinenumbering
@@ -125,24 +127,34 @@
                 \c!step=\externalfilterparameter{\c!number\c!step},
             \c!continue=\externalfilterparameter{\c!number\c!continue},
           ]}
-   \dostarttagged\t!syntaxhighlighting\colorscheme@name
+        {\let\SYNBOL=\syntaxhighlighting_begin_lines
+         \let\SYNEOL=\syntaxhighlighting_end_lines}
 \stoptexdefinition
 
 \starttexdefinition syntaxhighlighting@linenumbering_stop
-  \dostoptagged
    \doifinset{\externalfilterparameter\c!numbering}\syntaxhighlighting@yes
       {\stoplinenumbering}
 \stoptexdefinition
 
 \newcount\nofsyntaxhighlightinglines
 
+\starttexdefinition syntaxhighlighting_begin_lines
+  \dostarttaggednodetail\t!verbatimline
+\stoptexdefinition
+
+\starttexdefinition syntaxhighlighting_end_lines
+  \dostoptagged
+\stoptexdefinition
+
 \starttexdefinition syntaxhighlighting_begin_number_lines
   \global\advance\nofsyntaxhighlightinglines\plusone
+  \dostarttaggednodetail\t!verbatimline
   \attribute\verbatimlineattribute\nofsyntaxhighlightinglines
 \stoptexdefinition
 
 \starttexdefinition syntaxhighlighting_end_number_lines
-   \attribute\verbatimlineattribute\attributeunsetvalue
+  \attribute\verbatimlineattribute\attributeunsetvalue
+  \dostoptagged
 \stoptexdefinition