From b6a7abf02652b74872b9c676fcfa545e18d9bde7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= <mosra@centrum.cz>
Date: Sat, 25 Nov 2017 18:37:26 +0100
Subject: [PATCH] Encode invalid XML characters instead of skipping them.

The <sp> element in <programlisting> has a new optional attribute
`value`, which contains value of given invalid ASCII character. In
case of space, the `value` attribute is omitted.

Use cases for this: including snippets of *very esoteric* languages,
markup that makes use of the advanced ASCII formatting characters or
for example highlighting a console output containing ANSI color codes
(which is my case, in fact).

Regarding backwards compatibility -- as files with such ASCII
characters are very rare, I don't expect this minor difference in the
output to be a problem. Besides that, such ASCII characters are often
replaced by a space in many applications anyway.

A test snippet was extended to contain a special character so this
difference in behavior could be verified.
---
 src/xmlgen.cpp                            | 4 +++-
 templates/xml/compound.xsd                | 6 +++++-
 testing/025/example_test_8cpp-example.xml | 2 +-
 testing/032/indexpage.xml                 | 2 +-
 testing/example_test.cpp                  | 2 ++
 5 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/xmlgen.cpp b/src/xmlgen.cpp
index 1b03c3bc17d..56e107b04ef 100644
--- a/src/xmlgen.cpp
+++ b/src/xmlgen.cpp
@@ -136,7 +136,9 @@ inline void writeXMLCodeString(FTextStream &t,const char *s, int &col)
       case 11: case 12: case 13: case 14: case 15: case 16: case 17: case 18:
       case 19: case 20: case 21: case 22: case 23: case 24: case 25: case 26:
       case 27: case 28: case 29: case 30: case 31:
-        break; // skip invalid XML characters (see http://www.w3.org/TR/2000/REC-xml-20001006#NT-Char)
+        // encode invalid XML characters (see http://www.w3.org/TR/2000/REC-xml-20001006#NT-Char)
+        t << "<sp value=\"" << int(c) << "\"/>";
+        break;
       default:   s=writeUtf8Char(t,s-1); col++; break;
     }
   }
diff --git a/templates/xml/compound.xsd b/templates/xml/compound.xsd
index 8affc20a5b7..efe14818b7e 100644
--- a/templates/xml/compound.xsd
+++ b/templates/xml/compound.xsd
@@ -279,12 +279,16 @@
 
   <xsd:complexType name="highlightType" mixed="true">
     <xsd:choice minOccurs="0" maxOccurs="unbounded">
-      <xsd:element name="sp" />
+      <xsd:element name="sp" type="spType" />
       <xsd:element name="ref" type="refTextType" />
     </xsd:choice>
     <xsd:attribute name="class" type="DoxHighlightClass" />
   </xsd:complexType>
 
+  <xsd:complexType name="spType" mixed="true">
+    <xsd:attribute name="value" type="xsd:integer" use="optional"/>
+  </xsd:complexType>
+
   <xsd:complexType name="referenceType" mixed="true">
     <xsd:attribute name="refid" type="xsd:string" />
     <xsd:attribute name="compoundref" type="xsd:string" use="optional" />
diff --git a/testing/025/example_test_8cpp-example.xml b/testing/025/example_test_8cpp-example.xml
index 96a1cd7876d..1e01cba166a 100644
--- a/testing/025/example_test_8cpp-example.xml
+++ b/testing/025/example_test_8cpp-example.xml
@@ -4,7 +4,7 @@
     <compoundname>example_test.cpp</compoundname>
     <detaileddescription>
       <para>This is an example of how to use the <ref refid="class_test" kindref="compound">Test</ref> class.</para>
-      <para>More details about this example. <programlisting filename="example_test.cpp"><codeline><highlight class="keywordtype">void</highlight><highlight class="normal"><sp/>main()</highlight></codeline><codeline><highlight class="normal">{</highlight></codeline><codeline><highlight class="normal"><sp/><sp/><ref refid="class_test" kindref="compound">Test</ref><sp/>t;</highlight></codeline><codeline><highlight class="normal"><sp/><sp/>t.<ref refid="class_test_1a47b775f65718978f1ffcd96376f8ecfa" kindref="member">example</ref>();</highlight></codeline><codeline><highlight class="normal">}</highlight></codeline><codeline><highlight class="normal"/></codeline></programlisting> </para>
+      <para>More details about this example. <programlisting filename="example_test.cpp"><codeline><highlight class="keywordtype">void</highlight><highlight class="normal"><sp/>main()</highlight></codeline><codeline><highlight class="normal">{</highlight></codeline><codeline><highlight class="normal"><sp/><sp/></highlight><highlight class="keyword">const</highlight><highlight class="normal"><sp/></highlight><highlight class="keywordtype">char</highlight><highlight class="normal">*<sp/>a<sp/>=<sp/></highlight><highlight class="stringliteral">"Some<sp/>special<sp/>character<sp/>here:<sp/><sp value="7"/><sp/>"</highlight><highlight class="normal">;</highlight></codeline><codeline><highlight class="normal"/></codeline><codeline><highlight class="normal"><sp/><sp/><ref refid="class_test" kindref="compound">Test</ref><sp/>t;</highlight></codeline><codeline><highlight class="normal"><sp/><sp/>t.<ref refid="class_test_1a47b775f65718978f1ffcd96376f8ecfa" kindref="member">example</ref>();</highlight></codeline><codeline><highlight class="normal">}</highlight></codeline><codeline><highlight class="normal"/></codeline></programlisting> </para>
     </detaileddescription>
   </compounddef>
 </doxygen>
diff --git a/testing/032/indexpage.xml b/testing/032/indexpage.xml
index 3a252723943..1894ae4b58e 100644
--- a/testing/032/indexpage.xml
+++ b/testing/032/indexpage.xml
@@ -4,7 +4,7 @@
     <compoundname>index</compoundname>
     <title>My Project</title>
     <detaileddescription>
-      <para>Some text. <programlisting filename="example_test.cpp"><codeline><highlight class="keywordtype">void</highlight><highlight class="normal"><sp/>main()</highlight></codeline><codeline><highlight class="normal">{</highlight></codeline><codeline><highlight class="normal"><sp/><sp/>Test<sp/>t;</highlight></codeline><codeline><highlight class="normal"><sp/><sp/>t.example();</highlight></codeline><codeline><highlight class="normal">}</highlight></codeline><codeline><highlight class="normal"/></codeline></programlisting> More text. <programlisting filename="example_test.cpp"><codeline lineno="1"><highlight class="keywordtype">void</highlight><highlight class="normal"><sp/>main()</highlight></codeline><codeline lineno="2"><highlight class="normal">{</highlight></codeline><codeline lineno="3"><highlight class="normal"><sp/><sp/>Test<sp/>t;</highlight></codeline><codeline lineno="4"><highlight class="normal"><sp/><sp/>t.example();</highlight></codeline><codeline lineno="5"><highlight class="normal">}</highlight></codeline><codeline lineno="6"><highlight class="normal"/></codeline></programlisting> End. </para>
+      <para>Some text. <programlisting filename="example_test.cpp"><codeline><highlight class="keywordtype">void</highlight><highlight class="normal"><sp/>main()</highlight></codeline><codeline><highlight class="normal">{</highlight></codeline><codeline><highlight class="normal"><sp/><sp/></highlight><highlight class="keyword">const</highlight><highlight class="normal"><sp/></highlight><highlight class="keywordtype">char</highlight><highlight class="normal">*<sp/>a<sp/>=<sp/></highlight><highlight class="stringliteral">"Some<sp/>special<sp/>character<sp/>here:<sp/><sp value="7"/><sp/>"</highlight><highlight class="normal">;</highlight></codeline><codeline><highlight class="normal"/></codeline><codeline><highlight class="normal"><sp/><sp/>Test<sp/>t;</highlight></codeline><codeline><highlight class="normal"><sp/><sp/>t.example();</highlight></codeline><codeline><highlight class="normal">}</highlight></codeline><codeline><highlight class="normal"/></codeline></programlisting> More text. <programlisting filename="example_test.cpp"><codeline lineno="1"><highlight class="keywordtype">void</highlight><highlight class="normal"><sp/>main()</highlight></codeline><codeline lineno="2"><highlight class="normal">{</highlight></codeline><codeline lineno="3"><highlight class="normal"><sp/><sp/></highlight><highlight class="keyword">const</highlight><highlight class="normal"><sp/></highlight><highlight class="keywordtype">char</highlight><highlight class="normal">*<sp/>a<sp/>=<sp/></highlight><highlight class="stringliteral">"Some<sp/>special<sp/>character<sp/>here:<sp/><sp value="7"/><sp/>"</highlight><highlight class="normal">;</highlight></codeline><codeline lineno="4"><highlight class="normal"/></codeline><codeline lineno="5"><highlight class="normal"><sp/><sp/>Test<sp/>t;</highlight></codeline><codeline lineno="6"><highlight class="normal"><sp/><sp/>t.example();</highlight></codeline><codeline lineno="7"><highlight class="normal">}</highlight></codeline><codeline lineno="8"><highlight class="normal"/></codeline></programlisting> End. </para>
     </detaileddescription>
   </compounddef>
 </doxygen>
diff --git a/testing/example_test.cpp b/testing/example_test.cpp
index f589023b3b2..a117b636c22 100644
--- a/testing/example_test.cpp
+++ b/testing/example_test.cpp
@@ -1,5 +1,7 @@
 void main()
 {
+  const char* a = "Some special character here:  ";
+
   Test t;
   t.example();
 }