feat: Permit non-ASCII within <t> without the use of <u> (#895)

This change allows bare Unicode in the t element. This also introduces a new flag `--warn-bare-unicode` when set, `xml2rfc` warns about bare Unicode in the `<t>` element. By default, this is set to False. See [RSAB decision](https://mailarchive.ietf.org/arch/msg/rsab/QHkUiHC0r99EvH-ikIb16THtcmM/) and [RSWG discussion](https://mailarchive.ietf.org/arch/msg/rswg/LtZcNiVXhkG8R1XLd5pyO-LE1Xo/). Co-authored-by: Kesara Rathnayake <kesara@fq.nz>
ietf-tools · Jan 18, 2023 · ad2e035 · ad2e035
1 parent 6c9be77
commit ad2e035
Show file tree

Hide file tree

Showing 19 changed files with 92 additions and 60 deletions.
diff --git a/tests/valid/docfile.html b/tests/valid/docfile.html
@@ -39,7 +39,7 @@
 <dd class="workgroup">xml2rfc(1)</dd>
 <dt class="label-published">Published:</dt>
 <dd class="published">
-<time datetime="2023-01-17" class="published">17 January 2023</time>
+<time datetime="2023-01-18" class="published">18 January 2023</time>
     </dd>
 <dt class="label-authors">Author:</dt>
 <dd class="authors">
@@ -5488,7 +5488,7 @@ <h3 id="name-documentation-options">
         </h3>
 <p id="appendix-B.2-1">
                Some options to generate built-in documentation.
-               The group has 8 options.<a href="#appendix-B.2-1" class="pilcrow">¶</a></p>
+               The group has 9 options.<a href="#appendix-B.2-1" class="pilcrow">¶</a></p>
 <span class="break"></span><dl class="dlNewline" id="appendix-B.2-2">
           <dt id="appendix-B.2-2.1">
 <div id="option--help">
@@ -5594,6 +5594,15 @@ <h3 id="name-documentation-options">
                           Display the version number and exit.<a href="#appendix-B.2-2.16.1" class="pilcrow">¶</a></p>
 <p id="appendix-B.2-2.16.2">
        With the <code>--verbose</code> switch, the versions of the external python modules used will also be shown.<a href="#appendix-B.2-2.16.2" class="pilcrow">¶</a></p>
+</dd>
+          <dd class="break"></dd>
+<dt id="appendix-B.2-2.17">
+<div id="option--warn-bare-unicode">
+<code>--warn-bare-unicode</code>                    </div>
+          </dt>
+<dd style="margin-left: 1.5em" id="appendix-B.2-2.18">
+            <p id="appendix-B.2-2.18.1">
+                          Warn use of bare Unicode in tags where bare use of Unicode should be closely inspected.<a href="#appendix-B.2-2.18.1" class="pilcrow">¶</a></p>
 </dd>
         <dd class="break"></dd>
 </dl>

diff --git a/tests/valid/draft-miek-test.html b/tests/valid/draft-miek-test.html
@@ -35,7 +35,7 @@
     intervaltree 3.1.0
     Jinja2 3.1.2
     lxml 4.9.2
-    MarkupSafe 2.1.1
+    MarkupSafe 2.1.2
     pycountry 22.3.5
     PyYAML 6.0
     requests 2.28.2

diff --git a/tests/valid/draft-template.html b/tests/valid/draft-template.html
@@ -24,7 +24,7 @@
     intervaltree 3.1.0
     Jinja2 3.1.2
     lxml 4.9.2
-    MarkupSafe 2.1.1
+    MarkupSafe 2.1.2
     pycountry 22.3.5
     PyYAML 6.0
     requests 2.28.2

diff --git a/tests/valid/indexes.pages.text b/tests/valid/indexes.pages.text
@@ -3,9 +3,9 @@
 
 
 Network Working Group                                     H. Person, Ed.
-Internet-Draft                                          January 17, 2023
+Internet-Draft                                          January 18, 2023
 Intended status: Experimental                                           
-Expires: July 21, 2023
+Expires: July 22, 2023
 
 
                           xml2rfc index tests
@@ -26,7 +26,7 @@ Status of This Memo
    time.  It is inappropriate to use Internet-Drafts as reference
    material or to cite them other than as "work in progress."
 
-   This Internet-Draft will expire on July 21, 2023.
+   This Internet-Draft will expire on July 22, 2023.
 
 Copyright Notice
 
@@ -53,7 +53,7 @@ Table of Contents
 
 
 
-Person                    Expires July 21, 2023                 [Page 1]
+Person                    Expires July 22, 2023                 [Page 1]
 
 Internet-Draft             xml2rfc index tests              January 2023
 
@@ -109,7 +109,7 @@ Index
 
 
 
-Person                    Expires July 21, 2023                 [Page 2]
+Person                    Expires July 22, 2023                 [Page 2]
 
 Internet-Draft             xml2rfc index tests              January 2023
 
@@ -165,4 +165,4 @@ Author's Address
 
 
 
-Person                    Expires July 21, 2023                 [Page 3]
+Person                    Expires July 22, 2023                 [Page 3]
diff --git a/tests/valid/indexes.prepped.xml b/tests/valid/indexes.prepped.xml
@@ -1,5 +1,5 @@
 <?xml version='1.0' encoding='utf-8'?>
-<rfc xmlns:xi="http://www.w3.org/2001/XInclude" version="3" category="exp" docName="indexes-00" indexInclude="true" ipr="trust200902" prepTime="2023-01-17T08:47:44" scripts="Common,Latin" sortRefs="true" submissionType="independent" symRefs="true" tocDepth="3" tocInclude="true">
+<rfc xmlns:xi="http://www.w3.org/2001/XInclude" version="3" category="exp" docName="indexes-00" indexInclude="true" ipr="trust200902" prepTime="2023-01-18T00:26:59" scripts="Common,Latin" sortRefs="true" submissionType="independent" symRefs="true" tocDepth="3" tocInclude="true">
   <!-- xml2rfc v2v3 conversion 3.15.3 -->
 
 
@@ -20,7 +20,7 @@
         </postal>
       </address>
     </author>
-    <date day="17" month="01" year="2023"/>
+    <date day="18" month="01" year="2023"/>
     <boilerplate>
       <section anchor="status-of-memo" numbered="false" removeInRFC="false" toc="exclude" pn="section-boilerplate.1">
         <name slugifiedName="name-status-of-this-memo">Status of This Memo</name>
@@ -41,7 +41,7 @@
         material or to cite them other than as "work in progress."
         </t>
         <t indent="0" pn="section-boilerplate.1-4">
-        This Internet-Draft will expire on 21 July 2023.
+        This Internet-Draft will expire on 22 July 2023.
         </t>
       </section>
       <section anchor="copyright" numbered="false" removeInRFC="false" toc="exclude" pn="section-boilerplate.2">

diff --git a/tests/valid/indexes.text b/tests/valid/indexes.text
@@ -3,9 +3,9 @@
 
 
 Network Working Group                                     H. Person, Ed.
-Internet-Draft                                          January 17, 2023
+Internet-Draft                                          January 18, 2023
 Intended status: Experimental                                           
-Expires: July 21, 2023
+Expires: July 22, 2023
 
 
                           xml2rfc index tests
@@ -26,7 +26,7 @@ Status of This Memo
    time.  It is inappropriate to use Internet-Drafts as reference
    material or to cite them other than as "work in progress."
 
-   This Internet-Draft will expire on July 21, 2023.
+   This Internet-Draft will expire on July 22, 2023.
 
 Copyright Notice
 

diff --git a/tests/valid/indexes.v3.html b/tests/valid/indexes.v3.html
@@ -23,7 +23,7 @@
 </tr></thead>
 <tfoot><tr>
 <td class="left">Person</td>
-<td class="center">Expires July 21, 2023</td>
+<td class="center">Expires July 22, 2023</td>
 <td class="right">[Page]</td>
 </tr></tfoot>
 </table>
@@ -36,12 +36,12 @@
 <dd class="internet-draft">indexes-00</dd>
 <dt class="label-published">Published:</dt>
 <dd class="published">
-<time datetime="2023-01-17" class="published">January 17, 2023</time>
+<time datetime="2023-01-18" class="published">January 18, 2023</time>
     </dd>
 <dt class="label-intended-status">Intended Status:</dt>
 <dd class="intended-status">Experimental</dd>
 <dt class="label-expires">Expires:</dt>
-<dd class="expires"><time datetime="2023-07-21">July 21, 2023</time></dd>
+<dd class="expires"><time datetime="2023-07-22">July 22, 2023</time></dd>
 <dt class="label-authors">Author:</dt>
 <dd class="authors">
 <div class="author">
@@ -71,7 +71,7 @@ <h2 id="name-status-of-this-memo">
         time. It is inappropriate to use Internet-Drafts as reference
         material or to cite them other than as "work in progress."<a href="#section-boilerplate.1-3" class="pilcrow">¶</a></p>
 <p id="section-boilerplate.1-4">
-        This Internet-Draft will expire on July 21, 2023.<a href="#section-boilerplate.1-4" class="pilcrow">¶</a></p>
+        This Internet-Draft will expire on July 22, 2023.<a href="#section-boilerplate.1-4" class="pilcrow">¶</a></p>
 </section>
 </div>
 <div id="copyright">

diff --git a/tests/valid/manpage.txt b/tests/valid/manpage.txt
@@ -1,5 +1,5 @@
 xml2rfc(1)                                                    xml2rfc(1)
-                                                         17 January 2023
+                                                         18 January 2023
 
 
                   Xml2rfc Vocabulary Version 3 Schema
@@ -3889,7 +3889,7 @@ B.1.  Positional arguments
 
 B.2.  Documentation options
 
-   Some options to generate built-in documentation.  The group has 8
+   Some options to generate built-in documentation.  The group has 9
    options.
 
    --help, -h
@@ -3966,6 +3966,10 @@ B.2.  Documentation options
       With the --verbose switch, the versions of the external python
       modules used will also be shown.
 
+   --warn-bare-unicode
+      Warn use of bare Unicode in tags where bare use of Unicode should
+      be closely inspected.
+
 B.3.  Format selection
 
    One or more of the following output formats may be specified.  The

diff --git a/tests/valid/rfc7911.html b/tests/valid/rfc7911.html
@@ -28,7 +28,7 @@
     intervaltree 3.1.0
     Jinja2 3.1.2
     lxml 4.9.2
-    MarkupSafe 2.1.1
+    MarkupSafe 2.1.2
     pycountry 22.3.5
     PyYAML 6.0
     requests 2.28.2

diff --git a/tests/valid/sourcecode.pages.text b/tests/valid/sourcecode.pages.text
@@ -3,9 +3,9 @@
 
 
 Network Working Group                                     H. Person, Ed.
-Internet-Draft                                          January 17, 2023
+Internet-Draft                                          January 18, 2023
 Intended status: Experimental                                           
-Expires: July 21, 2023
+Expires: July 22, 2023
 
 
                         xml2rfc sourcecode tests
@@ -26,7 +26,7 @@ Status of This Memo
    time.  It is inappropriate to use Internet-Drafts as reference
    material or to cite them other than as "work in progress."
 
-   This Internet-Draft will expire on July 21, 2023.
+   This Internet-Draft will expire on July 22, 2023.
 
 Copyright Notice
 
@@ -53,7 +53,7 @@ Table of Contents
 
 
 
-Person                    Expires July 21, 2023                 [Page 1]
+Person                    Expires July 22, 2023                 [Page 1]
 
 Internet-Draft          xml2rfc sourcecode tests            January 2023
 
@@ -109,7 +109,7 @@ Internet-Draft          xml2rfc sourcecode tests            January 2023
 
 
 
-Person                    Expires July 21, 2023                 [Page 2]
+Person                    Expires July 22, 2023                 [Page 2]
 
 Internet-Draft          xml2rfc sourcecode tests            January 2023
 
@@ -165,7 +165,7 @@ Internet-Draft          xml2rfc sourcecode tests            January 2023
 
 
 
-Person                    Expires July 21, 2023                 [Page 3]
+Person                    Expires July 22, 2023                 [Page 3]
 
 Internet-Draft          xml2rfc sourcecode tests            January 2023
 
@@ -221,4 +221,4 @@ Author's Address
 
 
 
-Person                    Expires July 21, 2023                 [Page 4]
+Person                    Expires July 22, 2023                 [Page 4]
diff --git a/tests/valid/sourcecode.prepped.xml b/tests/valid/sourcecode.prepped.xml
@@ -1,5 +1,5 @@
 <?xml version='1.0' encoding='utf-8'?>
-<rfc xmlns:xi="http://www.w3.org/2001/XInclude" version="3" category="exp" docName="sourcecode-00" indexInclude="true" ipr="trust200902" prepTime="2023-01-17T08:47:54" scripts="Common,Latin" sortRefs="true" submissionType="independent" symRefs="true" tocDepth="3" tocInclude="true">
+<rfc xmlns:xi="http://www.w3.org/2001/XInclude" version="3" category="exp" docName="sourcecode-00" indexInclude="true" ipr="trust200902" prepTime="2023-01-18T00:27:10" scripts="Common,Latin" sortRefs="true" submissionType="independent" symRefs="true" tocDepth="3" tocInclude="true">
   <!-- xml2rfc v2v3 conversion 3.15.3 -->
 
 
@@ -20,7 +20,7 @@
         </postal>
       </address>
     </author>
-    <date day="17" month="01" year="2023"/>
+    <date day="18" month="01" year="2023"/>
     <boilerplate>
       <section anchor="status-of-memo" numbered="false" removeInRFC="false" toc="exclude" pn="section-boilerplate.1">
         <name slugifiedName="name-status-of-this-memo">Status of This Memo</name>
@@ -41,7 +41,7 @@
         material or to cite them other than as "work in progress."
         </t>
         <t indent="0" pn="section-boilerplate.1-4">
-        This Internet-Draft will expire on 21 July 2023.
+        This Internet-Draft will expire on 22 July 2023.
         </t>
       </section>
       <section anchor="copyright" numbered="false" removeInRFC="false" toc="exclude" pn="section-boilerplate.2">

diff --git a/tests/valid/sourcecode.text b/tests/valid/sourcecode.text
@@ -3,9 +3,9 @@
 
 
 Network Working Group                                     H. Person, Ed.
-Internet-Draft                                          January 17, 2023
+Internet-Draft                                          January 18, 2023
 Intended status: Experimental                                           
-Expires: July 21, 2023
+Expires: July 22, 2023
 
 
                         xml2rfc sourcecode tests
@@ -26,7 +26,7 @@ Status of This Memo
    time.  It is inappropriate to use Internet-Drafts as reference
    material or to cite them other than as "work in progress."
 
-   This Internet-Draft will expire on July 21, 2023.
+   This Internet-Draft will expire on July 22, 2023.
 
 Copyright Notice
 

diff --git a/tests/valid/sourcecode.v3.html b/tests/valid/sourcecode.v3.html
@@ -23,7 +23,7 @@
 </tr></thead>
 <tfoot><tr>
 <td class="left">Person</td>
-<td class="center">Expires July 21, 2023</td>
+<td class="center">Expires July 22, 2023</td>
 <td class="right">[Page]</td>
 </tr></tfoot>
 </table>
@@ -36,12 +36,12 @@
 <dd class="internet-draft">sourcecode-00</dd>
 <dt class="label-published">Published:</dt>
 <dd class="published">
-<time datetime="2023-01-17" class="published">January 17, 2023</time>
+<time datetime="2023-01-18" class="published">January 18, 2023</time>
     </dd>
 <dt class="label-intended-status">Intended Status:</dt>
 <dd class="intended-status">Experimental</dd>
 <dt class="label-expires">Expires:</dt>
-<dd class="expires"><time datetime="2023-07-21">July 21, 2023</time></dd>
+<dd class="expires"><time datetime="2023-07-22">July 22, 2023</time></dd>
 <dt class="label-authors">Author:</dt>
 <dd class="authors">
 <div class="author">
@@ -71,7 +71,7 @@ <h2 id="name-status-of-this-memo">
         time. It is inappropriate to use Internet-Drafts as reference
         material or to cite them other than as "work in progress."<a href="#section-boilerplate.1-3" class="pilcrow">¶</a></p>
 <p id="section-boilerplate.1-4">
-        This Internet-Draft will expire on July 21, 2023.<a href="#section-boilerplate.1-4" class="pilcrow">¶</a></p>
+        This Internet-Draft will expire on July 22, 2023.<a href="#section-boilerplate.1-4" class="pilcrow">¶</a></p>
 </section>
 </div>
 <div id="copyright">

diff --git a/tests/valid/utf8.prepped.xml b/tests/valid/utf8.prepped.xml
@@ -1,6 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<rfc xmlns:xi="http://www.w3.org/2001/XInclude" version="3" category="info" consensus="false" docName="draft-sample-input-00" indexInclude="true" ipr="trust200902" prepTime="2021-10-08T15:41:43" scripts="Common,Latin" sortRefs="false" submissionType="IETF" symRefs="true" tocDepth="3" tocInclude="true">
-  <!-- xml2rfc v2v3 conversion 3.10.0 -->
+<rfc xmlns:xi="http://www.w3.org/2001/XInclude" version="3" category="info" consensus="false" docName="draft-sample-input-00" indexInclude="true" ipr="trust200902" prepTime="2023-01-18T00:26:40" scripts="Common,Han,Latin" sortRefs="false" submissionType="IETF" symRefs="true" tocDepth="3" tocInclude="true">
+  <!-- xml2rfc v2v3 conversion 3.15.3 -->
 
   <front>
     <title abbrev="Abbreviated Title">Put Your Internet Draft Title</title>
@@ -21,7 +21,7 @@
     <abstract pn="section-abstract">
       <t indent="0" pn="section-abstract-1">Insert an abstract: MANDATORY. This template is for creating an
       Internet-Draft.  With some out of scope characters
-      in Chinese, by Xing Xing, &amp;#36825;&amp;#37324;&amp;#26159;&amp;#20013;&amp;#25991;&amp;#35793;&amp;#26412;
+      in Chinese, by Xing Xing, 这里是中文译本
       </t>
     </abstract>
     <boilerplate>
@@ -81,16 +81,16 @@
     <section numbered="true" removeInRFC="false" toc="include" pn="section-1">
       <name slugifiedName="name-some-unicode-strings">Some unicode strings</name>
       <t indent="0" pn="section-1-1">
-        Text body needs to deal with 'funny' characters
+        Text body needs to deal with ‘funny’ characters
       </t>
       <t indent="0" pn="section-1-2">
-        Pure out of scope &amp;#36825;&amp;#37324;&amp;#26159;&amp;#20013;&amp;#25991;&amp;#35793;&amp;#26412;
+        Pure out of scope 这里是中文译本
       </t>
       <t indent="0" pn="section-1-3">
-        Some re-mapped characters are [cents] or (C)
+        Some re-mapped characters are ¢ or ©
       </t>
       <t indent="0" pn="section-1-4">
-        More re-mapped characters are ~ and EUR and ^
+        More re-mapped characters are ˜ and € and ^
       </t>
     </section>
   </middle>