vdoc: use markdown.default_html_transformer underneath, remove html_tag_escape (#19785)

Author: nedpals

cmd/tools/vdoc/html.v

@@ -4,7 +4,6 @@ import os
 import net.urllib
 import strings
 import markdown
-import regex
 import v.scanner
 import v.ast
 import v.token
@@ -19,9 +18,6 @@ const link_svg = '<svg xmlns="http://www.w3.org/2000/svg" height="24" viewBox="0
 const single_quote = "'"
 const double_quote = '"'
 const no_quotes_replacement = [single_quote, '', double_quote, '']
-
-const html_tag_escape_re = regex.regex_opt(r'`.+[(&lt;)(&gt;)].+`') or { panic(err) }
-const html_tag_escape_seq = ['<', '&lt;', '>', '&gt;']
 const md_script_escape_seq = ['<script>', '`', '</script>', '`']
 
 enum HighlightTokenTyp {
@@ -508,7 +504,7 @@ fn doc_node_html(dn doc.DocNode, link string, head bool, include_examples bool,
 		// Strip markdown [TOC] directives, since we generate our own.
 		comments.replace('[TOC]', '').replace_each(md_script_escape_seq)
 	} else {
-		html_tag_escape(comments)
+		comments
 	}
 	mut renderer := markdown.HtmlRenderer{
 		transformer: &MdHtmlCodeHighlighter{
@@ -574,37 +570,6 @@ fn doc_node_html(dn doc.DocNode, link string, head bool, include_examples bool,
 	return dnw_str
 }
 
-fn html_tag_escape(str string) string {
-	escaped_string := str.replace_each(html_tag_escape_seq)
-	mut re := html_tag_escape_re
-	if re.find_all_str(escaped_string).len > 0 {
-		return str
-	}
-	return escaped_string
-}
-
-/*
-fn js_compress(str string) string {
-	mut js := strings.new_builder(200)
-	lines := str.split_into_lines()
-	rules := [') {', ' = ', ', ', '{ ', ' }', ' (', '; ', ' + ', ' < ', ' - ', ' || ', ' var',
-		': ', ' >= ', ' && ', ' else if', ' === ', ' !== ', ' else ']
-	clean := ['){', '=', ',', '{', '}', '(', ';', '+', '<', '-', '||', 'var', ':', '>=', '&&',
-		'else if', '===', '!==', 'else']
-	for line in lines {
-		mut trimmed := line.trim_space()
-		if trimmed.starts_with('//') || (trimmed.starts_with('/*') && trimmed.ends_with('*/')) {
-			continue
-		}
-		for i in 0 .. rules.len - 1 {
-			trimmed = trimmed.replace(rules[i], clean[i])
-		}
-		js.write_string(trimmed)
-	}
-	js_str := js.str()
-	return js_str
-}
-*/
 fn write_toc(dn doc.DocNode, mut toc strings.Builder) {
 	mut toc_slug := if dn.name.len == 0 || dn.content.len == 0 { '' } else { slug(dn.name) }
 	if toc_slug == '' && dn.children.len > 0 {
@@ -648,12 +613,15 @@ fn (f &MdHtmlCodeHighlighter) transform_attribute(p markdown.ParentType, name st
 }
 
 fn (f &MdHtmlCodeHighlighter) transform_content(parent markdown.ParentType, text string) string {
+	// NOTE: markdown.default_html_transformer uses html.escape internally.
+	initial_transformed_text := markdown.default_html_transformer.transform_content(parent,
+		text)
 	if parent is markdown.MD_BLOCKTYPE && parent == .md_block_code {
 		if f.language == 'v' || f.language == 'vlang' {
-			return html_highlight(html_tag_escape(text), f.table)
+			return html_highlight(initial_transformed_text, f.table)
 		}
 	}
-	return markdown.default_html_transformer.transform_content(parent, html_tag_escape(text))
+	return initial_transformed_text
 }
 
 fn (mut f MdHtmlCodeHighlighter) config_set(key string, val string) {

cmd/tools/vdoc/html_tag_escape_test.v

@@ -5,6 +5,10 @@ documentation in a project or module.
 
 This is a [link](https://vlang.io/) to the main V site.
 
+This is a <b>bold text</b>.
+
+This is a script <script>console.log('hi from README.md');</script> .
+
 ## Examples:
 
 ### Processing command line args:

cmd/tools/vdoc/tests/testdata/output_formats/README.md

@@ -2,6 +2,8 @@ Description:
 This is an example of a an .md file, used for adding more rich text
 documentation in a project or module.
 This is a link to the main V site.
+This is a bold text.
+This is a script console.log('hi from README.md'); .
 Examples:
 Processing command line args:
 import os
@@ -65,6 +67,10 @@ fn auth_verify(secret string, token string) bool {
 
     This is a [link](https://vlang.io/) to the main V site.
 
+    This is a <b>bold text</b>.
+    
+    This is a script <script>console.log('hi from README.md');</script> .
+    
     ## Examples:
 
     ### Processing command line args:
@@ -135,7 +141,7 @@ fn auth_verify(secret string, token string) bool {
 fn def()
     def - should be first
 fn xyz()
-    xyz - should be in the middle
+    xyz - should be in the middle a small script <script>console.log('hello');</script> bold text <b>bold</b> end underlined text <u>underline</u> end a link [main v repo](https://github.com/vlang/v)
 fn MyXMLDocument.abc(text string) ?(string, int)
     MyXMLDocument.abc does something too... I just do not know what.
 fn MyXMLDocument.from_file(path string) !MyXMLDocument

cmd/tools/vdoc/tests/testdata/output_formats/main.ansi

@@ -1,5 +1,5 @@
 <section id="readme_main" class="doc-node">
-			<div class="title"><h1> main <a href="#readme_main">#</a></h1></div><h2>Description:</h2><p>This is an example of a an .md file, used for adding more rich text documentation in a project or module.</p><p>This is a <a href="https://vlang.io/">link</a> to the main V site.</p><h2>Examples:</h2><h3>Processing command line args:</h3><pre><code class="language-v"><span class="token keyword">import</span> os
+			<div class="title"><h1> main <a href="#readme_main">#</a></h1></div><h2>Description:</h2><p>This is an example of a an .md file, used for adding more rich text documentation in a project or module.</p><p>This is a <a href="https://vlang.io/">link</a> to the main V site.</p><p>This is a <b>bold text</b>.</p><p>This is a script <code>console.log('hi from README.md');</code> .</p><h2>Examples:</h2><h3>Processing command line args:</h3><pre><code class="language-v"><span class="token keyword">import</span> os
 
 <span class="token keyword">fn</span> <span class="token function">main</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
 <span class="token keyword">dump</span><span class="token punctuation">(</span>os<span class="token punctuation">.</span>args<span class="token punctuation">)</span>
@@ -83,7 +83,7 @@
 </section>
 		<section id="xyz" class="doc-node">
 			<div class="title"><h2>fn xyz <a href="#xyz">#</a></h2></div><pre class="signature"><code><span class="token keyword">fn</span> <span class="token function">xyz</span><span class="token punctuation">(</span><span class="token punctuation">)</span></code></pre>
-<p>xyz - should be in the middle</p>
+<p>xyz - should be in the middle a small script <script>console.log('hello');</script> bold text <b>bold</b> end underlined text <u>underline</u> end a link <a href="https://github.com/vlang/v">main v repo</a></p>
 
 </section>
 		<section id="MyXMLDocument.abc" class="doc-node">

cmd/tools/vdoc/tests/testdata/output_formats/main.html

@@ -6,6 +6,10 @@ module main
 
     This is a [link](https://vlang.io/) to the main V site.
 
+    This is a <b>bold text</b>.
+    
+    This is a script <script>console.log('hi from README.md');</script> .
+    
     ## Examples:
 
     ### Processing command line args:
@@ -76,7 +80,7 @@ fn abc()
 fn def()
     def - should be first
 fn xyz()
-    xyz - should be in the middle
+    xyz - should be in the middle a small script <script>console.log('hello');</script> bold text <b>bold</b> end underlined text <u>underline</u> end a link [main v repo](https://github.com/vlang/v)
 fn MyXMLDocument.abc(text string) ?(string, int)
     MyXMLDocument.abc does something too... I just do not know what.
 fn MyXMLDocument.from_file(path string) !MyXMLDocument

cmd/tools/vdoc/tests/testdata/output_formats/main.text

@@ -10,6 +10,10 @@ pub fn def() {
 }
 
 // xyz - should be in the middle
+// a small script <script>console.log('hello');</script>
+// bold text <b>bold</b> end
+// underlined text <u>underline</u> end
+// a link [main v repo](https://github.com/vlang/v)
 pub fn xyz() {
 	println(2)
 }