vlib: add an encoding.xml module with parser, validation, entity encoding, unit tests (#19708)

Author: hungrybluedev

.gitignore

@@ -130,4 +130,9 @@ vls.log
 wasm.v
 TAGS
 tags
-vlib/builtin/js/*.js
+
+# ignore large GTK *.gir files
+Gtk-4.0.gir
+*.gir
+
+vlib/builtin/js/*.js

vlib/encoding/xml/README.md

@@ -0,0 +1,44 @@
+## Description
+
+`xml` is a module to parse XML documents into a tree structure. It also supports
+validation of XML documents against a DTD.
+
+Note that this is not a streaming XML parser. It reads the entire document into
+memory and then parses it. This is not a problem for small documents, but it
+might be a problem for extremely large documents (several hundred megabytes or more).
+
+## Usage
+
+### Parsing XML Files
+
+There are three different ways to parse an XML Document:
+
+1. Pass the entire XML document as a string to `XMLDocument.from_string`.
+2. Specify a file path to `XMLDocument.from_file`.
+3. Use a source that implements `io.Reader` and pass it to `XMLDocument.from_reader`.
+
+```v
+import encoding.xml
+
+//...
+doc := xml.XMLDocument.from_file('test/sample.xml')!
+```
+
+### Validating XML Documents
+
+Simply call `validate` on the parsed XML document.
+
+### Querying
+
+Check the `get_element...` methods defined on the XMLDocument struct.
+
+### Escaping and Un-escaping XML Entities
+
+When the `validate` method is called, the XML document is parsed and all text
+nodes are un-escaped. This means that the text nodes will contain the actual
+text and not the escaped version of the text.
+
+When the XML document is serialized (using `str` or `pretty_str`), all text nodes are escaped.
+
+The escaping and un-escaping can also be done manually using the `escape_text` and
+`unescape_text` methods.

vlib/encoding/xml/encoding.v

@@ -0,0 +1,148 @@
+module xml
+
+import strings
+
+// pretty_str returns a pretty-printed version of the XML node. It requires the current indentation
+// the node is at, the depth of the node in the tree, and a map of reverse entities to use when
+// escaping text.
+pub fn (node XMLNode) pretty_str(original_indent string, depth int, reverse_entities map[string]string) string {
+	// Create the proper indentation first
+	mut indent_builder := strings.new_builder(original_indent.len * depth)
+	for _ in 0 .. depth {
+		indent_builder.write_string(original_indent)
+	}
+	indent := indent_builder.str()
+
+	// Now we can stringify the node
+	mut builder := strings.new_builder(1024)
+	builder.write_string(indent)
+	builder.write_u8(`<`)
+	builder.write_string(node.name)
+
+	for key, value in node.attributes {
+		builder.write_u8(` `)
+		builder.write_string(key)
+		builder.write_string('="')
+		builder.write_string(value)
+		builder.write_u8(`"`)
+	}
+	builder.write_string('>\n')
+	for child in node.children {
+		match child {
+			string {
+				builder.write_string(indent)
+				builder.write_string(original_indent)
+				builder.write_string(escape_text(child, reverse_entities: reverse_entities))
+			}
+			XMLNode {
+				builder.write_string(child.pretty_str(original_indent, depth + 1, reverse_entities))
+			}
+			XMLComment {
+				builder.write_string(indent)
+				builder.write_string(original_indent)
+				builder.write_string('<!--')
+				builder.write_string(child.text)
+				builder.write_string('-->')
+			}
+			XMLCData {
+				builder.write_string(indent)
+				builder.write_string(original_indent)
+				builder.write_string('<![CDATA[')
+				builder.write_string(child.text)
+				builder.write_string(']]>')
+			}
+		}
+		builder.write_u8(`\n`)
+	}
+	builder.write_string(indent)
+	builder.write_string('</')
+	builder.write_string(node.name)
+	builder.write_u8(`>`)
+	return builder.str()
+}
+
+fn (list []DTDListItem) pretty_str(indent string) string {
+	if list.len == 0 {
+		return ''
+	}
+
+	mut builder := strings.new_builder(1024)
+	builder.write_u8(`[`)
+	builder.write_u8(`\n`)
+
+	for item in list {
+		match item {
+			DTDEntity {
+				builder.write_string('${indent}<!ENTITY ${item.name} "${item.value}">')
+			}
+			DTDElement {
+				builder.write_string('${indent}<!ELEMENT ${item.name} ${item.definition}>')
+			}
+		}
+		builder.write_u8(`\n`)
+	}
+	builder.write_u8(`]`)
+	return builder.str()
+}
+
+fn (doctype DocumentType) pretty_str(indent string) string {
+	match doctype.dtd {
+		string {
+			content := doctype.dtd
+			return if content.len > 0 {
+				'<!DOCTYPE ${doctype.name} SYSTEM "${content}">'
+			} else {
+				''
+			}
+		}
+		DocumentTypeDefinition {
+			if doctype.dtd.list.len == 0 {
+				return ''
+			}
+
+			mut builder := strings.new_builder(1024)
+			builder.write_string('<!DOCTYPE ')
+			builder.write_string(doctype.name)
+			builder.write_string(' ')
+			builder.write_string(doctype.dtd.list.pretty_str(indent))
+			builder.write_string('>')
+			builder.write_u8(`\n`)
+			return builder.str()
+		}
+	}
+}
+
+// pretty_str returns a pretty-printed version of the XML document. It requires the string used to
+// indent each level of the document.
+pub fn (doc XMLDocument) pretty_str(indent string) string {
+	mut document_builder := strings.new_builder(1024)
+
+	prolog := '<?xml version="${doc.version}" encoding="${doc.encoding}"?>'
+	comments := if doc.comments.len > 0 {
+		mut comments_buffer := strings.new_builder(512)
+		for comment in doc.comments {
+			comments_buffer.write_string('<!--')
+			comments_buffer.write_string(comment.text)
+			comments_buffer.write_string('-->')
+			comments_buffer.write_u8(`\n`)
+		}
+		comments_buffer.str()
+	} else {
+		''
+	}
+
+	document_builder.write_string(prolog)
+	document_builder.write_u8(`\n`)
+	document_builder.write_string(doc.doctype.pretty_str(indent))
+	document_builder.write_u8(`\n`)
+	document_builder.write_string(comments)
+	document_builder.write_string(doc.root.pretty_str(indent, 0, doc.parsed_reverse_entities))
+
+	return document_builder.str()
+}
+
+// str returns a string representation of the XML document. It uses a 2-space indentation
+// to pretty-print the document.
+pub fn (doc XMLDocument) str() string {
+	return doc.pretty_str('  ')
+}

vlib/encoding/xml/entity.v

@@ -0,0 +1,79 @@
+module xml
+
+import strings
+
+pub const default_entities = {
+	'lt':   '<'
+	'gt':   '>'
+	'amp':  '&'
+	'apos': "'"
+	'quot': '"'
+}
+
+pub const default_entities_reverse = {
+	'<': 'lt'
+	'>': 'gt'
+	'&': 'amp'
+	"'": 'apos'
+	'"': 'quot'
+}
+
+[params]
+pub struct EscapeConfig {
+	reverse_entities map[string]string = xml.default_entities_reverse
+}
+
+// escape_text replaces all entities in the given string with their respective
+// XML entity strings. See default_entities, which can be overridden.
+pub fn escape_text(content string, config EscapeConfig) string {
+	mut flattened_entities := []string{cap: 2 * config.reverse_entities.len}
+
+	for target, replacement in config.reverse_entities {
+		flattened_entities << target
+		flattened_entities << '&' + replacement + ';'
+	}
+
+	return content.replace_each(flattened_entities)
+}
+
+[params]
+pub struct UnescapeConfig {
+	entities map[string]string = xml.default_entities
+}
+
+// unescape_text replaces all entities in the given string with their respective
+// original characters or strings. See default_entities_reverse, which can be overridden.
+pub fn unescape_text(content string, config UnescapeConfig) !string {
+	mut buffer := strings.new_builder(content.len)
+	mut index := 0
+	runes := content.runes()
+	for index < runes.len {
+		match runes[index] {
+			`&` {
+				mut offset := 1
+				mut entity_buf := strings.new_builder(8)
+				for index + offset < runes.len && runes[index + offset] != `;` {
+					entity_buf.write_rune(runes[index + offset])
+					offset++
+				}
+				// Did we reach the end of the string?
+				if index + offset == runes.len {
+					return error('Unexpected end of string while parsing entity.')
+				}
+				// Did we find a valid entity?
+				entity := entity_buf.str()
+				if entity in config.entities {
+					buffer.write_string(config.entities[entity])
+					index += offset
+				} else {
+					return error('Unknown entity: ' + entity)
+				}
+			}
+			else {
+				buffer.write_rune(runes[index])
+			}
+		}
+		index++
+	}
+	return buffer.str()
+}

vlib/encoding/xml/entity_test.v

@@ -0,0 +1,35 @@
+module main
+
+import encoding.xml
+
+fn test_escape() {
+	assert xml.escape_text('Normal string') == 'Normal string'
+	assert xml.escape_text('12 < 34') == '12 &lt; 34'
+	assert xml.escape_text('12 > 34') == '12 &gt; 34'
+	assert xml.escape_text('12 & 34') == '12 &amp; 34'
+	assert xml.escape_text('He said, "Very well, let us proceed."') == 'He said, &quot;Very well, let us proceed.&quot;'
+	assert xml.escape_text("He said, 'Very well, let us proceed.'") == 'He said, &apos;Very well, let us proceed.&apos;'
+
+	assert xml.escape_text('Do not escape ©.') == 'Do not escape ©.'
+
+	mut reverse_entities := xml.default_entities_reverse.clone()
+	reverse_entities['©'] = 'copy'
+	assert xml.escape_text('Do escape ©.', reverse_entities: reverse_entities) == 'Do escape &copy;.'
+}
+
+fn test_unescape() ! {
+	assert xml.unescape_text('Normal string')! == 'Normal string'
+	assert xml.unescape_text('12 &lt; 34')! == '12 < 34'
+	assert xml.unescape_text('12 &gt; 34')! == '12 > 34'
+	assert xml.unescape_text('12 &amp; 34')! == '12 & 34'
+	assert xml.unescape_text('He said, &quot;Very well, let us proceed.&quot;')! == 'He said, "Very well, let us proceed."'
+	assert xml.unescape_text('He said, &apos;Very well, let us proceed.&apos;')! == "He said, 'Very well, let us proceed.'"
+
+	xml.unescape_text('12 &invalid; 34') or { assert err.msg() == 'Unknown entity: invalid' }
+
+	xml.unescape_text('Do not unescape &copy;') or { assert err.msg() == 'Unknown entity: copy' }
+
+	mut entities := xml.default_entities.clone()
+	entities['copy'] = '©'
+	assert xml.unescape_text('Do unescape &copy;.', entities: entities)! == 'Do unescape ©.'
+}