From c159a9aadee619f66efcf221f2519f4f0fbb2538 Mon Sep 17 00:00:00 2001
From: Andrei Vasiliu <whyte.vuhuni@gmail.com>
Date: Sun, 25 Jul 2021 14:54:20 +0300
Subject: [PATCH 1/4] Add tests for escaped and encoded XML

---
 tests/documents/sample_5_full.txt     |  23 ++++++
 tests/documents/sample_5_short.txt    |  15 ++++
 tests/documents/sample_5_utf16bom.xml | Bin 0 -> 662 bytes
 tests/xmlrs_reader_tests.rs           |  97 +++++++++++++++++++++++---
 4 files changed, 127 insertions(+), 8 deletions(-)
 create mode 100644 tests/documents/sample_5_full.txt
 create mode 100644 tests/documents/sample_5_short.txt
 create mode 100644 tests/documents/sample_5_utf16bom.xml

diff --git a/tests/documents/sample_5_full.txt b/tests/documents/sample_5_full.txt
new file mode 100644
index 00000000..b239dbc1
--- /dev/null
+++ b/tests/documents/sample_5_full.txt
@@ -0,0 +1,23 @@
+StartDocument(1.0, utf-16)
+StartElement({urn:x}p:data [z=">"])
+Whitespace("\n    ")
+Comment(" abcd &lt; &gt; &amp; ")
+Whitespace("\n    ")
+StartElement(a)
+Characters("test")
+EndElement(a)
+Whitespace("\n    ")
+StartElement(b)
+Characters("kkss\" = ddd\' >")
+EndElement(b)
+Whitespace("\n    ")
+CData("\n            <a>ddddd</b>!e3--><!-- ddckx\n    ")
+Characters("\n    ")
+StartElement(c)
+EndElement(c)
+Whitespace("\n    ")
+CData("\n    <![CDATA[zzzz]]")
+CData(">")
+Characters("\n")
+EndElement({urn:x}p:data)
+EndDocument
diff --git a/tests/documents/sample_5_short.txt b/tests/documents/sample_5_short.txt
new file mode 100644
index 00000000..b18e9de0
--- /dev/null
+++ b/tests/documents/sample_5_short.txt
@@ -0,0 +1,15 @@
+StartDocument(1.0, utf-16)
+DocType(DOCTYPE data SYSTEM "abcd.dtd")
+StartElement({urn:x}p:data [z=">"])
+StartElement(a)
+Characters("test")
+EndElement(a)
+StartElement(b)
+Characters("kkss\" = ddd\' >")
+EndElement(b)
+Characters("<a>ddddd</b>!e3--><!-- ddckx")
+StartElement(c)
+EndElement(c)
+Characters("<![CDATA[zzzz]]>")
+EndElement({urn:x}p:data)
+EndDocument
diff --git a/tests/documents/sample_5_utf16bom.xml b/tests/documents/sample_5_utf16bom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..7e4c3d06d1df129a3a1a1e661e5892ab194dca00
GIT binary patch
literal 662
zcmb7?K~KU!5QV?9iT|PHfG0#_;-QujA{>n-crk=S>8demZD=)wAFsaIQVMc1lkINb
zym|9>+Mn;3AuHZ_BjAfimXy5kLC$~<=bUn;Jy0em6YiAFdG^^)DxT<xbt%q}gkz3~
z2^n+CHTOJl$3)eRRNPdjsj#Us6K?d@5w?)7Qy(XW6Vj*@L_2l0$6W3ricR;WoJ4YO
z{?;B#w)KH93v<&`1lvvlUEQ|gN?2DnX-=u8P<Yh%wTo6I6;f#mrJgZ;sr_XvguE(B
zg*h5%IUgrFCBDiyHltQIWTr`t8EGaXop-YhtJ!1Uwbrz!`7p&$zApSNtxI!XV`TEP
r+S8mfKk$%A=Rdxz$G+>e=5Jx@b%#SvvdPr{NMEyVzi-{$GPZsJ6VX*f

literal 0
HcmV?d00001

diff --git a/tests/xmlrs_reader_tests.rs b/tests/xmlrs_reader_tests.rs
index d396fc09..d10ad349 100644
--- a/tests/xmlrs_reader_tests.rs
+++ b/tests/xmlrs_reader_tests.rs
@@ -2,6 +2,7 @@ extern crate quick_xml;
 
 use quick_xml::events::{BytesStart, Event};
 use quick_xml::{Reader, Result};
+use std::borrow::Cow;
 use std::str::from_utf8;
 
 #[test]
@@ -71,6 +72,55 @@ fn html5() {
     );
 }
 
+#[test]
+fn escaped_characters() {
+    test(
+        r#"<e attr="&quot;Hello&quot;">&apos;a&apos; &lt; &apos;&amp;&apos;</e>"#,
+        r#"
+            |StartElement(e [attr=""Hello""])
+            |Characters('a' < '&')
+            |EndElement(e)
+            |EndDocument
+        "#,
+        true,
+    )
+}
+
+#[cfg(feature = "escape-html")]
+#[test]
+fn escaped_characters_html() {
+    test(
+        r#"<e attr="&planck;&Egrave;&ell;&#x1D55D;&bigodot;">&boxDR;&boxDL;&#x02554;&#x02557;&#9556;&#9559;</e>"#,
+        r#"
+            |StartElement(e [attr="ℏÈℓ𝕝⨀"])
+            |Characters(╔╗╔╗╔╗)
+            |EndElement(e)
+            |EndDocument
+        "#,
+        true,
+    )
+}
+
+#[cfg(feature = "encoding")]
+#[test]
+fn encoded_characters() {
+    test_bytes(
+        b"\
+            <?xml version = \"1.0\" encoding = \"Shift_JIS\" ?>\n\
+            <a>\x82\xA0\x82\xA2\x82\xA4</a>\
+        ",
+        "
+            |StartDocument(1.0, Shift_JIS)
+            |StartElement(a)
+            |Characters(あいう)
+            |EndElement(a)
+            |EndDocument
+        "
+        .as_bytes(),
+        true,
+    )
+}
+
 // #[test]
 // fn sample_3_short() {
 //     test(
@@ -108,6 +158,19 @@ fn html5() {
 //
 // }
 
+#[test]
+// FIXME: Trips on the first byte-order-mark byte
+// Expected: StartDocument(1.0, utf-16)
+// Found: InvalidUtf8([255, 254]; invalid utf-8 sequence of 1 bytes from index 0)
+#[ignore]
+fn sample_5_short() {
+    test_bytes(
+        include_bytes!("documents/sample_5_utf16bom.xml"),
+        include_bytes!("documents/sample_5_short.txt"),
+        true,
+    );
+}
+
 #[test]
 fn sample_ns_short() {
     test(
@@ -308,19 +371,23 @@ fn default_namespace_applies_to_end_elem() {
 }
 
 fn test(input: &str, output: &str, is_short: bool) {
+    test_bytes(input.as_bytes(), output.as_bytes(), is_short);
+}
+
+fn test_bytes(input: &[u8], output: &[u8], is_short: bool) {
     // Normalize newlines on Windows to just \n, which is what the reader and
     // writer use.
     // let input = input.replace("\r\n", "\n");
     // let input = input.as_bytes();
     // let output = output.replace("\r\n", "\n");
     // let output = output.as_bytes();
-    let mut reader = Reader::from_reader(input.as_bytes());
+    let mut reader = Reader::from_reader(input);
     reader
         .trim_text(is_short)
         .check_comments(true)
         .expand_empty_elements(false);
 
-    let mut spec_lines = SpecIter(output.as_bytes()).enumerate();
+    let mut spec_lines = SpecIter(output).enumerate();
     let mut buf = Vec::new();
     let mut ns_buffer = Vec::new();
 
@@ -332,7 +399,7 @@ fn test(input: &str, output: &str, is_short: bool) {
     loop {
         buf.clear();
         let event = reader.read_namespaced_event(&mut buf, &mut ns_buffer);
-        let line = xmlrs_display(&event);
+        let line = xmlrs_display(&event, &reader);
         if let Some((n, spec)) = spec_lines.next() {
             if spec.trim() == "EndDocument" {
                 break;
@@ -397,10 +464,21 @@ fn make_attrs(e: &BytesStart) -> ::std::result::Result<String, String> {
     Ok(atts.join(", "))
 }
 
-fn xmlrs_display(opt_event: &Result<(Option<&[u8]>, Event)>) -> String {
+// FIXME: The public API differs based on the "encoding" feature
+fn decode<'a>(text: &'a [u8], reader: &Reader<&[u8]>) -> Cow<'a, str> {
+    #[cfg(feature = "encoding")]
+    let decoded = reader.decode(text);
+
+    #[cfg(not(feature = "encoding"))]
+    let decoded = Cow::Borrowed(reader.decode(text).unwrap());
+
+    decoded
+}
+
+fn xmlrs_display(opt_event: &Result<(Option<&[u8]>, Event)>, reader: &Reader<&[u8]>) -> String {
     match opt_event {
         Ok((ref n, Event::Start(ref e))) => {
-            let name = namespace_name(n, e.name());
+            let name = namespace_name(n, decode(e.name(), reader).as_bytes());
             match make_attrs(e) {
                 Ok(ref attrs) if attrs.is_empty() => format!("StartElement({})", &name),
                 Ok(ref attrs) => format!("StartElement({} [{}])", &name, &attrs),
@@ -408,18 +486,21 @@ fn xmlrs_display(opt_event: &Result<(Option<&[u8]>, Event)>) -> String {
             }
         }
         Ok((ref n, Event::Empty(ref e))) => {
-            let name = namespace_name(n, e.name());
+            let name = namespace_name(n, decode(e.name(), reader).as_bytes());
             match make_attrs(e) {
                 Ok(ref attrs) if attrs.is_empty() => format!("EmptyElement({})", &name),
                 Ok(ref attrs) => format!("EmptyElement({} [{}])", &name, &attrs),
                 Err(e) => format!("EmptyElement({}, attr-error: {})", &name, &e),
             }
         }
-        Ok((ref n, Event::End(ref e))) => format!("EndElement({})", namespace_name(n, e.name())),
+        Ok((ref n, Event::End(ref e))) => {
+            let name = namespace_name(n, decode(e.name(), reader).as_bytes());
+            format!("EndElement({})", name)
+        }
         Ok((_, Event::Comment(ref e))) => format!("Comment({})", from_utf8(e).unwrap()),
         Ok((_, Event::CData(ref e))) => format!("CData({})", from_utf8(e).unwrap()),
         Ok((_, Event::Text(ref e))) => match e.unescaped() {
-            Ok(c) => match from_utf8(&*c) {
+            Ok(c) => match from_utf8(decode(&*c, reader).as_bytes()) {
                 Ok(c) => format!("Characters({})", c),
                 Err(ref err) => format!("InvalidUtf8({:?}; {})", e.escaped(), err),
             },

From 8158f29c95e57aed6b57f93b95f1aefc5befdc6d Mon Sep 17 00:00:00 2001
From: Andrei Vasiliu <whyte.vuhuni@gmail.com>
Date: Sun, 25 Jul 2021 15:16:17 +0300
Subject: [PATCH 2/4] Add test for cloned reader

---
 tests/test.rs | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/test.rs b/tests/test.rs
index aeefa2b5..c318d6ac 100644
--- a/tests/test.rs
+++ b/tests/test.rs
@@ -459,6 +459,23 @@ fn test_default_namespace() {
     }
 }
 
+#[test]
+fn test_clone_reader() {
+    let mut reader = Reader::from_str("<tag>text</tag>");
+    reader.trim_text(true);
+    let mut buf = Vec::new();
+
+    assert!(matches!(reader.read_event(&mut buf).unwrap(), Start(_)));
+
+    let mut cloned = reader.clone();
+
+    assert!(matches!(reader.read_event(&mut buf).unwrap(), Text(_)));
+    assert!(matches!(reader.read_event(&mut buf).unwrap(), End(_)));
+
+    assert!(matches!(cloned.read_event(&mut buf).unwrap(), Text(_)));
+    assert!(matches!(cloned.read_event(&mut buf).unwrap(), End(_)));
+}
+
 #[cfg(feature = "serialize")]
 #[test]
 fn line_score() {

From 522d313b0e6038d40da630d56b7f8dce290ece23 Mon Sep 17 00:00:00 2001
From: Andrei Vasiliu <whyte.vuhuni@gmail.com>
Date: Sun, 15 Aug 2021 13:11:06 +0300
Subject: [PATCH 3/4] Add trim tests

---
 tests/test.rs | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/tests/test.rs b/tests/test.rs
index c318d6ac..60657fea 100644
--- a/tests/test.rs
+++ b/tests/test.rs
@@ -459,6 +459,38 @@ fn test_default_namespace() {
     }
 }
 
+#[test]
+fn test_no_trim() {
+    let mut reader = Reader::from_str(" <tag> text </tag> ");
+
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), Text(_)));
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), Start(_)));
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), Text(_)));
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), End(_)));
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), Text(_)));
+}
+
+#[test]
+fn test_trim_end() {
+    let mut reader = Reader::from_str(" <tag> text </tag> ");
+    reader.trim_text_end(true);
+
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), Text(_)));
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), Start(_)));
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), Text(_)));
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), End(_)));
+}
+
+#[test]
+fn test_trim() {
+    let mut reader = Reader::from_str(" <tag> text </tag> ");
+    reader.trim_text(true);
+
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), Start(_)));
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), Text(_)));
+    assert!(matches!(reader.read_event_unbuffered().unwrap(), End(_)));
+}
+
 #[test]
 fn test_clone_reader() {
     let mut reader = Reader::from_str("<tag>text</tag>");

From d368b635561b135132748b7287d53da803ea40c0 Mon Sep 17 00:00:00 2001
From: Andrei Vasiliu <whyte.vuhuni@gmail.com>
Date: Sun, 15 Aug 2021 12:53:06 +0300
Subject: [PATCH 4/4] Fix html5 test

---
 tests/documents/html5-windows.txt |  8 --------
 tests/xmlrs_reader_tests.rs       | 23 +----------------------
 2 files changed, 1 insertion(+), 30 deletions(-)
 delete mode 100644 tests/documents/html5-windows.txt

diff --git a/tests/documents/html5-windows.txt b/tests/documents/html5-windows.txt
deleted file mode 100644
index ae6f701d..00000000
--- a/tests/documents/html5-windows.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-DocType( html)
-Characters(
-)
-StartElement(a, attr-error: error while parsing attribute at position 7: Attribute value must start with a quote.)
-Characters(Hey)
-EndElement(a)
-InvalidUtf8([13, 10, 38, 110, 98, 115, 112, 59, 13, 10]; invalid utf-8 sequence of 1 bytes from index 2)
-EndDocument
diff --git a/tests/xmlrs_reader_tests.rs b/tests/xmlrs_reader_tests.rs
index d10ad349..f7f10ff3 100644
--- a/tests/xmlrs_reader_tests.rs
+++ b/tests/xmlrs_reader_tests.rs
@@ -41,17 +41,8 @@ fn sample_2_full() {
     );
 }
 
-#[cfg(all(not(windows), feature = "escape-html"))]
+#[cfg(feature = "escape-html")]
 #[test]
-// FIXME: Fails with:
-// ```
-// Unexpected event at line 6:
-// Expected: InvalidUtf8([10, 38, 110, 98, 115, 112, 59, 10]; invalid utf-8 sequence of 1 bytes from index 1)
-// Found: Characters(
-//
-// )
-// ```
-#[ignore]
 fn html5() {
     test(
         include_str!("documents/html5.html"),
@@ -60,18 +51,6 @@ fn html5() {
     );
 }
 
-#[cfg(all(windows, feature = "escape-html"))]
-#[test]
-// FIXME: Fails the same way as the one above
-#[ignore]
-fn html5() {
-    test(
-        include_str!("documents/html5.html"),
-        include_str!("documents/html5-windows.txt"),
-        false,
-    );
-}
-
 #[test]
 fn escaped_characters() {
     test(