Auto merge of #18255 - CYBAI:prevent-reentrancy-pr18218, r=jdm

Update concept of node filter algorithm

Implement new [filter](https://dom.spec.whatwg.org/#concept-node-filter) algorithm from specification

---
- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [X] These changes fix #18218 (github issue number if applicable).
- [X] These changes do not require tests because changes are minimal and the error was triggered by a test

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/18255)
<!-- Reviewable:end -->

components/script/dom/nodeiterator.rs

@@ -8,7 +8,7 @@ use dom::bindings::codegen::Bindings::NodeFilterBinding::NodeFilter;
 use dom::bindings::codegen::Bindings::NodeFilterBinding::NodeFilterConstants;
 use dom::bindings::codegen::Bindings::NodeIteratorBinding;
 use dom::bindings::codegen::Bindings::NodeIteratorBinding::NodeIteratorMethods;
-use dom::bindings::error::Fallible;
+use dom::bindings::error::{Error, Fallible};
 use dom::bindings::reflector::{Reflector, reflect_dom_object};
 use dom::bindings::root::{Dom, DomRoot, MutDom};
 use dom::document::Document;
@@ -27,6 +27,7 @@ pub struct NodeIterator {
     what_to_show: u32,
     #[ignore_malloc_size_of = "Can't measure due to #6870"]
     filter: Filter,
+    active: Cell<bool>,
 }
 
 impl NodeIterator {
@@ -39,7 +40,8 @@ impl NodeIterator {
             reference_node: MutDom::new(root_node),
             pointer_before_reference_node: Cell::new(true),
             what_to_show: what_to_show,
-            filter: filter
+            filter: filter,
+            active: Cell::new(false),
         }
     }
 
@@ -192,15 +194,29 @@ impl NodeIterator {
     // https://dom.spec.whatwg.org/#concept-node-filter
     fn accept_node(&self, node: &Node) -> Fallible<u16> {
         // Step 1.
-        let n = node.NodeType() - 1;
+        if self.active.get() {
+            return Err(Error::InvalidState);
+        }
         // Step 2.
+        let n = node.NodeType() - 1;
+        // Step 3.
         if (self.what_to_show & (1 << n)) == 0 {
             return Ok(NodeFilterConstants::FILTER_SKIP)
         }
-        // Step 3-5.
+
         match self.filter {
+            // Step 4.
             Filter::None => Ok(NodeFilterConstants::FILTER_ACCEPT),
-            Filter::Callback(ref callback) => callback.AcceptNode_(self, node, Rethrow)
+            Filter::Callback(ref callback) => {
+                // Step 5.
+                self.active.set(true);
+                // Step 6.
+                let result = callback.AcceptNode_(self, node, Rethrow);
+                // Step 7.
+                self.active.set(false);
+                // Step 8.
+                result
+            }
         }
     }
 }

components/script/dom/treewalker.rs

@@ -8,12 +8,13 @@ use dom::bindings::codegen::Bindings::NodeFilterBinding::NodeFilter;
 use dom::bindings::codegen::Bindings::NodeFilterBinding::NodeFilterConstants;
 use dom::bindings::codegen::Bindings::TreeWalkerBinding;
 use dom::bindings::codegen::Bindings::TreeWalkerBinding::TreeWalkerMethods;
-use dom::bindings::error::Fallible;
+use dom::bindings::error::{Error, Fallible};
 use dom::bindings::reflector::{Reflector, reflect_dom_object};
 use dom::bindings::root::{Dom, DomRoot, MutDom};
 use dom::document::Document;
 use dom::node::Node;
 use dom_struct::dom_struct;
+use std::cell::Cell;
 use std::rc::Rc;
 
 // https://dom.spec.whatwg.org/#interface-treewalker
@@ -24,7 +25,8 @@ pub struct TreeWalker {
     current_node: MutDom<Node>,
     what_to_show: u32,
     #[ignore_malloc_size_of = "function pointers and Rc<T> are hard"]
-    filter: Filter
+    filter: Filter,
+    active: Cell<bool>,
 }
 
 impl TreeWalker {
@@ -36,7 +38,8 @@ impl TreeWalker {
             root_node: Dom::from_ref(root_node),
             current_node: MutDom::new(root_node),
             what_to_show: what_to_show,
-            filter: filter
+            filter: filter,
+            active: Cell::new(false),
         }
     }
 
@@ -411,22 +414,30 @@ impl TreeWalker {
 
     // https://dom.spec.whatwg.org/#concept-node-filter
     fn accept_node(&self, node: &Node) -> Fallible<u16> {
-        // "To filter node run these steps:"
-        // "1. Let n be node's nodeType attribute value minus 1."
+        // Step 1.
+        if self.active.get() {
+            return Err(Error::InvalidState);
+        }
+        // Step 2.
         let n = node.NodeType() - 1;
-        // "2. If the nth bit (where 0 is the least significant bit) of whatToShow is not set,
-        //     return FILTER_SKIP."
+        // Step 3.
         if (self.what_to_show & (1 << n)) == 0 {
             return Ok(NodeFilterConstants::FILTER_SKIP)
         }
-        // "3. If filter is null, return FILTER_ACCEPT."
-        // "4. Let result be the return value of invoking filter."
-        // "5. If an exception was thrown, re-throw the exception."
-        // "6. Return result."
         match self.filter {
+            // Step 4.
             Filter::None => Ok(NodeFilterConstants::FILTER_ACCEPT),
             Filter::Native(f) => Ok((f)(node)),
-            Filter::Dom(ref callback) => callback.AcceptNode_(self, node, Rethrow)
+            Filter::Dom(ref callback) => {
+                // Step 5.
+                self.active.set(true);
+                // Step 6.
+                let result = callback.AcceptNode_(self, node, Rethrow);
+                // Step 7.
+                self.active.set(false);
+                // Step 8.
+                result
+            },
         }
     }