🗓 Apr 4, 2022 7:46:29 PM

🔍 move more parsel based methods to plugins

chepy/modules/extractors.py

@@ -199,18 +199,6 @@ def extract_domains(self, is_binary: bool = False) -> ExtractorsT:
         self.state = matched
         return self
 
-    @ChepyDecorators.call_stack
-    def html_comments(self) -> ExtractorsT:
-        """Extract html comments
-
-        Returns:
-            Chepy: The Chepy object.
-        """
-        self.state = list(
-            filter(lambda x: x != "", self._parsel_obj().xpath("//comment()").getall())
-        )
-        return self
-
     @ChepyDecorators.call_stack
     def javascript_comments(self) -> ExtractorsT:
         """Extract javascript comments
@@ -225,36 +213,6 @@ def javascript_comments(self) -> ExtractorsT:
         )
         return self
 
-    @ChepyDecorators.call_stack
-    def html_tags(self, tag: str) -> ExtractorsT:
-        """Extract tags from html along with their attributes
-
-        Args:
-            tag (str): A HTML tag
-
-        Returns:
-            Chepy: The Chepy object.
-
-        Examples:
-            >>> Chepy("http://example.com").http_request().html_tags('p').o
-            [
-                {'tag': 'p', 'attributes': {}},
-                {'tag': 'p', 'attributes': {}},
-                {'tag': 'p', 'attributes': {}}
-            ]
-        """
-        tags = []
-
-        for element in self._parsel_obj().xpath("//{}".format(tag)):
-            attributes = []
-            for index, attribute in enumerate(element.xpath("@*"), start=1):
-                attribute_name = element.xpath("name(@*[%d])" % index).extract_first()
-                attributes.append((attribute_name, attribute.extract()))
-            tags.append({"tag": tag, "attributes": dict(attributes)})
-
-        self.state = tags
-        return self
-
     @ChepyDecorators.call_stack
     def extract_google_api(self) -> ExtractorsT:
         """Extract Goolge api keys

chepy/modules/extractors.pyi

@@ -13,9 +13,7 @@ class Extractors(ChepyCore):
     def extract_mac_address(self: ExtractorsT, is_binary: bool=...) -> ExtractorsT: ...
     def extract_urls(self: ExtractorsT, is_binary: bool=...) -> ExtractorsT: ...
     def extract_domains(self: ExtractorsT, is_binary: bool=...) -> ExtractorsT: ...
-    def html_comments(self: ExtractorsT) -> ExtractorsT: ...
     def javascript_comments(self: ExtractorsT) -> ExtractorsT: ...
-    def html_tags(self: ExtractorsT, tag: str) -> ExtractorsT: ...
     def extract_google_api(self: ExtractorsT) -> ExtractorsT: ...
     def extract_google_captcha(self: ExtractorsT) -> ExtractorsT: ...
     def extract_google_oauth(self: ExtractorsT) -> ExtractorsT: ...

tests/test_extractors.py

@@ -66,21 +66,10 @@ def test_extract_domains():
     )
 
 
-def test_html_comments():
-    assert len(Chepy("tests/files/test.html").load_file().html_comments().o) == 3
-
-
 def test_js_comments():
     assert len(Chepy("tests/files/test.js").load_file().javascript_comments().o) == 3
 
 
-def test_html_tag():
-    assert Chepy("tests/files/test.html").load_file().html_tags("p").o == [
-        {"tag": "p", "attributes": {"someval": "someval", "ano-ther": "another"}},
-        {"tag": "p", "attributes": {}},
-    ]
-
-
 def test_extract_basicauth():
     assert (
         len(Chepy("tests/files/fake_secrets.txt").read_file().extract_auth_basic().o)

tests_plugins/test_additionalextractors.py

@@ -49,3 +49,14 @@ def test_beautify_xml():
         len(Chepy("tests/files/test.xml").load_file().minify_xml().beautify_xml().o)
         == 7690
     )
+
+
+def test_html_tag():
+    assert Chepy("tests/files/test.html").load_file().html_tags("p").o == [
+        {"tag": "p", "attributes": {"someval": "someval", "ano-ther": "another"}},
+        {"tag": "p", "attributes": {}},
+    ]
+
+
+def test_html_comments():
+    assert len(Chepy("tests/files/test.html").load_file().html_comments().o) == 3