remove lxml-dependent code

mitmproxy · Dec 10, 2016 · 293b79a · 293b79a
1 parent a7ba2f7
commit 293b79a
Show file tree

Hide file tree

Showing 11 changed files with 30 additions and 130 deletions.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -27,7 +27,7 @@ test_script:
   - ps: |
       $Env:VERSION = $(python mitmproxy/version.py)
       $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
-      tox -e wheel -- https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl
+      tox -e wheel
       tox -e rtool -- bdist
 
 deploy_script:

diff --git a/examples/simple/modify_body_inject_iframe.py b/examples/simple/modify_body_inject_iframe.py
@@ -11,7 +11,7 @@ def __init__(self, iframe_url):
     def response(self, flow):
         if flow.request.host in self.iframe_url:
             return
-        html = BeautifulSoup(flow.response.content, "lxml")
+        html = BeautifulSoup(flow.response.content)
         if html.body:
             iframe = html.new_tag(
                 "iframe",

diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py
@@ -22,7 +22,7 @@
 from mitmproxy.net import http
 from mitmproxy.utils import strutils
 from . import (
-    auto, raw, hex, json, xml, wbxml, html, javascript, css,
+    auto, raw, hex, json, html_outline, wbxml, javascript, css,
     urlencoded, multipart, image, query, protobuf
 )
 from .base import View, VIEW_CUTOFF, KEY_MAX, format_text, format_dict
@@ -163,10 +163,8 @@ def get_content_view(viewmode: View, data: bytes, **metadata):
 add(raw.ViewRaw())
 add(hex.ViewHex())
 add(json.ViewJSON())
-add(xml.ViewXML())
 add(wbxml.ViewWBXML())
-add(html.ViewHTML())
-add(html.ViewHTMLOutline())
+add(html_outline.ViewHTMLOutline())
 add(javascript.ViewJavaScript())
 add(css.ViewCSS())
 add(urlencoded.ViewURLEncoded())

diff --git a/mitmproxy/contentviews/html.py b/mitmproxy/contentviews/html.py
diff --git a/mitmproxy/contentviews/html_outline.py b/mitmproxy/contentviews/html_outline.py
@@ -0,0 +1,17 @@
+import html2text
+
+from mitmproxy.contentviews import base
+
+
+class ViewHTMLOutline(base.View):
+    name = "HTML Outline"
+    prompt = ("html outline", "o")
+    content_types = ["text/html"]
+
+    def __call__(self, data, **metadata):
+        data = data.decode("utf-8", "replace")
+        h = html2text.HTML2Text(baseurl="")
+        h.ignore_images = True
+        h.body_width = 0
+        outline = h.handle(data)
+        return "HTML Outline", base.format_text(outline)
diff --git a/mitmproxy/contentviews/xml.py b/mitmproxy/contentviews/xml.py
diff --git a/requirements.txt b/requirements.txt
@@ -1,2 +1 @@
-https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5'
 -e .[dev,examples,contentviews]
diff --git a/setup.py b/setup.py
@@ -70,7 +70,6 @@
         "html2text>=2016.1.8, <=2016.9.19",
         "hyperframe>=4.0.1, <5",
         "jsbeautifier>=1.6.3, <1.7",
-        "lxml>=3.5.0, <=3.6.0",  # no wheels for 3.6.1 yet.
         "Pillow>=3.2, <3.5",
         "passlib>=1.6.5, <1.8",
         "pyasn1>=0.1.9, <0.2",

diff --git a/test/mitmproxy/contentviews/test_html.py b/test/mitmproxy/contentviews/test_html.py
diff --git a/test/mitmproxy/contentviews/test_html_outline.py b/test/mitmproxy/contentviews/test_html_outline.py
@@ -0,0 +1,9 @@
+from mitmproxy.contentviews import html_outline
+from test.mitmproxy.contentviews import full_eval
+
+
+def test_view_html_outline():
+    v = full_eval(html_outline.ViewHTMLOutline())
+    s = b"<html><br><br></br><p>one</p></html>"
+    assert v(s)
+    assert v(b'\xfe')
diff --git a/test/mitmproxy/contentviews/test_xml.py b/test/mitmproxy/contentviews/test_xml.py