New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[WIP][web]Static viewer converter for mitmweb #2510
Merged
Merged
Changes from all commits
Commits
Show all changes
13 commits
Select commit
Hold shift + click to select a range
3976ea0
[web] Import static.js in index.html
MatthewShao bf08340
[web] Attempt of static viewer addon.
MatthewShao 1b129b5
[web] Update static_viewer addon.
MatthewShao e925015
[web] Update fetch api to suite the static mode.
MatthewShao 7d92cdf
[web] Hide ContentViewOptions in static mode
MatthewShao cbdddef
[web] change the path name like `_flows` to `flows.json`.
MatthewShao 6560b0d
[web] Change the api routing and minor fix.
MatthewShao 3f49764
[web] Update tests.
MatthewShao 4fb255a
make static viewer more testable, print slow contentviews
mhils d687ebc
[web] Minor fixes.
MatthewShao af30930
[web] Add tests for StaticViewer and minor fixes.
MatthewShao d008f17
[web] Static_viewer coverage ++.
MatthewShao 2baa2c4
[web] Move static viewer from default addon to WebMaster.
MatthewShao File filter
Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
import json | ||
import os.path | ||
import pathlib | ||
import shutil | ||
import time | ||
import typing | ||
|
||
from mitmproxy import contentviews | ||
from mitmproxy import ctx | ||
from mitmproxy import flowfilter | ||
from mitmproxy import io, flow | ||
from mitmproxy.tools.web.app import flow_to_json | ||
|
||
web_dir = pathlib.Path(__file__).absolute().parent.parent / "tools" / "web" | ||
|
||
|
||
def save_static(path: pathlib.Path) -> None: | ||
""" | ||
Save the files for the static web view. | ||
""" | ||
# We want to overwrite the static files to keep track of the update. | ||
if (path / "static").exists(): | ||
shutil.rmtree(str(path / "static")) | ||
shutil.copytree(str(web_dir / "static"), str(path / "static")) | ||
shutil.copyfile(str(web_dir / 'templates' / 'index.html'), str(path / "index.html")) | ||
|
||
with open(str(path / "static" / "static.js"), "w") as f: | ||
f.write("MITMWEB_STATIC = true;") | ||
|
||
|
||
def save_filter_help(path: pathlib.Path) -> None: | ||
with open(str(path / 'filter-help.json'), 'w') as f: | ||
json.dump(dict(commands=flowfilter.help), f) | ||
|
||
|
||
def save_flows(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: | ||
with open(str(path / 'flows.json'), 'w') as f: | ||
json.dump( | ||
[flow_to_json(f) for f in flows], | ||
f | ||
) | ||
|
||
|
||
def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: | ||
for f in flows: | ||
for m in ('request', 'response'): | ||
message = getattr(f, m) | ||
message_path = path / "flows" / f.id / m | ||
os.makedirs(str(message_path / "content"), exist_ok=True) | ||
|
||
with open(str(message_path / '_content'), 'wb') as content_file: | ||
# don't use raw_content here as this is served with a default content type | ||
if message: | ||
content_file.write(message.content) | ||
else: | ||
content_file.write(b'No content.') | ||
|
||
# content_view | ||
t = time.time() | ||
if message: | ||
description, lines, error = contentviews.get_message_content_view( | ||
'Auto', message | ||
) | ||
else: | ||
description, lines = 'No content.', [] | ||
if time.time() - t > 0.1: | ||
ctx.log( | ||
"Slow content view: {} took {}s".format( | ||
description.strip(), | ||
round(time.time() - t, 1) | ||
), | ||
"info" | ||
) | ||
with open(str(message_path / "content" / "Auto.json"), "w") as content_view_file: | ||
json.dump( | ||
dict(lines=list(lines), description=description), | ||
content_view_file | ||
) | ||
|
||
|
||
class StaticViewer: | ||
# TODO: make this a command at some point. | ||
def load(self, loader): | ||
loader.add_option( | ||
"web_static_viewer", typing.Optional[str], "", | ||
"The path to output a static viewer." | ||
) | ||
|
||
def configure(self, updated): | ||
if "web_static_viewer" in updated and ctx.options.web_static_viewer: | ||
flows = io.read_flows_from_paths([ctx.options.rfile]) | ||
p = pathlib.Path(ctx.options.web_static_viewer).expanduser() | ||
self.export(p, flows) | ||
|
||
def export(self, path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: | ||
save_static(path) | ||
save_filter_help(path) | ||
save_flows(path, flows) | ||
save_flows_content(path, flows) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
import json | ||
from unittest import mock | ||
|
||
from mitmproxy.test import taddons | ||
from mitmproxy.test import tflow | ||
|
||
from mitmproxy import flowfilter | ||
from mitmproxy.tools.web.app import flow_to_json | ||
|
||
from mitmproxy.addons import static_viewer | ||
from mitmproxy.addons import save | ||
|
||
|
||
def test_save_static(tmpdir): | ||
tmpdir.mkdir('static') | ||
static_viewer.save_static(tmpdir) | ||
assert len(tmpdir.listdir()) == 2 | ||
assert tmpdir.join('index.html').check(file=1) | ||
assert tmpdir.join('static/static.js').read() == 'MITMWEB_STATIC = true;' | ||
|
||
|
||
def test_save_filter_help(tmpdir): | ||
static_viewer.save_filter_help(tmpdir) | ||
f = tmpdir.join('/filter-help.json') | ||
assert f.check(file=1) | ||
assert f.read() == json.dumps(dict(commands=flowfilter.help)) | ||
|
||
|
||
def test_save_flows(tmpdir): | ||
flows = [tflow.tflow(req=True, resp=None), tflow.tflow(req=True, resp=True)] | ||
static_viewer.save_flows(tmpdir, flows) | ||
assert tmpdir.join('flows.json').check(file=1) | ||
assert tmpdir.join('flows.json').read() == json.dumps([flow_to_json(f) for f in flows]) | ||
|
||
|
||
@mock.patch('mitmproxy.ctx.log') | ||
def test_save_flows_content(ctx, tmpdir): | ||
flows = [tflow.tflow(req=True, resp=None), tflow.tflow(req=True, resp=True)] | ||
with mock.patch('time.time', mock.Mock(side_effect=[1, 2, 2] * 4)): | ||
static_viewer.save_flows_content(tmpdir, flows) | ||
flows_path = tmpdir.join('flows') | ||
assert len(flows_path.listdir()) == len(flows) | ||
for p in flows_path.listdir(): | ||
assert p.join('request').check(dir=1) | ||
assert p.join('response').check(dir=1) | ||
assert p.join('request/_content').check(file=1) | ||
assert p.join('request/content').check(dir=1) | ||
assert p.join('response/_content').check(file=1) | ||
assert p.join('response/content').check(dir=1) | ||
assert p.join('request/content/Auto.json').check(file=1) | ||
assert p.join('response/content/Auto.json').check(file=1) | ||
|
||
|
||
def test_static_viewer(tmpdir): | ||
s = static_viewer.StaticViewer() | ||
sa = save.Save() | ||
with taddons.context() as tctx: | ||
sa.save([tflow.tflow(resp=True)], str(tmpdir.join('foo'))) | ||
tctx.master.addons.add(s) | ||
tctx.configure(s, web_static_viewer=str(tmpdir), rfile=str(tmpdir.join('foo'))) | ||
assert tmpdir.join('index.html').check(file=1) | ||
assert tmpdir.join('static').check(dir=1) | ||
assert tmpdir.join('flows').check(dir=1) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The variable
f
has a collision with the filetype var on line 51, let's make itflow
here.