From 9f2b7f7b93acf40cee3c39c436d4234db276bce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 18 Feb 2019 13:52:48 +0100 Subject: [PATCH] new: Add error page --- Pipfile.lock | 134 +++++++++++++++---------------- lookyloo/exceptions.py | 6 +- lookyloo/helpers.py | 1 - lookyloo/lookyloo.py | 20 +++-- website/web/__init__.py | 10 ++- website/web/templates/error.html | 9 +++ 6 files changed, 98 insertions(+), 82 deletions(-) create mode 100644 website/web/templates/error.html diff --git a/Pipfile.lock b/Pipfile.lock index 9a953908..84a966aa 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -83,40 +83,36 @@ }, "cffi": { "hashes": [ - "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", - "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", - "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", - "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", - "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", - "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", - "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", - "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", - "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", - "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", - "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", - "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", - "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", - "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", - "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", - "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", - "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", - "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", - "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", - "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", - "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", - "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", - "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", - "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", - "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", - "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", - "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", - "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", - "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", - "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", - "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", - "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" - ], - "version": "==1.11.5" + "sha256:0b5f895714a7a9905148fc51978c62e8a6cbcace30904d39dcd0d9e2265bb2f6", + "sha256:27cdc7ba35ee6aa443271d11583b50815c4bb52be89a909d0028e86c21961709", + "sha256:2d4a38049ea93d5ce3c7659210393524c1efc3efafa151bd85d196fa98fce50a", + "sha256:3262573d0d60fc6b9d0e0e6e666db0e5045cbe8a531779aa0deb3b425ec5a282", + "sha256:358e96cfffc185ab8f6e7e425c7bb028931ed08d65402fbcf3f4e1bff6e66556", + "sha256:37c7db824b5687fbd7ea5519acfd054c905951acc53503547c86be3db0580134", + "sha256:39b9554dfe60f878e0c6ff8a460708db6e1b1c9cc6da2c74df2955adf83e355d", + "sha256:42b96a77acf8b2d06821600fa87c208046decc13bd22a4a0e65c5c973443e0da", + "sha256:5b37dde5035d3c219324cac0e69d96495970977f310b306fa2df5910e1f329a1", + "sha256:5d35819f5566d0dd254f273d60cf4a2dcdd3ae3003dfd412d40b3fe8ffd87509", + "sha256:5df73aa465e53549bd03c819c1bc69fb85529a5e1a693b7b6cb64408dd3970d1", + "sha256:7075b361f7a4d0d4165439992d0b8a3cdfad1f302bf246ed9308a2e33b046bd3", + "sha256:7678b5a667b0381c173abe530d7bdb0e6e3b98e062490618f04b80ca62686d96", + "sha256:7dfd996192ff8a535458c17f22ff5eb78b83504c34d10eefac0c77b1322609e2", + "sha256:8a3be5d31d02c60f84c4fd4c98c5e3a97b49f32e16861367f67c49425f955b28", + "sha256:9812e53369c469506b123aee9dcb56d50c82fad60c5df87feb5ff59af5b5f55c", + "sha256:9b6f7ba4e78c52c1a291d0c0c0bd745d19adde1a9e1c03cb899f0c6efd6f8033", + "sha256:a85bc1d7c3bba89b3d8c892bc0458de504f8b3bcca18892e6ed15b5f7a52ad9d", + "sha256:aa6b9c843ad645ebb12616de848cc4e25a40f633ccc293c3c9fe34107c02c2ea", + "sha256:bae1aa56ee00746798beafe486daa7cfb586cd395c6ce822ba3068e48d761bc0", + "sha256:bae96e26510e4825d5910a196bf6b5a11a18b87d9278db6d08413be8ea799469", + "sha256:bd78df3b594013b227bf31d0301566dc50ba6f40df38a70ded731d5a8f2cb071", + "sha256:c2711197154f46d06f73542c539a0ff5411f1951fab391e0a4ac8359badef719", + "sha256:d998c20e3deed234fca993fd6c8314cb7cbfda05fd170f1bd75bb5d7421c3c5a", + "sha256:df4f840d77d9e37136f8e6b432fecc9d6b8730f18f896e90628712c793466ce6", + "sha256:f5653c2581acb038319e6705d4e3593677676df14b112f13e0b5b44b6a18df1a", + "sha256:f7c7aa485a2e2250d455148470ffd0195eecc3d845122635202d7467d6f7b4cf", + "sha256:f9e2c66a6493147de835f207f198540a56b26745ce4f272fbc7c2f2cfebeb729" + ], + "version": "==1.12.1" }, "chardet": { "hashes": [ @@ -250,7 +246,7 @@ "har2tree": { "editable": true, "git": "https://github.com/viper-framework/har2tree.git", - "ref": "98305ff26ddfb7ebc7f5fe28239351db00900175" + "ref": "a90418ace183e5ebf8891bebe0a7efc7a8d910f7" }, "hyperlink": { "hashes": [ @@ -293,34 +289,34 @@ }, "lxml": { "hashes": [ - "sha256:0dd6589fa75d369ba06d2b5f38dae107f76ea127f212f6a7bee134f6df2d1d21", - "sha256:1afbac344aa68c29e81ab56c1a9411c3663157b5aee5065b7fa030b398d4f7e0", - "sha256:1baad9d073692421ad5dbbd81430aba6c7f5fdc347f03537ae046ddf2c9b2297", - "sha256:1d8736421a2358becd3edf20260e41a06a0bf08a560480d3a5734a6bcbacf591", - "sha256:1e1d9bddc5afaddf0de76246d3f2152f961697ad7439c559f179002682c45801", - "sha256:1f179dc8b2643715f020f4d119d5529b02cd794c1c8f305868b73b8674d2a03f", - "sha256:241fb7bdf97cb1df1edfa8f0bcdfd80525d4023dac4523a241907c8b2f44e541", - "sha256:2f9765ee5acd3dbdcdc0d0c79309e01f7c16bc8d39b49250bf88de7b46daaf58", - "sha256:312e1e1b1c3ce0c67e0b8105317323e12807955e8186872affb667dbd67971f6", - "sha256:3273db1a8055ca70257fd3691c6d2c216544e1a70b673543e15cc077d8e9c730", - "sha256:34dfaa8c02891f9a246b17a732ca3e99c5e42802416628e740a5d1cb2f50ff49", - "sha256:3aa3f5288af349a0f3a96448ebf2e57e17332d99f4f30b02093b7948bd9f94cc", - "sha256:51102e160b9d83c1cc435162d90b8e3c8c93b28d18d87b60c56522d332d26879", - "sha256:56115fc2e2a4140e8994eb9585119a1ae9223b506826089a3ba753a62bd194a6", - "sha256:69d83de14dbe8fe51dccfd36f88bf0b40f5debeac763edf9f8325180190eba6e", - "sha256:99fdce94aeaa3ccbdfcb1e23b34273605c5853aa92ec23d84c84765178662c6c", - "sha256:a7c0cd5b8a20f3093ee4a67374ccb3b8a126743b15a4d759e2a1bf098faac2b2", - "sha256:abe12886554634ed95416a46701a917784cb2b4c77bfacac6916681d49bbf83d", - "sha256:b4f67b5183bd5f9bafaeb76ad119e977ba570d2b0e61202f534ac9b5c33b4485", - "sha256:bdd7c1658475cc1b867b36d5c4ed4bc316be8d3368abe03d348ba906a1f83b0e", - "sha256:c6f24149a19f611a415a51b9bc5f17b6c2f698e0d6b41ffb3fa9f24d35d05d73", - "sha256:d1e111b3ab98613115a208c1017f266478b0ab224a67bc8eac670fa0bad7d488", - "sha256:d6520aa965773bbab6cb7a791d5895b00d02cf9adc93ac2bf4edb9ac1a6addc5", - "sha256:dd185cde2ccad7b649593b0cda72021bc8a91667417001dbaf24cd746ecb7c11", - "sha256:de2e5b0828a9d285f909b5d2e9d43f1cf6cf21fe65bc7660bdaa1780c7b58298", - "sha256:f726444b8e909c4f41b4fde416e1071cf28fa84634bfb4befdf400933b6463af" - ], - "version": "==4.3.0" + "sha256:0537eee4902e8bf4f41bfee8133f7edf96533dd175930a12086d6a40d62376b2", + "sha256:0562ec748abd230ab87d73384e08fa784f9b9cee89e28696087d2d22c052cc27", + "sha256:09e91831e749fbf0f24608694e4573be0ef51430229450c39c83176cc2e2d353", + "sha256:1ae4c0722fc70c0d4fba43ae33c2885f705e96dce1db41f75ae14a2d2749b428", + "sha256:1c630c083d782cbaf1f7f37f6cac87bda9cff643cf2803a5f180f30d97955cef", + "sha256:2fe74e3836bd8c0fa7467ffae05545233c7f37de1eb765cacfda15ad20c6574a", + "sha256:37af783c2667ead34a811037bda56a0b142ac8438f7ed29ae93f82ddb812fbd6", + "sha256:3f2d9eafbb0b24a33f56acd16f39fc935756524dcb3172892721c54713964c70", + "sha256:47d8365a8ef14097aa4c65730689be51851b4ade677285a3b2daa03b37893e26", + "sha256:510e904079bc56ea784677348e151e1156040dbfb736f1d8ea4b9e6d0ab2d9f4", + "sha256:58d0851da422bba31c7f652a7e9335313cf94a641aa6d73b8f3c67602f75b593", + "sha256:7940d5c2185ffb989203dacbb28e6ae88b4f1bb25d04e17f94b0edd82232bcbd", + "sha256:7cf39bb3a905579836f7a8f3a45320d9eb22f16ab0c1e112efb940ced4d057a5", + "sha256:9563a23c1456c0ab550c087833bc13fcc61013a66c6420921d5b70550ea312bf", + "sha256:95b392952935947e0786a90b75cc33388549dcb19af716b525dae65b186138fc", + "sha256:983129f3fd3cef5c3cf067adcca56e30a169656c00fcc6c648629dbb850b27fa", + "sha256:a0b75b1f1854771844c647c464533def3e0a899dd094a85d1d4ed72ecaaee93d", + "sha256:b5db89cc0ef624f3a81214b7961a99f443b8c91e88188376b6b322fd10d5b118", + "sha256:c0a7751ba1a4bfbe7831920d98cee3ce748007eab8dfda74593d44079568219a", + "sha256:c0c5a7d4aafcc30c9b6d8613a362567e32e5f5b708dc41bc3a81dac56f8af8bb", + "sha256:d4d63d85eacc6cb37b459b16061e1f100d154bee89dc8d8f9a6128a5a538e92e", + "sha256:da5e7e941d6e71c9c9a717c93725cda0708c2474f532e3680ac5e39ec57d224d", + "sha256:dccad2b3c583f036f43f80ac99ee212c2fa9a45151358d55f13004d095e683b2", + "sha256:df46307d39f2aeaafa1d25309b8a8d11738b73e9861f72d4d0a092528f498baa", + "sha256:e70b5e1cb48828ddd2818f99b1662cb9226dc6f57d07fc75485405c77da17436", + "sha256:ea825562b8cd057cbc9810d496b8b5dec37a1e2fc7b27bc7c1e72ce94462a09a" + ], + "version": "==4.3.1" }, "markupsafe": { "hashes": [ @@ -428,11 +424,11 @@ }, "redis": { "hashes": [ - "sha256:74c892041cba46078ae1ef845241548baa3bd3634f9a6f0f952f006eb1619c71", - "sha256:7ba8612bbfd966dea8c62322543fed0095da2834dbd5a7c124afbc617a156aa7" + "sha256:724932360d48e5407e8f82e405ab3650a36ed02c7e460d1e6fddf0f038422b54", + "sha256:9b19425a38fd074eb5795ff2b0d9a55b46a44f91f5347995f27e3ad257a7d775" ], "index": "pypi", - "version": "==3.1.0" + "version": "==3.2.0" }, "requests": { "hashes": [ @@ -477,10 +473,10 @@ }, "soupsieve": { "hashes": [ - "sha256:466910df7561796a60748826781ebe9a888f7a1668a636ae86783f44d10aae73", - "sha256:87db12ae79194f0ff9808d2b1641c4f031ae39ffa3cab6b907ea7c1e5e5ed445" + "sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b", + "sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26" ], - "version": "==1.7.3" + "version": "==1.8" }, "twisted": { "hashes": [ diff --git a/lookyloo/exceptions.py b/lookyloo/exceptions.py index 7a63706c..f0f29b2f 100644 --- a/lookyloo/exceptions.py +++ b/lookyloo/exceptions.py @@ -3,8 +3,12 @@ class LookylooException(Exception): - pass + pass class MissingEnv(LookylooException): pass + + +class NoValidHarFile(LookylooException): + pass diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py index a6a828e5..37fd106f 100644 --- a/lookyloo/helpers.py +++ b/lookyloo/helpers.py @@ -43,7 +43,6 @@ def get_socket_path(name: str) -> str: def check_running(name: str) -> bool: socket_path = get_socket_path(name) - print(socket_path) try: r = Redis(unix_socket_path=socket_path) if r.ping(): diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 21b5c5b1..bb77f9dc 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -4,7 +4,7 @@ import json from scrapysplashwrapper import crawl -from har2tree import CrawledTree +from har2tree import CrawledTree, Har2TreeError import pickle from datetime import datetime @@ -21,6 +21,7 @@ from pathlib import Path from .helpers import get_homedir, get_socket_path +from .exceptions import NoValidHarFile from redis import Redis import logging @@ -109,13 +110,16 @@ def process_scrape_queue(self): def load_tree(self, report_dir: Path): har_files = sorted(report_dir.glob('*.har')) - ct = CrawledTree(har_files) - ct.find_parents() - ct.join_trees() - temp = tempfile.NamedTemporaryFile(prefix='lookyloo', delete=False) - pickle.dump(ct, temp) - temp.close() - return temp.name, ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url + try: + ct = CrawledTree(har_files) + ct.find_parents() + ct.join_trees() + temp = tempfile.NamedTemporaryFile(prefix='lookyloo', delete=False) + pickle.dump(ct, temp) + temp.close() + return temp.name, ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url + except Har2TreeError as e: + raise NoValidHarFile(e.message) def cleanup_old_tmpfiles(self): for tmpfile in pathlib.Path(tempfile.gettempdir()).glob('lookyloo*'): diff --git a/website/web/__init__.py b/website/web/__init__.py index 7f67946d..730da195 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -12,6 +12,7 @@ from lookyloo.helpers import get_homedir from lookyloo.lookyloo import Lookyloo +from lookyloo.exceptions import NoValidHarFile app = Flask(__name__) @@ -127,9 +128,12 @@ def tree(tree_uuid): if not report_dir: return redirect(url_for('index')) - tree_json, start_time, user_agent, root_url = load_tree(report_dir) - return render_template('tree.html', tree_json=tree_json, start_time=start_time, - user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid) + try: + tree_json, start_time, user_agent, root_url = load_tree(report_dir) + return render_template('tree.html', tree_json=tree_json, start_time=start_time, + user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid) + except NoValidHarFile as e: + return render_template('error.html', error_message=e.message) @app.route('/', methods=['GET']) diff --git a/website/web/templates/error.html b/website/web/templates/error.html new file mode 100644 index 00000000..6011227b --- /dev/null +++ b/website/web/templates/error.html @@ -0,0 +1,9 @@ +{% extends "main.html" %} +{% block title %}Error{% endblock %} + +{% block content %} +
+

Something went wrong

+ {{ error_message }} +
+{% endblock %}