From 21d857cc75034d69411fc37dfe6c206f41728c4a Mon Sep 17 00:00:00 2001 From: meticulousfan Date: Wed, 21 Mar 2018 20:06:23 +0100 Subject: [PATCH] update query.py: Errors during JSON serialization are catched. If a request which is supposed to be non-html (JSON) returns an empty response or a html response, it serialization leads to an error. This error is catched with a try / except. This fixes issue https://github.com/taspinar/twitterscraper/issues/93 --- twitterscraper/query.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/twitterscraper/query.py b/twitterscraper/query.py index 0d016e3..861eac8 100644 --- a/twitterscraper/query.py +++ b/twitterscraper/query.py @@ -33,10 +33,14 @@ def query_single_page(url, html_response=True, retry=10): try: response = requests.get(url, headers=headers) if html_response: - html = response.text + html = response.text or '' else: - json_resp = response.json() - html = json_resp['items_html'] + html = '' + try: + json_resp = json.loads(response.text) + html = json_resp['items_html'] or '' + except ValueError as e: + logging.exception('Failed to parse JSON "{}" while requesting "{}"'.format(e, url)) tweets = list(Tweet.from_html(html))