instagram: improve HTML scraping error handling

https://console.cloud.google.com/errors/4217523877087496950

README.md

@@ -290,6 +290,8 @@ Changelog
 ### 1.7 - unreleased
 * Twitter:
     * Linkify @-mentions and hashtags in `preview_create()`.
+* Instagram:
+    * Improve HTML scraping error handling.
 
 ### 1.6 - 2016-11-26
 * Twitter:

granary/instagram.py

@@ -722,6 +722,7 @@ def html_to_activities(self, html):
     script_start = '<script type="text/javascript">window._sharedData = '
     start = html.find(script_start)
     if start == -1:
+      logging.warning('JSON script tag not found! Raw HTML:\n' + html)
       return [], None
 
     # App Engine's Python 2.7.5 json module doesn't support unpaired surrogate
@@ -736,7 +737,10 @@ def html_to_activities(self, html):
       json_module = json
 
     start += len(script_start)
-    end = html.index(';</script>', start)
+    end = html.find(';</script>', start)
+    if end == -1:
+      logging.warning('JSON script close tag not found! Raw HTML:\n' + html)
+      return [], None
     data = json_module.loads(html[start:end])
 
     entry_data = data.get('entry_data', {})

granary/test/test_instagram.py

@@ -772,6 +772,8 @@ def tag_uri(name):
 HTML_PROFILE_PRIVATE_COMPLETE = HTML_HEADER + json.dumps(HTML_PROFILE_PRIVATE) + HTML_FOOTER
 HTML_PHOTO_COMPLETE = HTML_HEADER + json.dumps(HTML_PHOTO_PAGE) + HTML_FOOTER
 HTML_VIDEO_COMPLETE = HTML_HEADER + json.dumps(HTML_VIDEO_PAGE) + HTML_FOOTER
+HTML_PHOTO_MISSING_HEADER = json.dumps(HTML_PHOTO_PAGE) + HTML_FOOTER
+HTML_PHOTO_MISSING_FOOTER = HTML_HEADER + json.dumps(HTML_PHOTO_PAGE)
 
 
 class InstagramTest(testutil.HandlerTest):
@@ -1363,6 +1365,16 @@ def test_html_to_activities_missing_video_url(self):
     del expected[1]['object']['attachments'][0]['stream'][0]['url']
     self.assert_equals(expected, activities)
 
+  def test_html_to_activities_missing_header(self):
+    activities, viewer = self.instagram.html_to_activities(HTML_PHOTO_MISSING_HEADER)
+    self.assert_equals([], activities)
+    self.assertIsNone(viewer)
+
+  def test_html_to_activities_missing_footer(self):
+    activities, viewer = self.instagram.html_to_activities(HTML_PHOTO_MISSING_FOOTER)
+    self.assert_equals([], activities)
+    self.assertIsNone(viewer)
+
   def test_id_to_shortcode(self):
     for shortcode, id in (
         (None, None),