instagram scraping: skip GraphSuggestedUserFeedUnit nodes

https://console.cloud.google.com/errors/12882644530379539744?project=instagram-atom
snarfed · Mar 20, 2017 · 93c8c9f · 93c8c9f
1 parent 4d41c1f
commit 93c8c9f
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 3 deletions.
diff --git a/granary/instagram.py b/granary/instagram.py
@@ -560,7 +560,7 @@ def media_to_object(self, media):
     }
 
     for version in ('standard_resolution', 'low_resolution', 'thumbnail'):
-      image = media.get('images').get(version)
+      image = media.get('images', {}).get(version)
       if image:
         object['image'] = {'url': image.get('url')}
         break
@@ -766,8 +766,10 @@ def html_to_activities(self, html):
       medias.extend(page.get('feed', {}).get('media', {}).get('nodes', []))
       # new schema
       edges = page.get('graphql', {}).get('user', {})\
-                        .get('edge_web_feed_timeline', {}).get('edges', [])
-      medias.extend(e.get('node') for e in edges)
+                  .get('edge_web_feed_timeline', {}).get('edges', [])
+      medias.extend(e.get('node') for e in edges
+                    if e.get('node', {}).get('__typename') not in
+                    ('GraphSuggestedUserFeedUnit',))
 
     # profiles
     for page in entry_data.get('ProfilePage', []):

diff --git a/granary/test/test_instagram.py b/granary/test/test_instagram.py
@@ -684,6 +684,21 @@ def tag_uri(name):
   },
 })
 
+HTML_SUGGESTED_USERS = {
+  'suggested_users': [{
+    'id': '123',
+    'username': 'ms_person',
+    'full_name': 'Ms Person',
+    'biography': 'a person who did stuff',
+    'profile_pic_url': 'https://scontent.cdninstagram.com/t51.2885-19/s150x150/13398501_243709166011988_1998688411_a.jpg',
+    'edge_followed_by': {'count': 106},
+    'is_private': True,
+    'is_verified': False,
+    'is_viewer': False,
+  }],
+  '__typename': 'GraphSuggestedUserFeedUnit',
+}
+
 HTML_FEED_OLD = {  # eg https://www.instagram.com/ when you're logged in
   'environment_switcher_visible_server_guess': True,
   'config': {
@@ -758,6 +773,7 @@ def tag_uri(name):
       'edges': [
         {'node': HTML_PHOTO_FULL_NEW},
         {'node': HTML_VIDEO_FULL_NEW},
+        {'node': HTML_SUGGESTED_USERS},
       ],
     },
   }}}]},