Skip to content
Permalink
Browse files

Added threadurls for reddit crawler

  • Loading branch information...
noqqe committed Nov 2, 2017
1 parent 61dead2 commit 719dc1b4a95ed53f9a6f86eb3da2f3d933b2488b
Showing with 7 additions and 4 deletions.
  1. +4 −3 crawler/__init__.py
  2. +3 −1 crawler/reddit.py
@@ -226,11 +226,12 @@ def __images_clear(cls):
cls.__images = {} # alternative: cls.__images[:] = [] # be aware: list.clean() is not available in py2

@classmethod
def __add_image(cls, uri, crawler, site):
def __add_image(cls, uri, crawler, site, threadurl=None):
"""
:type uri: str
:type crawler: str
:type site: str
:type threadurl: str
:return: bool
"""
if not cls._is_image(uri):
@@ -247,8 +248,8 @@ def __add_image(cls, uri, crawler, site):
cls.__images[crawler][site] = []

cls._blacklist(uri) # add it to the blacklist to detect duplicates
cls.__images[crawler][site].append("%s#%s" % (uri, crawler))
cls._log("debug", "added %s-%s: %s" % (crawler, site, uri))
cls.__images[crawler][site].append("%s#%s#%s" % (uri, crawler, threadurl))
cls._log("debug", "added %s-%s-%s: %s" % (crawler, site, threadurl, uri))
return True

@classmethod
@@ -46,7 +46,9 @@ def _crawl(self):
for child in data['data']['children']:
image = child['data']['url']
if image:
if self._add_image(image, self.__site):
threadurl = 'https://reddit.com/' + child['data']['permalink']
self.__class__._log("debug", threadurl)
if self._add_image(image, self.__site, threadurl):
images_added += 1

if not images_added:

0 comments on commit 719dc1b

Please sign in to comment.
You can’t perform that action at this time.