diff --git a/chapter16/multiprocess_queue.py b/chapter16/multiprocess_queue.py index caea096..df55f30 100644 --- a/chapter16/multiprocess_queue.py +++ b/chapter16/multiprocess_queue.py @@ -21,6 +21,8 @@ def task_delegator(taskQueue, foundUrlsQueue): for link in links: #Add new link to the taskQueue taskQueue.put(link) + #Add new link to the visited list + visited.append(link) def get_links(bsObj): links = bsObj.find('div', {'id':'bodyContent'}).find_all('a', href=re.compile('^(/wiki/)((?!:).)*$')) @@ -51,4 +53,4 @@ def scrape_article(taskQueue, foundUrlsQueue): processes.append(Process(target=scrape_article, args=(taskQueue, foundUrlsQueue,))) for p in processes: - p.start() \ No newline at end of file + p.start()