Skip to content

Commit

Permalink
change time mark to 10 minute for testing purpose
Browse files Browse the repository at this point in the history
  • Loading branch information
andela-ookoro committed Nov 2, 2017
1 parent b1b8223 commit 5226f74
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions scraper.py
Expand Up @@ -114,20 +114,20 @@ def scrapers():
# record end time
end_time = time.time()
timeSent = (end_time - start_time) / (60)
if(timeSent >= 30):
if(timeSent >= 15):
log.warning('Scraper: {} ran for about {} minutes'.format(scraper_id, timeSent))

if __name__ == "__main__":
import multiprocessing
# Start the scrapers
scraping = multiprocessing.Process(target=scrapers)
scraping.start()
scraping.join(30*60)
scraping.join(10*60)

# log error if scraping is still running after 30 minutes
if scraping.is_alive():
# create a random Id for this scrap instance
import random
scraper_id = random.randint(1, 100000)
log.warning('Scraper: {} is running for more than 30 minutes'.format(scraper_id))
log.warning('Scraper: {} is running for more than 10 minutes'.format(scraper_id))

0 comments on commit 5226f74

Please sign in to comment.