Merge pull request #6 from andela-ookoro/morphbranch

reduce time limit

healthtools/scrapers/doctors.py

@@ -1,7 +1,7 @@
 from healthtools.scrapers.base_scraper import Scraper
 from healthtools.config import SITES
 from datetime import datetime
-
+import logging
 
 class DoctorsScraper(Scraper):
     '''
@@ -10,6 +10,7 @@ class DoctorsScraper(Scraper):
 
     def __init__(self):
         super(DoctorsScraper, self).__init__()
+        log = logging.getLogger('__main__')
         self.site_url = SITES["DOCTORS"]
         self.fields = [
             "name", "reg_date", "reg_no", "postal_address", "qualifications",

scraper.py

@@ -43,8 +43,6 @@ def setup_logging(default_level=logging.INFO):
         except Exception as ex:
             log.error('Unable to add slack_logger', str(ex))
 
-
-
 # create a random Id for this scrap instance
 scraper_id = random.randint(1, 100000)
 
@@ -126,7 +124,7 @@ def scrapers():
     scraper_stats.data_archive_key = "stats/stats-{}.json"
     scraper_stats.archive_data(json.dumps(scraping_statistics, indent=4))
     # log warning when scraper ran more than 30 minutes
-    if(m >= 30):
+    if(m >= 10):
         log.warning('Scraper: {} ran for about {}'.format(scraper_id, time_taken))
 
 if __name__ == "__main__":
@@ -135,7 +133,7 @@ def scrapers():
     # Start the scrapers
     scraping = multiprocessing.Process(target=scrapers)
     scraping.start()
-    scraping.join(10*60)
+    scraping.join(10 * 60)
 
     # log error if scraping is still running after 30 minutes
     if scraping.is_alive():