diff --git a/healthtools/scrapers/base_scraper.py b/healthtools/scrapers/base_scraper.py index b9b6706..4f5532e 100644 --- a/healthtools/scrapers/base_scraper.py +++ b/healthtools/scrapers/base_scraper.py @@ -67,14 +67,15 @@ def scrape_site(self): print "[{0}] ".format(re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__)) print "[{0}] - Started Scraper.".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")) self.get_total_number_of_pages() - divisor = self.num_pages_to_scrape / 5 + divisor = self.num_pages_to_scrape / 10 for page_num in range(1, self.num_pages_to_scrape + 1): + if page_num == divisor: - print "[{}] - Scraped {}% out of {} pages.".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (float(page_num) / self.num_pages_to_scrape) * 100, self.num_pages_to_scrape) + print "[{}] - Scraped {} out of {} pages.".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), page_num, self.num_pages_to_scrape) divisor = divisor + page_num elif page_num == self.num_pages_to_scrape: - print "[{}] - Scraped {}% out of this {} pages.".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), page_num, self.num_pages_to_scrape) + print "[{}] - Scraped {} out of this {} pages.".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), page_num, self.num_pages_to_scrape) else: test = 0