Skip to content

Commit

Permalink
error fix
Browse files Browse the repository at this point in the history
  • Loading branch information
omorilewa committed Jun 30, 2017
1 parent 1c4a821 commit 88c4627
Showing 1 changed file with 7 additions and 4 deletions.
11 changes: 7 additions & 4 deletions healthtools/scrapers/base_scraper.py
Expand Up @@ -66,14 +66,18 @@ def scrape_site(self):
'''
Scrape the whole site
'''
test = 1
try:
print "[{0}] ".format(re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__))
except Exception as err:
print err
print "[{0}] - Started Scraper.".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

all_results = []
delete_batch = []
skipped_pages = 0
widgets = [progressbar.Percentage(), progressbar.Bar()]

print "[{0}] ".format(re.sub(r"(\w)([A-Z])", r"\1 \2", type(self).__name__))
print "[{0}] - Started Scraper.".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

self.get_total_number_of_pages()
bar = progressbar.ProgressBar(widgets=widgets, max_value=10).start()
# for i in range(20):
Expand All @@ -82,7 +86,6 @@ def scrape_site(self):
divisor = self.num_pages_to_scrape / 10

for page_num in range(1, self.num_pages_to_scrape + 1):

if page_num == divisor:
time.sleep(1)
bar.update(i + 1)
Expand Down

0 comments on commit 88c4627

Please sign in to comment.