Skip to content

Commit

Permalink
Give up after 10 attempts
Browse files Browse the repository at this point in the history
  • Loading branch information
andylolz committed Oct 15, 2021
1 parent ef9442e commit f8ebc79
Showing 1 changed file with 10 additions and 4 deletions.
14 changes: 10 additions & 4 deletions scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def find_elements(browser, css):
url = base_url + '#iso:pub:PUB500001:en'

load_page(browser, url)
while True:
for x in range(10):
try:
exceptionally_reserved_code_elements_css = 'table[class="grs-grid"] td[class="grs-status4"]'
exceptionally_reserved = find_elements(
Expand All @@ -56,9 +56,11 @@ def find_elements(browser, css):
} for cell in exceptionally_reserved]
break
except StaleElementReferenceException:
pass
print("Retrying ...")
else:
raise Exception("Giving up")

while True:
for x in range(10):
try:
officially_assigned_code_elements_css = 'table[class="grs-grid"] td[class="grs-status1"]'
officially_assigned = find_elements(
Expand All @@ -73,20 +75,24 @@ def find_elements(browser, css):
break
except StaleElementReferenceException:
print("Retrying ...")
else:
raise Exception("Giving up")

for country in countries:
el = 'div[class="core-view-summary"] div[class="core-view-line"] div[class="core-view-field-value"]'
for language in languages:
url = base_url + language + '/' + country['href']
load_page(browser, url)
while True:
for x in range(10):
try:
values = find_elements(browser, el)
country['name_' + language] = values[2].text.rstrip('*')
country['code_3_digit'] = values[4].text
break
except StaleElementReferenceException:
print("Retrying ...")
else:
raise Exception("Giving up")

os.makedirs("output", exist_ok=True)
with open(os.path.join("output", "country_codes.csv"), 'w') as csv_f:
Expand Down

0 comments on commit f8ebc79

Please sign in to comment.