Skip to content

Commit

Permalink
Testcase for geocoder user agent
Browse files Browse the repository at this point in the history
  • Loading branch information
wkoot committed May 14, 2020
1 parent 2faaab5 commit a8bb835
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 10 deletions.
13 changes: 3 additions & 10 deletions src/scrapers/tools/geocoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,13 @@


class Geocoder:

def __init__(self):
self.geolocator = Nominatim(user_agent="codefornl-covid19")

def geocode(self):
db = Db()
locationset = db.session.query(InitiativeImport)\
.filter(InitiativeImport.location.isnot(None))\
locationset = db.session.query(InitiativeImport) \
.filter(InitiativeImport.location.isnot(None)) \
.with_for_update().all()

# Regex voor postcode geschreven ls `9999XX`
Expand Down Expand Up @@ -56,10 +55,4 @@ def geocode(self):
print("SUCCESS: " + match.address)
db.session.add(item)
db.session.commit()
time.sleep(1) # Sleep so we don't overstretch the nominatim api






time.sleep(1) # Sleep so we don't overstretch the nominatim api
9 changes: 9 additions & 0 deletions src/tests/test_scraper_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from models.initiatives import InitiativeImportState
from platformen import Scraper
from platformen.scraper import ScrapeException
from tools import Geocoder


class TestDatabase(TestCase):
Expand All @@ -21,6 +22,14 @@ def test_postgres_from_env(self):
assert test_db.get_db_url().startswith("postgres")


class TestGeocoder(TestCase):
def setUp(self):
self.geocoder = Geocoder()

def test_user_agent_set(self):
assert self.geocoder.geolocator.headers.get('User-Agent') is not None


class TestScraper(TestCase):

def setUp(self):
Expand Down

0 comments on commit a8bb835

Please sign in to comment.