Add a little check for possible extra rows

openaustralia · Apr 12, 2022 · 72533e3 · 72533e3
1 parent d8d3cf8
commit 72533e3
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/scraper.py b/scraper.py
@@ -29,6 +29,10 @@
     bs4 = BeautifulSoup(page, "html.parser")
 
     rows = bs4.find(id='ContentPlaceHolderBody_gridViewLocalities').find_all('tr')
+    if len(rows) == 20:
+        # These postcodes probably will have extra pages
+        print("WARNING: Maximum number of rows reached on postcode {}".format(postcode))
+
     for row in rows:
         tds = row.find_all('td')
         if len(tds) == 6: