Skip to content

Commit

Permalink
convert to a HashOnly scraper
Browse files Browse the repository at this point in the history
  • Loading branch information
chris48s committed May 12, 2017
1 parent c22aa4f commit 050fffa
Showing 1 changed file with 7 additions and 45 deletions.
52 changes: 7 additions & 45 deletions scraper.py
@@ -1,50 +1,12 @@
from dc_base_scrapers.ckan_scraper import CkanScraper
from dc_base_scrapers.geojson_scraper import GeoJsonScraper
from dc_base_scrapers.hashonly_scraper import HashOnlyScraper


base_url = 'http://dataworks.calderdale.gov.uk/api/3/action/package_show?id='

stations_info = {
'dataset': 'polling-stations',
'extra_fields': [],
'return_format': 'geojson',
}

districts_info = {
'dataset': 'polling-station-districts',
'extra_fields': [
'coverage_start_date',
'coverage_end_date',
],
'return_format': 'geojson'
}

stations_url = "http://dataworks.calderdale.gov.uk/api/3/action/package_show?id=polling-stations"
districts_url = "http://dataworks.calderdale.gov.uk/api/3/action/package_show?id=polling-station-districts"
council_id = 'E08000033'


stations_meta_scraper = CkanScraper(
base_url,
stations_info['dataset'],
stations_info['return_format'],
stations_info['extra_fields'],
'utf-8')
stations_url = stations_meta_scraper.scrape()

districts_meta_scraper = CkanScraper(
base_url,
districts_info['dataset'],
districts_info['return_format'],
districts_info['extra_fields'],
'utf-8')
districts_url = districts_meta_scraper.scrape()


if stations_url:
stations_scraper = GeoJsonScraper(
stations_url, council_id, 'utf-8', 'stations')
stations_scraper.scrape()

if districts_url:
districts_scraper = GeoJsonScraper(
districts_url, council_id, 'utf-8', 'districts')
districts_scraper.scrape()
stations_scraper = HashOnlyScraper(stations_url, council_id, 'stations')
stations_scraper.scrape()
districts_scraper = HashOnlyScraper(districts_url, council_id, 'districts')
districts_scraper.scrape()

0 comments on commit 050fffa

Please sign in to comment.