Skip to content

Commit

Permalink
Update scraper.py
Browse files Browse the repository at this point in the history
  • Loading branch information
mxfh committed Oct 26, 2014
1 parent aa46cc5 commit 1df3696
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions scraper.py
Expand Up @@ -2,6 +2,7 @@
import re
import urlparse
import datetime
import os
from bs4 import BeautifulSoup # BeautifulSoup helps you to find what you want in HTML

def urlEncodeNonAscii(b):
Expand All @@ -16,8 +17,9 @@ def iriToUri(iri):

scraperwiki.sqlite.execute("drop table if exists swdata")

domain = MORPH_DOMAIN
url = doamin + "index.php/demonstrationen"
if 'MORPH_DOMAIN' in os.environ:
domain = os.environ['MORPH_DOMAIN']
url = domain + "index.php/demonstrationen"
html = scraperwiki.scrape(url) # download the html content of the page
soup = BeautifulSoup(html) # load the html into beautifulsoup

Expand Down

0 comments on commit 1df3696

Please sign in to comment.