Add runner for morph.io

.gitignore

@@ -3,5 +3,6 @@ __pycache__/
 .venv
 *.json
 *.jl
+*.sqlite
 .scrapy
 

requirements.txt

@@ -18,6 +18,7 @@ PyHamcrest==1.9.0
 pyOpenSSL==19.0.0
 queuelib==1.5.0
 Scrapy==1.7.3
+scraperwiki==0.5.1
 service-identity==18.1.0
 six==1.12.0
 Twisted==19.7.0

scraper.py

@@ -0,0 +1,37 @@
+import os
+os.environ['SCRAPERWIKI_DATABASE_NAME'] = 'sqlite:///data.sqlite'
+import scraperwiki
+
+import scrapy
+import scrapy.signals
+from scrapy.crawler import CrawlerProcess
+from allris.spiders.meetings import MeetingsSpider
+
+def item_scraped(item, response, spider):
+    record = dict((k, item[k]) for k in ('id', 'name', 'start', 'end', 'web'))
+    scraperwiki.sql.save(['id'], record)
+
+def spider_closed(spider, reason):
+    # close pending transactions
+    scraperwiki.sql.commit_transactions()
+    # save the last url scraped
+    last_url = getattr(spider, 'last_url', '')
+    scraperwiki.sql.save_var('last_url', last_url)
+
+def get_last_url():
+    last_url = scraperwiki.sql.get_var('last_url')
+    if last_url is None:
+        last_url = 'https://ratsinfo.leipzig.de/bi/oparl/1.0/meetings.asp?body=2387&p=100'
+    return last_url
+
+def setup():
+    settings = { 'HTTPCACHE_ENABLED': True, 'LOG_LEVEL': 'INFO'}
+    process = CrawlerProcess(settings)
+    process.crawl(MeetingsSpider, start_url=get_last_url())
+    for p in process.crawlers:
+        p.signals.connect(item_scraped, signal=scrapy.signals.item_scraped)
+        p.signals.connect(spider_closed, signal=scrapy.signals.spider_closed)
+    return process
+
+runner = setup()
+runner.start()