example project

example_project/example_project/items.py

@@ -0,0 +1,11 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# http://doc.scrapy.org/en/latest/topics/items.html
+
+from scrapy.item import Item, Field
+
+class ExampleProjectItem(Item):
+    # define the fields for your item here like:
+    # name = Field()
+    pass

example_project/example_project/pipelines.py

@@ -0,0 +1,8 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
+
+class ExampleProjectPipeline(object):
+    def process_item(self, item, spider):
+        return item

example_project/example_project/settings.py

@@ -0,0 +1,35 @@
+# Scrapy settings for example_project project
+#
+# For simplicity, this file contains only the most important settings by
+# default. All the other settings are documented here:
+#
+#     http://doc.scrapy.org/en/latest/topics/settings.html
+#
+
+BOT_NAME = 'example_project'
+
+SPIDER_MODULES = ['example_project.spiders']
+NEWSPIDER_MODULE = 'example_project.spiders'
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'example_project (+http://www.yourdomain.com)'
+
+import os
+if os.environ.get("SENTRY_DSN", None) is None:
+    import sys
+    print >> sys.stderr, "Please define SENTRY_DSN in your environment to run this example_project"
+    exit(1)
+
+# log into sentry
+SENTRY_DSN = os.environ["SENTRY_DSN"]  # << set your sentry_dsn here >> 
+
+import scrapy_sentry
+scrapy_sentry.init(SENTRY_DSN)
+
+SPIDER_MIDDLEWARES = {
+    'scrapy_sentry.middlewares.SentryMiddleware': 10,
+}
+
+DOWNLOADER_MIDDLEWARES = {
+    'scrapy_sentry.middlewares.SentryMiddleware': 10,
+}

example_project/example_project/spiders/__init__.py

@@ -0,0 +1,5 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.
+

example_project/example_project/spiders/example.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+
+from scrapy.spider import BaseSpider
+
+class NitidumSpider(BaseSpider):
+
+    name = 'example'
+    allowed_domains = ['localhost']
+    start_urls = ['http://localhost/']
+
+    def parse(self, response):
+      raise Exception("this is an exception in the spider")

example_project/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# http://doc.scrapy.org/en/latest/topics/scrapyd.html
+
+[settings]
+default = example_project.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = example_project