Permalink
Browse files

promoted DjangoItem to main contrib

  • Loading branch information...
1 parent 52151e8 commit 70f8e517a19c177e1546157a39a96d00c8c814aa @pablohoffman pablohoffman committed Aug 29, 2012
@@ -16,7 +16,4 @@ it's properly merged) . Use at your own risk.
This documentation is a work in progress. Use at your own risk.
-.. toctree::
- :maxdepth: 1
-
- djangoitems
+*No experimental features at this time*
View
@@ -136,6 +136,7 @@ Solving specific problems
topics/ubuntu
topics/scrapyd
topics/jobs
+ topics/djangoitem
:doc:`faq`
Get answers to most frequently asked questions.
@@ -164,6 +165,9 @@ Solving specific problems
:doc:`topics/jobs`
Learn how to pause and resume crawls for large spiders.
+:doc:`topics/djangoitem`
+ Write scraped items using Django models.
+
.. _extending-scrapy:
Extending Scrapy
View
@@ -26,6 +26,7 @@ Scrapy changes:
- ``USER_AGENT`` spider attribute will no longer work, use ``user_agent`` attribute instead
- ``DOWNLOAD_TIMEOUT`` spider attribute will no longer work, use ``download_timeout`` attribute instead
- removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
+- promoted :ref:`topics-djangoitem` to main contrib
Scrapyd changes:
@@ -1,6 +1,6 @@
.. _topics-djangoitem:
-.. module:: scrapy.contrib_exp.djangoitem
+.. module:: scrapy.contrib.djangoitem
==========
DjangoItem
@@ -87,3 +87,12 @@ And we can override the fields of the model with your own::
This is usefull to provide properties to the field, like a default or any other
property that your project uses.
+
+DjangoItem caveats
+==================
+
+DjangoItem is a rather convenient way to integrate Scrapy projects with Django
+models, but bear in mind that Django ORM may not scale well if you scrape a lot
+of items (ie. millions) with Scrapy. This is because a relational backend is
+often not a good choice for a write intensive application (such as a web
+crawler), specially if the database is highly normalized and with many indices.
@@ -1,33 +1,6 @@
-from scrapy.item import Field, Item, ItemMeta
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib_exp.djangoitem` is deprecated, use `scrapy.contrib.djangoitem` instead",
+ ScrapyDeprecationWarning, stacklevel=2)
-
-class DjangoItemMeta(ItemMeta):
-
- def __new__(mcs, class_name, bases, attrs):
- cls = super(DjangoItemMeta, mcs).__new__(mcs, class_name, bases, attrs)
- cls.fields = cls.fields.copy()
-
- if cls.django_model:
- cls._model_fields = []
- cls._model_meta = cls.django_model._meta
- for model_field in cls._model_meta.fields:
- if model_field.auto_created == False:
- if model_field.name not in cls.fields:
- cls.fields[model_field.name] = Field()
- cls._model_fields.append(model_field.name)
- return cls
-
-
-class DjangoItem(Item):
-
- __metaclass__ = DjangoItemMeta
-
- django_model = None
-
- def save(self, commit=True):
- modelargs = dict((k, self.get(k)) for k in self._values
- if k in self._model_fields)
- model = self.django_model(**modelargs)
- if commit:
- model.save()
- return model
+from scrapy.contrib.djangoitem import DjangoItem
@@ -1,7 +1,7 @@
import os
from twisted.trial import unittest
-from scrapy.contrib_exp.djangoitem import DjangoItem, Field
+from scrapy.contrib.djangoitem import DjangoItem, Field
os.environ['DJANGO_SETTINGS_MODULE'] = 'scrapy.tests.test_djangoitem.settings'

0 comments on commit 70f8e51

Please sign in to comment.