Skip to content
Permalink
Browse files

Ignore None's values when using the ItemLoader.

  • Loading branch information
rmax authored and dangra committed Feb 5, 2014
1 parent cde9a8c commit 06325467d814d4111f6a46c12c4cb919e3e09174
Showing with 32 additions and 1 deletion.
  1. +3 −1 scrapy/contrib/loader/__init__.py
  2. +29 −0 scrapy/tests/test_contrib_loader.py
@@ -81,7 +81,9 @@ def get_value(self, value, *processors, **kw):
def load_item(self):
item = self.item
for field_name in self._values:
item[field_name] = self.get_output_value(field_name)
value = self.get_output_value(field_name)
if value is not None:
item[field_name] = value
return item

def get_output_value(self, field_name):
@@ -57,6 +57,35 @@ def test_load_item_using_custom_loader(self):
item = il.load_item()
self.assertEqual(item['name'], [u'Marta'])

def test_load_item_ignore_none_field_values(self):
def validate_sku(value):
# Let's assume a SKU is only digits.
if value.isdigit():
return value

class MyLoader(ItemLoader):
name_out = Compose(lambda vs: vs[0]) # take first which allows empty values
price_out = Compose(TakeFirst(), float)
sku_out = Compose(TakeFirst(), validate_sku)

valid_fragment = u'SKU: 1234'
invalid_fragment = u'SKU: not available'
sku_re = 'SKU: (.+)'

il = MyLoader(item={})
# Should not return "sku: None".
il.add_value('sku', [invalid_fragment], re=sku_re)
# Should not ignore empty values.
il.add_value('name', u'')
il.add_value('price', [u'0'])
self.assertEqual(il.load_item(), {
'name': u'',
'price': 0.0,
})

il.replace_value('sku', [valid_fragment], re=sku_re)
self.assertEqual(il.load_item()['sku'], u'1234')

def test_add_value(self):
il = TestItemLoader()
il.add_value('name', u'marta')

0 comments on commit 0632546

Please sign in to comment.
You can’t perform that action at this time.