Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix test warnings #24

Merged
merged 1 commit into from Sep 2, 2020
Merged

Fix test warnings #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
64 changes: 32 additions & 32 deletions tests/test_base_loader.py
View file
Open in desktop
Expand Up @@ -5,7 +5,7 @@
from itemloaders.processors import Compose, Identity, Join, MapCompose, TakeFirst


class TestItemLoader(ItemLoader):
class CustomItemLoader(ItemLoader):
name_in = MapCompose(lambda v: v.title())


Expand All @@ -32,7 +32,7 @@ def test_load_item_using_default_loader(self):
assert item['name'] == ['marta']

def test_load_item_using_custom_loader(self):
il = TestItemLoader()
il = CustomItemLoader()
il.add_value('name', 'marta')
item = il.load_item()
assert item['name'] == ['Marta']
Expand Down Expand Up @@ -83,7 +83,7 @@ def img_url_out(self, values):
assert il.load_item() == {'img_url': '1234.png'}

def test_add_value(self):
il = TestItemLoader()
il = CustomItemLoader()
il.add_value('name', 'marta')
assert il.get_collected_values('name') == ['Marta']
assert il.get_output_value('name') == ['Marta']
Expand All @@ -105,7 +105,7 @@ def test_add_zero(self):
assert il.get_collected_values('name') == [0]

def test_replace_value(self):
il = TestItemLoader()
il = CustomItemLoader()
il.replace_value('name', 'marta')
self.assertEqual(il.get_collected_values('name'), ['Marta'])
self.assertEqual(il.get_output_value('name'), ['Marta'])
Expand Down Expand Up @@ -159,10 +159,10 @@ def filter_world(x):
['HELLO', 'THIS', 'IS', 'SCRAPY'])

def test_map_compose_filter_multil(self):
class TestItemLoader(ItemLoader):
class CustomItemLoader(ItemLoader):
name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])

il = TestItemLoader()
il = CustomItemLoader()
il.add_value('name', 'marta')
self.assertEqual(il.get_output_value('name'), ['Mart'])
item = il.load_item()
Expand All @@ -182,7 +182,7 @@ class InheritDefaultedItemLoader(DefaultedItemLoader):
self.assertEqual(il.get_output_value('name'), ['mart'])

def test_input_processor_inheritance(self):
class ChildItemLoader(TestItemLoader):
class ChildItemLoader(CustomItemLoader):
url_in = MapCompose(lambda v: v.lower())

il = ChildItemLoader()
Expand Down Expand Up @@ -218,8 +218,8 @@ class IdentityDefaultedItemLoader(DefaultedItemLoader):
self.assertEqual(il.get_output_value('name'), ['marta'])

def test_extend_custom_input_processors(self):
class ChildItemLoader(TestItemLoader):
name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
class ChildItemLoader(CustomItemLoader):
name_in = MapCompose(CustomItemLoader.name_in, str.swapcase)

il = ChildItemLoader()
il.add_value('name', 'marta')
Expand All @@ -234,22 +234,22 @@ class ChildDefaultedItemLoader(DefaultedItemLoader):
self.assertEqual(il.get_output_value('name'), ['MART'])

def test_output_processor_using_function(self):
il = TestItemLoader()
il = CustomItemLoader()
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])

class TakeFirstItemLoader(TestItemLoader):
class TakeFirstItemLoader(CustomItemLoader):
name_out = u" ".join

il = TakeFirstItemLoader()
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), 'Mar Ta')

def test_output_processor_error(self):
class TestItemLoader(ItemLoader):
class CustomItemLoader(ItemLoader):
name_out = MapCompose(float)

il = TestItemLoader()
il = CustomItemLoader()
il.add_value('name', ['$10'])
try:
float('$10')
Expand All @@ -269,38 +269,38 @@ class TestItemLoader(ItemLoader):
assert expected_exc_str in s, s

def test_output_processor_using_classes(self):
il = TestItemLoader()
il = CustomItemLoader()
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])

class TakeFirstItemLoader(TestItemLoader):
class TakeFirstItemLoader(CustomItemLoader):
name_out = Join()

il = TakeFirstItemLoader()
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), 'Mar Ta')

class TakeFirstItemLoader(TestItemLoader):
class TakeFirstItemLoader(CustomItemLoader):
name_out = Join("<br>")

il = TakeFirstItemLoader()
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), 'Mar<br>Ta')

def test_default_output_processor(self):
il = TestItemLoader()
il = CustomItemLoader()
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])

class LalaItemLoader(TestItemLoader):
class LalaItemLoader(CustomItemLoader):
default_output_processor = Identity()

il = LalaItemLoader()
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])

def test_loader_context_on_declaration(self):
class ChildItemLoader(TestItemLoader):
class ChildItemLoader(CustomItemLoader):
url_in = MapCompose(processor_with_args, key='val')

il = ChildItemLoader()
Expand All @@ -310,7 +310,7 @@ class ChildItemLoader(TestItemLoader):
self.assertEqual(il.get_output_value('url'), ['val'])

def test_loader_context_on_instantiation(self):
class ChildItemLoader(TestItemLoader):
class ChildItemLoader(CustomItemLoader):
url_in = MapCompose(processor_with_args)

il = ChildItemLoader(key='val')
Expand All @@ -320,7 +320,7 @@ class ChildItemLoader(TestItemLoader):
self.assertEqual(il.get_output_value('url'), ['val'])

def test_loader_context_on_assign(self):
class ChildItemLoader(TestItemLoader):
class ChildItemLoader(CustomItemLoader):
url_in = MapCompose(processor_with_args)

il = ChildItemLoader()
Expand All @@ -334,7 +334,7 @@ def test_item_passed_to_input_processor_functions(self):
def processor(value, loader_context):
return loader_context['item']['name']

class ChildItemLoader(TestItemLoader):
class ChildItemLoader(CustomItemLoader):
url_in = MapCompose(processor)

it = dict(name='marta')
Expand All @@ -345,14 +345,14 @@ class ChildItemLoader(TestItemLoader):
self.assertEqual(il.get_output_value('url'), ['marta'])

# def test_add_value_on_unknown_field(self):
# il = TestItemLoader()
# il = CustomItemLoader()
# self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo'])

def test_compose_processor(self):
class TestItemLoader(ItemLoader):
class CustomItemLoader(ItemLoader):
name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])

il = TestItemLoader()
il = CustomItemLoader()
il.add_value('name', ['marta', 'other'])
self.assertEqual(il.get_output_value('name'), 'Mart')
item = il.load_item()
Expand All @@ -367,12 +367,12 @@ def join(values, sep=None, loader_context=None, ignored=None):
else:
return ''.join(values)

class TestItemLoader(ItemLoader):
class CustomItemLoader(ItemLoader):
name_out = Compose(partial(join, sep='+'))
url_out = Compose(partial(join, loader_context={'sep': '.'}))
summary_out = Compose(partial(join, ignored='foo'))

il = TestItemLoader()
il = CustomItemLoader()
il.add_value('name', ['rabbit', 'hole'])
il.add_value('url', ['rabbit', 'hole'])
il.add_value('summary', ['rabbit', 'hole'])
Expand All @@ -382,24 +382,24 @@ class TestItemLoader(ItemLoader):
self.assertEqual(item['summary'], 'rabbithole')

def test_error_input_processor(self):
class TestItemLoader(ItemLoader):
class CustomItemLoader(ItemLoader):
name_in = MapCompose(float)

il = TestItemLoader()
il = CustomItemLoader()
self.assertRaises(ValueError, il.add_value, 'name',
['marta', 'other'])

def test_error_output_processor(self):
class TestItemLoader(ItemLoader):
class CustomItemLoader(ItemLoader):
name_out = Compose(Join(), float)

il = TestItemLoader()
il = CustomItemLoader()
il.add_value('name', 'marta')
with self.assertRaises(ValueError):
il.load_item()

def test_error_processor_as_argument(self):
il = TestItemLoader()
il = CustomItemLoader()
self.assertRaises(ValueError, il.add_value, 'name',
['marta', 'other'], Compose(float))

Expand Down
30 changes: 15 additions & 15 deletions tests/test_selector_loader.py
View file
Open in desktop
Expand Up @@ -6,7 +6,7 @@
from itemloaders.processors import MapCompose, TakeFirst


class TestItemLoader(ItemLoader):
class CustomItemLoader(ItemLoader):
name_in = MapCompose(lambda v: v.title())


Expand All @@ -23,11 +23,11 @@ class SelectortemLoaderTest(unittest.TestCase):
""")

def test_init_method(self):
loader = TestItemLoader()
loader = CustomItemLoader()
self.assertEqual(loader.selector, None)

def test_init_method_errors(self):
loader = TestItemLoader()
loader = CustomItemLoader()
self.assertRaises(RuntimeError, loader.add_xpath, 'url', '//a/@href')
self.assertRaises(RuntimeError, loader.replace_xpath, 'url', '//a/@href')
self.assertRaises(RuntimeError, loader.get_xpath, '//a/@href')
Expand All @@ -36,14 +36,14 @@ def test_init_method_errors(self):
self.assertRaises(RuntimeError, loader.get_css, '#name::text')

def test_init_method_with_selector(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
self.assertTrue(loader.selector)

loader.add_xpath('name', '//div/text()')
self.assertEqual(loader.get_output_value('name'), ['Marta'])

def test_init_method_with_selector_css(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
self.assertTrue(loader.selector)

loader.add_css('name', 'div::text')
Expand All @@ -60,12 +60,12 @@ def test_init_method_with_selector_css(self):
self.assertEqual(loader.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png'])

def test_add_xpath_re(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
loader.add_xpath('name', '//div/text()', re='ma')
self.assertEqual(loader.get_output_value('name'), ['Ma'])

def test_replace_xpath(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
self.assertTrue(loader.selector)
loader.add_xpath('name', '//div/text()')
self.assertEqual(loader.get_output_value('name'), ['Marta'])
Expand All @@ -76,38 +76,38 @@ def test_replace_xpath(self):
self.assertEqual(loader.get_output_value('name'), ['Paragraph', 'Marta'])

def test_get_xpath(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
self.assertEqual(loader.get_xpath('//p/text()'), ['paragraph'])
self.assertEqual(loader.get_xpath('//p/text()', TakeFirst()), 'paragraph')
self.assertEqual(loader.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa')

self.assertEqual(loader.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta'])

def test_replace_xpath_multi_fields(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
loader.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})
self.assertEqual(loader.get_output_value('name'), ['Marta'])
loader.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})
self.assertEqual(loader.get_output_value('name'), ['Paragraph'])

def test_replace_xpath_re(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
self.assertTrue(loader.selector)
loader.add_xpath('name', '//div/text()')
self.assertEqual(loader.get_output_value('name'), ['Marta'])
loader.replace_xpath('name', '//div/text()', re='ma')
self.assertEqual(loader.get_output_value('name'), ['Ma'])

def test_add_css_re(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
loader.add_css('name', 'div::text', re='ma')
self.assertEqual(loader.get_output_value('name'), ['Ma'])

loader.add_css('url', 'a::attr(href)', re='http://(.+)')
self.assertEqual(loader.get_output_value('url'), ['www.scrapy.org'])

def test_replace_css(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
self.assertTrue(loader.selector)
loader.add_css('name', 'div::text')
self.assertEqual(loader.get_output_value('name'), ['Marta'])
Expand All @@ -123,7 +123,7 @@ def test_replace_css(self):
self.assertEqual(loader.get_output_value('url'), ['/images/logo.png'])

def test_get_css(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
self.assertEqual(loader.get_css('p::text'), [u'paragraph'])
self.assertEqual(loader.get_css('p::text', TakeFirst()), 'paragraph')
self.assertEqual(loader.get_css('p::text', TakeFirst(), re='pa'), u'pa')
Expand All @@ -133,7 +133,7 @@ def test_get_css(self):
[u'http://www.scrapy.org', '/images/logo.png'])

def test_replace_css_multi_fields(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
loader.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x})
self.assertEqual(loader.get_output_value('name'), ['Marta'])
loader.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x})
Expand All @@ -145,7 +145,7 @@ def test_replace_css_multi_fields(self):
self.assertEqual(loader.get_output_value('url'), ['/images/logo.png'])

def test_replace_css_re(self):
loader = TestItemLoader(selector=self.selector)
loader = CustomItemLoader(selector=self.selector)
self.assertTrue(loader.selector)
loader.add_css('url', 'a::attr(href)')
self.assertEqual(loader.get_output_value('url'), ['http://www.scrapy.org'])
Expand Down