Skip to content

Commit

Permalink
Cast values when loading fom datapackage
Browse files Browse the repository at this point in the history
  • Loading branch information
akariv committed May 20, 2019
1 parent 97b92e0 commit fcff0dd
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 7 deletions.
2 changes: 1 addition & 1 deletion dataflows/processors/load.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ def process_datapackage(self, dp: Package):
for resource in self.load_dp.resources:
if resource_matcher.match(resource.name):
self.resource_descriptors.append(resource.descriptor)
self.iterators.append(resource.iter(keyed=True, cast=False))
self.iterators.append(resource.iter(keyed=True, cast=True))

# Loading for any other source
else:
Expand Down
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ def read(*paths):
},
zip_safe=False,
long_description=README,
long_description_content_type='text/markdown',
description='A nifty data processing framework, based on data packages',
author='Adam Kariv',
author_email='adam.kariv@gmail.com',
Expand Down
12 changes: 6 additions & 6 deletions tests/test_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ def test_load_from_package():
from dataflows import dump_to_path, load

Flow(
[{'foo': 'bar'}],
[{'foo': 'bar', 'moo': 12}],
dump_to_path('data/load_from_package')
).process()

Expand All @@ -446,7 +446,7 @@ def test_load_from_package():
).datastream()

assert len(ds.dp.resources) == 1
assert [list(res) for res in ds.res_iter] == [[{'foo': 'bar'}]]
assert [list(res) for res in ds.res_iter] == [[{'foo': 'bar', 'moo': 12}]]


def test_load_from_env_var():
Expand All @@ -455,10 +455,10 @@ def test_load_from_env_var():

Flow(
[{'foo': 'bar'}],
dump_to_path('data/load_from_package')
dump_to_path('data/load_from_env_var')
).process()

os.environ['MY_DATAPACKAGE'] = 'data/load_from_package/datapackage.json'
os.environ['MY_DATAPACKAGE'] = 'data/load_from_env_var/datapackage.json'
results, dp, _ = Flow(
load('env://MY_DATAPACKAGE')
).results()
Expand All @@ -473,11 +473,11 @@ def test_load_from_package_resource_matching():
Flow(
[{'foo': 'bar'}],
[{'foo': 'baz'}],
dump_to_path('data/load_from_package')
dump_to_path('data/load_from_package_resource_matching(')
).process()

ds = Flow(
load('data/load_from_package/datapackage.json', resources=['res_2'])
load('data/load_from_package_resource_matching(/datapackage.json', resources=['res_2'])
).datastream()

assert len(ds.dp.resources) == 1
Expand Down

0 comments on commit fcff0dd

Please sign in to comment.