Skip to content

Commit

Permalink
Make FileFormat respect missing values (#129)
Browse files Browse the repository at this point in the history
* Make FileFormat respect missing values

* Fixed the implementation

* Fixed tests
  • Loading branch information
roll committed Apr 6, 2020
1 parent 88f1c03 commit 1fa2c56
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 2 deletions.
4 changes: 2 additions & 2 deletions dataflows/helpers/iterable_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@ def field_type(self, values):
for value in values:
if isinstance(value, str):
types.add('string')
elif isinstance(value, bool):
types.add('boolean')
elif isinstance(value, int):
types.add('integer')
elif isinstance(value, (float, decimal.Decimal)):
types.add('number')
elif isinstance(value, bool):
types.add('boolean')
elif isinstance(value, list):
types.add('array')
elif isinstance(value, dict):
Expand Down
5 changes: 5 additions & 0 deletions dataflows/processors/dumpers/file_formats.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ def __init__(self, writer, schema, temporal_format_property=None):
self.headers = [f.name for f in schema.fields]
self.fields = dict((f.name, f) for f in schema.fields)
self.temporal_format_property = temporal_format_property
self.missing_values = schema.descriptor.get('missingValues', [])

# Set fields' serializers
for field in schema.fields:
Expand Down Expand Up @@ -62,6 +63,10 @@ def __transform_row(self, row):
def __transform_value(self, value, field):
if value is None:
return self.NULL_VALUE
# It supports a `tableschema`'s mode of perserving missing values
# https://github.com/frictionlessdata/tableschema-py#experimental
if value in self.missing_values:
return value
return field.descriptor['serializer'](value)

def write_transformed_row(self, *_):
Expand Down

0 comments on commit 1fa2c56

Please sign in to comment.