diff --git a/petl/io/avro.py b/petl/io/avro.py index 91c88ae6..745c395b 100644 --- a/petl/io/avro.py +++ b/petl/io/avro.py @@ -7,7 +7,7 @@ from datetime import datetime, date, time from decimal import Decimal -from petl.compat import izip, izip_longest, text_type, string_types, PY3 +from petl.compat import izip_longest, text_type, string_types, PY3 from petl.io.sources import read_source_from_arg, write_source_from_arg from petl.transform.headers import skip, setheader from petl.util.base import Table, dicts, fieldnames, iterpeek, wrap @@ -15,7 +15,7 @@ # region API -def fromavro(source, limit=None, skip=0, **avro_args): +def fromavro(source, limit=None, skips=0, **avro_args): """Extract a table from the records of a avro file. The `source` argument (string or file-like or fastavro.reader) can either @@ -92,7 +92,7 @@ def fromavro(source, limit=None, skip=0, **avro_args): source2 = read_source_from_arg(source) return AvroView(source=source2, limit=limit, - skip=skip, + skips=skips, **avro_args) @@ -242,10 +242,10 @@ def appendavro(table, target, schema=None, sample=9, **avro_args): class AvroView(Table): '''Read rows from avro file with their types and logical types''' - def __init__(self, source, limit, skip, **avro_args): + def __init__(self, source, limit, skips, **avro_args): self.source = source self.limit = limit - self.skip = skip + self.skip = skips self.avro_args = avro_args self.avro_schema = None @@ -457,7 +457,6 @@ def _get_definition_from_record(prop, val, fprev, dprev, fill_missing): fprev = OrderedDict() if dprev is None: dprev = OrderedDict() - props = list(val.keys()) row = list(val.values()) @@ -475,9 +474,9 @@ def _get_definition_from_record(prop, val, fprev, dprev, fill_missing): def _get_precision_from_decimal(curr, val, prev): if val is None: - prec = scale = bytes_req = num = 0 + prec = scale = 0 else: - prec, scale, bytes_req, num = precision_and_scale(val) + prec, scale, _, _ = precision_and_scale(val) if prev is not None: # get the greatests precision and scale of the sample prec0, scale0 = prev.get('precision'), prev.get('scale') @@ -508,7 +507,7 @@ def precision_and_scale(numeric_value): def _fix_missing_headers(table, schema): '''add missing columns headers from schema''' - if schema is None or not 'fields' in schema: + if schema is None or 'fields' not in schema: return table # table2: try not advance iterators sample, table2 = iterpeek(table, 2) @@ -541,6 +540,7 @@ def _get_schema_header_names(schema): header = [field.get('name') for field in fields] return header + def _raise_error(details): if PY3: raise ValueError(details).with_traceback(sys.exc_info()[2]) diff --git a/petl/test/io/test_avro.py b/petl/test/io/test_avro.py index d8530373..d44f34dc 100644 --- a/petl/test/io/test_avro.py +++ b/petl/test/io/test_avro.py @@ -4,7 +4,7 @@ import sys import math -from datetime import datetime, date, time +from datetime import datetime, date from decimal import Decimal from tempfile import NamedTemporaryFile @@ -12,7 +12,7 @@ from petl.compat import izip_longest, PY3 from petl.transform.basics import cat -from petl.util.base import dicts, records +from petl.util.base import dicts from petl.util.vis import look from petl.io.avro import fromavro, toavro, appendavro @@ -105,7 +105,7 @@ def test_toavro_troubleshooting(): wrong_schema = dict(schema0) schema_fields = wrong_schema['fields'] for field in schema_fields: - field['type'] = ['null', 'string'] + field['type'] = ['null', 'string'] try: _write_temp_avro_file(table1, wrong_schema) except ValueError: @@ -196,7 +196,6 @@ def _eq2(ve, va, re, ra): def _decs(float_value, rounding=12): return Decimal(str(round(float_value, rounding))) - def _utc(year, month, day, hour=0, minute=0, second=0, microsecond=0): u = datetime(year, month, day, hour, minute, second, microsecond) if PY3: