From b3c295bfc854ae99b35dbce8939ac645389e2f16 Mon Sep 17 00:00:00 2001 From: Augusto Men Date: Mon, 10 Jul 2023 16:53:39 -0400 Subject: [PATCH] Fix other functions to conform with PEP 479 Applies the same fix that was done for iterrowslice to other functions that may receive an empty (as in headerless) table as input. https://github.com/petl-developers/petl/pull/613 --- petl/io/csv_py2.py | 5 +- petl/io/csv_py3.py | 17 ++-- petl/io/html.py | 33 ++++--- petl/io/pandas.py | 5 +- petl/io/pickle.py | 10 ++- petl/io/text.py | 10 ++- petl/io/xls.py | 15 ++-- petl/io/xlsx.py | 18 ++-- petl/test/helpers.py | 2 +- petl/test/io/test_html.py | 18 ++++ petl/test/io/test_pandas.py | 6 ++ petl/test/io/test_pickle.py | 24 +++-- petl/test/io/test_text.py | 15 ++++ petl/test/io/test_xls.py | 11 ++- petl/test/io/test_xlsx.py | 10 +++ petl/test/transform/test_basics.py | 112 ++++++++++++++++++++++++ petl/test/transform/test_conversions.py | 15 ++++ petl/test/transform/test_dedup.py | 19 ++++ petl/test/transform/test_fills.py | 21 +++++ petl/test/transform/test_headers.py | 59 +++++++++++++ petl/test/transform/test_maps.py | 33 +++++++ petl/test/transform/test_regex.py | 26 +++++- petl/test/transform/test_reshape.py | 30 +++++++ petl/test/transform/test_selects.py | 16 ++++ petl/test/transform/test_sorts.py | 21 ++++- petl/test/transform/test_unpacks.py | 9 ++ petl/test/transform/test_validation.py | 11 +++ petl/test/util/test_base.py | 53 ++++++++++- petl/test/util/test_lookups.py | 15 +++- petl/test/util/test_materialise.py | 21 +++++ petl/test/util/test_vis.py | 7 ++ petl/transform/basics.py | 71 ++++++++++++--- petl/transform/conversions.py | 9 +- petl/transform/dedup.py | 22 ++++- petl/transform/fills.py | 15 +++- petl/transform/headers.py | 30 +++++-- petl/transform/maps.py | 15 +++- petl/transform/regex.py | 20 ++++- petl/transform/reshape.py | 15 +++- petl/transform/selects.py | 17 +++- petl/transform/sorts.py | 19 +++- petl/transform/unpacks.py | 10 ++- petl/transform/validation.py | 5 +- petl/util/base.py | 25 ++++-- petl/util/lookups.py | 25 ++++-- petl/util/materialise.py | 10 ++- petl/util/vis.py | 20 ++++- 47 files changed, 878 insertions(+), 117 deletions(-) diff --git a/petl/io/csv_py2.py b/petl/io/csv_py2.py index a7057ad3..13ad48b8 100644 --- a/petl/io/csv_py2.py +++ b/petl/io/csv_py2.py @@ -112,7 +112,10 @@ def __iter__(self): it = iter(self.table) # deal with header row - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return if self.write_header: writer.writerow(hdr) # N.B., always yield header, even if we don't write it diff --git a/petl/io/csv_py3.py b/petl/io/csv_py3.py index 9b800baa..cf228c1c 100644 --- a/petl/io/csv_py3.py +++ b/petl/io/csv_py3.py @@ -20,15 +20,15 @@ def fromcsv_impl(source, **kwargs): class CSVView(Table): def __init__(self, source, encoding, errors, header, **csvargs): - self.source = source - self.encoding = encoding - self.errors = errors - self.csvargs = csvargs - self.header = header + self.source = source + self.encoding = encoding + self.errors = errors + self.csvargs = csvargs + self.header = header def __iter__(self): if self.header is not None: - yield tuple(self.header) + yield tuple(self.header) with self.source.open('rb') as buf: csvfile = io.TextIOWrapper(buf, encoding=self.encoding, errors=self.errors, newline='') @@ -86,7 +86,10 @@ def __iter__(self): try: writer = csv.writer(csvfile, **self.csvargs) it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return if self.write_header: writer.writerow(hdr) yield tuple(hdr) diff --git a/petl/io/html.py b/petl/io/html.py index fe6ef07c..029723e4 100644 --- a/petl/io/html.py +++ b/petl/io/html.py @@ -75,7 +75,10 @@ def tohtml(table, source=None, encoding=None, errors='strict', caption=None, it = iter(table) # write header - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] _write_begin(f, hdr, lineterminator, caption, index_header, truncate) @@ -166,10 +169,13 @@ def __iter__(self): it = iter(table) # write header - hdr = next(it) + try: + hdr = next(it) + yield hdr + except StopIteration: + hdr = [] _write_begin(f, hdr, lineterminator, caption, index_header, truncate) - yield hdr # write body if tr_style and callable(tr_style): @@ -193,16 +199,17 @@ def _write_begin(f, flds, lineterminator, caption, index_header, truncate): f.write("" + lineterminator) if caption is not None: f.write(('' % caption) + lineterminator) - f.write('' + lineterminator) - f.write('' + lineterminator) - for i, h in enumerate(flds): - if index_header: - h = '%s|%s' % (i, h) - if truncate: - h = h[:truncate] - f.write(('' % h) + lineterminator) - f.write('' + lineterminator) - f.write('' + lineterminator) + if flds: + f.write('' + lineterminator) + f.write('' + lineterminator) + for i, h in enumerate(flds): + if index_header: + h = '%s|%s' % (i, h) + if truncate: + h = h[:truncate] + f.write(('' % h) + lineterminator) + f.write('' + lineterminator) + f.write('' + lineterminator) f.write('' + lineterminator) diff --git a/petl/io/pandas.py b/petl/io/pandas.py index f3632f0e..68961670 100644 --- a/petl/io/pandas.py +++ b/petl/io/pandas.py @@ -29,7 +29,10 @@ def todataframe(table, index=None, exclude=None, columns=None, """ import pandas as pd it = iter(table) - header = next(it) + try: + header = next(it) + except StopIteration: + header = None # Will create an Empty DataFrame if columns is None: columns = header return pd.DataFrame.from_records(it, index=index, exclude=exclude, diff --git a/petl/io/pickle.py b/petl/io/pickle.py index 0bf77802..e7ba4da6 100644 --- a/petl/io/pickle.py +++ b/petl/io/pickle.py @@ -119,7 +119,10 @@ def _writepickle(table, source, mode, protocol, write_header): source = write_source_from_arg(source, mode) with source.open(mode) as f: it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return if write_header: pickle.dump(hdr, f, protocol) for row in it: @@ -153,7 +156,10 @@ def __iter__(self): source = write_source_from_arg(self.source) with source.open('wb') as f: it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return if self.write_header: pickle.dump(hdr, f, protocol) yield tuple(hdr) diff --git a/petl/io/text.py b/petl/io/text.py index 570d2efe..f9f79126 100644 --- a/petl/io/text.py +++ b/petl/io/text.py @@ -194,7 +194,10 @@ def _writetext(table, source, mode, encoding, errors, template, prologue, if prologue is not None: f.write(prologue) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) for row in it: rec = asdict(flds, row) @@ -266,7 +269,10 @@ def _iterteetext(table, source, encoding, errors, template, prologue, epilogue): if prologue is not None: f.write(prologue) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return yield tuple(hdr) flds = list(map(text_type, hdr)) for row in it: diff --git a/petl/io/xls.py b/petl/io/xls.py index b9a4599b..8b6559fa 100644 --- a/petl/io/xls.py +++ b/petl/io/xls.py @@ -85,12 +85,15 @@ def toxls(tbl, filename, sheet, encoding=None, style_compression=0, else: # handle styles it = iter(tbl) - hdr = next(it) - flds = list(map(str, hdr)) - for c, f in enumerate(flds): - ws.write(0, c, label=f) - if f not in styles or styles[f] is None: - styles[f] = xlwt.Style.default_style + try: + hdr = next(it) + flds = list(map(str, hdr)) + for c, f in enumerate(flds): + ws.write(0, c, label=f) + if f not in styles or styles[f] is None: + styles[f] = xlwt.Style.default_style + except StopIteration: + pass # no header written # convert to list for easy zipping styles = [styles[f] for f in flds] for r, row in enumerate(it): diff --git a/petl/io/xlsx.py b/petl/io/xlsx.py index c1f8bfac..612f7c36 100644 --- a/petl/io/xlsx.py +++ b/petl/io/xlsx.py @@ -113,9 +113,12 @@ def toxlsx(tbl, filename, sheet=None, write_header=True, mode="replace"): ws = _insert_sheet_on_workbook(mode, sheet, wb) if write_header: it = iter(tbl) - hdr = next(it) - flds = list(map(text_type, hdr)) - rows = itertools.chain([flds], it) + try: + hdr = next(it) + flds = list(map(text_type, hdr)) + rows = itertools.chain([flds], it) + except StopIteration: + rows = it else: rows = data(tbl) for row in rows: @@ -184,9 +187,12 @@ def appendxlsx(tbl, filename, sheet=None, write_header=False): ws = wb[str(sheet)] if write_header: it = iter(tbl) - hdr = next(it) - flds = list(map(text_type, hdr)) - rows = itertools.chain([flds], it) + try: + hdr = next(it) + flds = list(map(text_type, hdr)) + rows = itertools.chain([flds], it) + except StopIteration: + rows = it else: rows = data(tbl) for row in rows: diff --git a/petl/test/helpers.py b/petl/test/helpers.py index 60c780f9..5a799558 100644 --- a/petl/test/helpers.py +++ b/petl/test/helpers.py @@ -10,7 +10,7 @@ def eq_(expect, actual, msg=None): """Test when two values from a python variable are exactly equals (==)""" - assert expect == actual, msg + assert expect == actual, msg or ('%r != %s' % (expect, actual)) def assert_almost_equal(first, second, places=None, msg=None): diff --git a/petl/test/io/test_html.py b/petl/test/io/test_html.py index 127f5d09..8d6b565a 100644 --- a/petl/test/io/test_html.py +++ b/petl/test/io/test_html.py @@ -118,3 +118,21 @@ def test_tohtml_with_style(): u"
%s
%s
%s
\n" ) eq_(expect, actual) + + +def test_tohtml_headerless(): + table = [] + + f = NamedTemporaryFile(delete=False) + tohtml(table, f.name, encoding='ascii', lineterminator='\n') + + # check what it did + with io.open(f.name, mode='rt', encoding='ascii', newline='') as o: + actual = o.read() + expect = ( + u"\n" + u"\n" + u"\n" + u"
\n" + ) + eq_(expect, actual) diff --git a/petl/test/io/test_pandas.py b/petl/test/io/test_pandas.py index d592e350..0361aec3 100644 --- a/petl/test/io/test_pandas.py +++ b/petl/test/io/test_pandas.py @@ -26,6 +26,12 @@ def test_todataframe(): actual = todataframe(tbl) assert expect.equals(actual) + def test_headerless(): + tbl = [] + expect = pd.DataFrame() + actual = todataframe(tbl) + assert expect.equals(actual) + def test_fromdataframe(): tbl = [('foo', 'bar', 'baz'), ('apples', 1, 2.5), diff --git a/petl/test/io/test_pickle.py b/petl/test/io/test_pickle.py index 7f8f160f..3107b5d3 100644 --- a/petl/test/io/test_pickle.py +++ b/petl/test/io/test_pickle.py @@ -10,6 +10,14 @@ from petl.io.pickle import frompickle, topickle, appendpickle +def picklereader(fl): + try: + while True: + yield pickle.load(fl) + except EOFError: + pass + + def test_frompickle(): f = NamedTemporaryFile(delete=False) @@ -36,13 +44,6 @@ def test_topickle_appendpickle(): f = NamedTemporaryFile(delete=False) topickle(table, f.name) - def picklereader(fl): - try: - while True: - yield pickle.load(fl) - except EOFError: - pass - # check what it did with open(f.name, 'rb') as o: actual = picklereader(o) @@ -66,3 +67,12 @@ def picklereader(fl): ('e', 9), ('f', 1)) ieq(expect, actual) + + +def test_topickle_headerless(): + table = [] + f = NamedTemporaryFile(delete=False) + topickle(table, f.name) + expect = [] + with open(f.name, 'rb') as o: + ieq(expect, picklereader(o)) diff --git a/petl/test/io/test_text.py b/petl/test/io/test_text.py index 4ffdf19d..90a31ca8 100644 --- a/petl/test/io/test_text.py +++ b/petl/test/io/test_text.py @@ -174,3 +174,18 @@ def test_totext_gz(): eq_(expect, actual) finally: o.close() + + +def test_totext_headerless(): + table = [] + f = NamedTemporaryFile(delete=False) + prologue = "-- START\n" + template = "+ {f1}\n" + epilogue = "-- END\n" + totext(table, f.name, encoding='ascii', template=template, + prologue=prologue, epilogue=epilogue) + + with io.open(f.name, mode='rt', encoding='ascii', newline='') as o: + actual = o.read() + expect = prologue + epilogue + eq_(expect, actual) diff --git a/petl/test/io/test_xls.py b/petl/test/io/test_xls.py index c29d51c8..b748255d 100644 --- a/petl/test/io/test_xls.py +++ b/petl/test/io/test_xls.py @@ -85,6 +85,15 @@ def test_toxls(): ieq(expect, actual) ieq(expect, actual) + def test_toxls_headerless(): + expect = [] + f = NamedTemporaryFile(delete=False) + f.close() + toxls(expect, f.name, 'Sheet1') + actual = fromxls(f.name, 'Sheet1') + ieq(expect, actual) + ieq(expect, actual) + def test_toxls_date(): expect = (('foo', 'bar'), (u'é', datetime(2012, 1, 1)), @@ -128,4 +137,4 @@ def wrapper(self, *args, **kwargs): ('C', 2), (u'é', datetime(2012, 1, 1))) ieq(expect, tbl) - ieq(expect, tbl) \ No newline at end of file + ieq(expect, tbl) diff --git a/petl/test/io/test_xlsx.py b/petl/test/io/test_xlsx.py index 9076e7e4..469d2fda 100644 --- a/petl/test/io/test_xlsx.py +++ b/petl/test/io/test_xlsx.py @@ -226,3 +226,13 @@ def test_appendxlsx_with_non_str_header(xlsx_table_with_non_str_header, xlsx_tes appendxlsx(xlsx_table_with_non_str_header, f.name, 'Sheet1') expect = etl.cat(xlsx_test_table, xlsx_table_with_non_str_header) ieq(expect, actual) + + +def test_toxlsx_headerless(): + expect = [] + f = NamedTemporaryFile(delete=False) + f.close() + toxlsx(expect, f.name) + actual = fromxlsx(f.name) + ieq(expect, actual) + ieq(expect, actual) diff --git a/petl/test/transform/test_basics.py b/petl/test/transform/test_basics.py index 2424dab6..dfcdaf2d 100644 --- a/petl/test/transform/test_basics.py +++ b/petl/test/transform/test_basics.py @@ -2,6 +2,7 @@ import pytest +from petl.errors import FieldSelectionError from petl.test.helpers import ieq from petl.util import expr, empty, coalesce from petl.transform.basics import cut, cat, addfield, rowslice, head, tail, \ @@ -71,6 +72,13 @@ def test_cut_empty(): ieq(expect, actual) +def test_cut_headerless(): + table = () + with pytest.raises(FieldSelectionError): + for i in cut(table, 'bar'): + pass + + def test_cutout(): table = (('foo', 'bar', 'baz'), @@ -108,6 +116,13 @@ def test_cutout(): ieq(expectation, cut3) +def test_cutout_headerless(): + table = () + with pytest.raises(FieldSelectionError): + for i in cutout(table, 'bar'): + pass + + def test_cat(): table1 = (('foo', 'bar'), @@ -198,6 +213,16 @@ def test_cat_empty(): ieq(expect, actual) +def test_cat_headerless(): + table1 = (('foo', 'bar'), + (1, 'A'), + (2, 'B')) + table2 = () + expect = table1 # basically does nothing + actual = cat(table1, table2) + ieq(expect, actual) + + def test_cat_dupfields(): table1 = (('foo', 'foo'), (1, 'A'), @@ -253,6 +278,16 @@ def test_stack_dupfields(): ieq(expect, actual) +def test_stack_headerless(): + table1 = (('foo', 'bar'), + (1, 'A'), + (2, 'B')) + table2 = () + expect = table1 # basically does nothing + actual = stack(table1, table2) + ieq(expect, actual) + + def test_addfield(): table = (('foo', 'bar'), ('M', 12), @@ -308,6 +343,15 @@ def test_addfield_empty(): ieq(expect, actual) +def test_addfield_headerless(): + """When adding a field to a headerless table, implicitly add a header.""" + table = () + expect = (('foo',),) + actual = addfield(table, 'foo', 1) + ieq(expect, actual) + ieq(expect, actual) + + def test_addfield_coalesce(): table = (('foo', 'bar', 'baz', 'quux'), ('M', 12, 23, 44), @@ -438,6 +482,13 @@ def test_rowslice_empty(): ieq(expect, actual) +def test_rowslice_headerless(): + table = () + expect = () + actual = rowslice(table, 1, 2) + ieq(expect, actual) + + def test_head(): table1 = (('foo', 'bar'), @@ -505,6 +556,13 @@ def test_tail_empty(): ieq(expect, actual) +def test_tail_headerless(): + table = () + expect = () + actual = tail(table) + ieq(expect, actual) + + def test_skipcomments(): table1 = (('##aaa', 'bbb', 'ccc'), @@ -576,6 +634,16 @@ def test_annex_uneven_rows(): ieq(expect, actual) +def test_annex_headerless(): + table1 = (('foo', 'bar'), + ('C', 2)) + table2 = () # does nothing + expect = table1 + actual = annex(table1, table2) + ieq(expect, actual) + ieq(expect, actual) + + def test_addrownumbers(): table1 = (('foo', 'bar'), @@ -606,6 +674,15 @@ def test_addrownumbers_field_name(): ieq(expect, actual) +def test_addrownumbers_headerless(): + """Adds a column row if there is none.""" + table = () + expect = (('id',),) + actual = addrownumbers(table, field='id') + ieq(expect, actual) + ieq(expect, actual) + + def test_addcolumn(): table1 = (('foo', 'bar'), @@ -655,6 +732,17 @@ def test_empty_addcolumn(): ieq(expect, table3) +def test_addcolumn_headerless(): + """Adds a header row if none exists.""" + table1 = () + expect = (('foo',), + ('A',), + ('B',)) + actual = addcolumn(table1, 'foo', ['A', 'B']) + ieq(expect, actual) + ieq(expect, actual) + + def test_addfieldusingcontext(): table1 = (('foo', 'bar'), @@ -720,6 +808,30 @@ def downstream(prv, cur, nxt): ieq(expect, table3) +def test_addfieldusingcontext_empty(): + table = empty() + expect = (('foo',),) + + def query(prv, cur, nxt): + return 0 + + actual = addfieldusingcontext(table, 'foo', query) + ieq(expect, actual) + ieq(expect, actual) + + +def test_addfieldusingcontext_headerless(): + table = () + expect = (('foo',),) + + def query(prv, cur, nxt): + return 0 + + actual = addfieldusingcontext(table, 'foo', query) + ieq(expect, actual) + ieq(expect, actual) + + def test_movefield(): table1 = (('foo', 'bar', 'baz'), diff --git a/petl/test/transform/test_conversions.py b/petl/test/transform/test_conversions.py index 0a7e7400..caa22a58 100644 --- a/petl/test/transform/test_conversions.py +++ b/petl/test/transform/test_conversions.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest +from petl.errors import FieldSelectionError from petl.test.failonerror import assert_failonerror from petl.test.helpers import ieq from petl.transform.conversions import convert, convertall, convertnumbers, \ @@ -72,6 +74,19 @@ def test_convert_empty(): ieq(expect, actual) +def test_convert_headerless(): + table = () + with pytest.raises(FieldSelectionError): + for i in convert(table, 'foo', int): + pass + + +def test_convert_headerless_no_conversions(): + table = expect = () + actual = convert(table) + ieq(expect, actual) + + def test_convert_indexes(): table1 = (('foo', 'bar', 'baz'), diff --git a/petl/test/transform/test_dedup.py b/petl/test/transform/test_dedup.py index 519e9211..8f783188 100644 --- a/petl/test/transform/test_dedup.py +++ b/petl/test/transform/test_dedup.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest +from petl.errors import FieldSelectionError from petl.test.helpers import ieq from petl.transform.dedup import duplicates, unique, conflicts, distinct, \ isunique @@ -34,6 +36,23 @@ def test_duplicates(): ieq(expectation, result) +def test_duplicates_headerless_no_keys(): + """Removing the duplicates from an empty table without specifying which + columns shouldn't be a problem. + """ + table = [] + actual = duplicates(table) + expect = [] + ieq(expect, actual) + + +def test_duplicates_headerless_explicit(): + table = [] + with pytest.raises(FieldSelectionError): + for i in duplicates(table, 'foo'): + pass + + def test_duplicates_empty(): table = (('foo', 'bar'),) expect = (('foo', 'bar'),) diff --git a/petl/test/transform/test_fills.py b/petl/test/transform/test_fills.py index c6c8625e..e5df2cca 100644 --- a/petl/test/transform/test_fills.py +++ b/petl/test/transform/test_fills.py @@ -53,6 +53,13 @@ def test_filldown(): ieq(expect, actual) +def test_filldown_headerless(): + table = [] + actual = filldown(table, 'foo') + expect = [] + ieq(expect, actual) + + def test_fillright(): table = (('foo', 'bar', 'baz'), @@ -77,6 +84,13 @@ def test_fillright(): ieq(expect, actual) +def test_fillright_headerless(): + table = [] + actual = fillright(table, 'foo') + expect = [] + ieq(expect, actual) + + def test_fillleft(): table = (('foo', 'bar', 'baz'), @@ -99,3 +113,10 @@ def test_fillleft(): ('c', 'c', .72)) ieq(expect, actual) ieq(expect, actual) + + +def test_fillleft_headerless(): + table = [] + actual = fillleft(table, 'foo') + expect = [] + ieq(expect, actual) diff --git a/petl/test/transform/test_headers.py b/petl/test/transform/test_headers.py index a40ef26f..50b98449 100644 --- a/petl/test/transform/test_headers.py +++ b/petl/test/transform/test_headers.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, print_function, division +import pytest from petl.test.helpers import ieq from petl.errors import FieldSelectionError @@ -29,6 +30,13 @@ def test_setheader_empty(): ieq(expect2, table2) +def test_setheader_headerless(): + table = [] + actual = setheader(table, ['foo', 'bar']) + expect = [('foo', 'bar')] + ieq(expect, actual) + + def test_extendheader(): table1 = (('foo',), @@ -50,6 +58,14 @@ def test_extendheader_empty(): ieq(expect2, table2) +def test_extendheader_headerless(): + table = [] + actual = extendheader(table, ['foo', 'bar']) + expect = [('foo', 'bar')] + ieq(expect, actual) + ieq(expect, actual) + + def test_pushheader(): table1 = (('a', 1), @@ -81,6 +97,14 @@ def test_pushheader_empty(): ieq(expect2, table2) +def test_pushheader_headerless(): + table = [] + actual = pushheader(table, ['foo', 'bar']) + expect = [('foo', 'bar')] + ieq(expect, actual) + ieq(expect, actual) + + def test_pushheader_positional(): table1 = (('a', 1), @@ -152,6 +176,13 @@ def test_skip_empty(): ieq(expect2, table2) +def test_skip_headerless(): + table = [] + actual = skip(table, 2) + expect = [] + ieq(expect, actual) + + def test_rename(): table = (('foo', 'bar'), @@ -212,6 +243,13 @@ def test_rename_empty(): ieq(expect, actual) +def test_rename_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + for i in rename(table, 'foo', 'foofoo'): + pass + + def test_prefixheader(): table1 = (('foo', 'bar'), @@ -227,6 +265,13 @@ def test_prefixheader(): ieq(expect, actual) +def test_prefixheader_headerless(): + table = [] + actual = prefixheader(table, 'pre_') + expect = [] + ieq(expect, actual) + + def test_suffixheader(): table1 = (('foo', 'bar'), @@ -242,6 +287,13 @@ def test_suffixheader(): ieq(expect, actual) +def test_suffixheader_headerless(): + table = [] + actual = suffixheader(table, '_suf') + expect = [] + ieq(expect, actual) + + def test_sortheaders(): table1 = ( ('id', 'foo', 'bar', 'baz'), @@ -274,3 +326,10 @@ def test_sortheaders_duplicate_headers(): actual = sortheader(table1) ieq(expect, actual) + + +def test_sortheader_headerless(): + table = [] + actual = sortheader(table) + expect = [] + ieq(expect, actual) diff --git a/petl/test/transform/test_maps.py b/petl/test/transform/test_maps.py index 155f6c2f..c23c7108 100644 --- a/petl/test/transform/test_maps.py +++ b/petl/test/transform/test_maps.py @@ -91,6 +91,16 @@ def test_fieldmap_empty(): ieq(expect, actual) +def test_fieldmap_headerless(): + table = [] + expect = [] + mappings = OrderedDict() + mappings['foo'] = 'foo' + mappings['baz'] = 'bar', lambda v: v * 2 + actual = fieldmap(table, mappings) + ieq(expect, actual) + + def test_fieldmap_failonerror(): input_ = (('foo',), ('A',), (1,)) mapper_ = {'bar': ('foo', lambda v: v.lower())} @@ -160,6 +170,17 @@ def rowmapper(row): ieq(expect, actual) +def test_rowmap_headerless(): + table = [] + + def rowmapper(row): + return row + + actual = rowmap(table, rowmapper, header=['subject_id', 'gender']) + expect = [] + ieq(expect, actual) + + def test_rowmap_failonerror(): input_ = (('foo',), ('A',), (1,), ('B',)) mapper = lambda r: [r[0].lower()] @@ -287,3 +308,15 @@ def rowgenerator(rec): ieq(expect, actual) ieq(expect, actual) # can iteratate twice? + +def test_recordmapmany_headerless(): + table = [] + + def duplicate(rec): + yield rec + yield rec + + actual = rowmapmany(table, duplicate, header=['subject_id', 'variable']) + expect = [] + ieq(expect, actual) + ieq(expect, actual) # can iteratate twice? diff --git a/petl/test/transform/test_regex.py b/petl/test/transform/test_regex.py index 95305f5f..fead5160 100644 --- a/petl/test/transform/test_regex.py +++ b/petl/test/transform/test_regex.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, division +import pytest from petl.compat import next - - +from petl.errors import ArgumentError from petl.test.helpers import ieq, eq_ from petl.transform.regex import capture, split, search, searchcomplement, splitdown from petl.transform.basics import TransformError @@ -57,6 +57,13 @@ def test_capture_empty(): ieq(expect, actual) +def test_capture_headerless(): + table = [] + with pytest.raises(ArgumentError): + for i in capture(table, 'bar', r'(\w)(\d)', ('baz', 'qux')): + pass + + def test_capture_nonmatching(): table = (('id', 'variable', 'value'), @@ -148,6 +155,13 @@ def test_split_empty(): ieq(expect, actual) +def test_split_headerless(): + table = [] + with pytest.raises(ArgumentError): + for i in split(table, 'bar', 'd', ('baz', 'qux')): + pass + + def test_search(): table1 = (('foo', 'bar', 'baz'), @@ -193,6 +207,14 @@ def test_search_2(): ieq(expect, actual) +def test_search_headerless(): + table = [] + actual = search(table, 'foo', '[ab]{2}') + expect = [] + ieq(expect, actual) + ieq(expect, actual) + + def test_searchcomplement(): table1 = (('foo', 'bar', 'baz'), diff --git a/petl/test/transform/test_reshape.py b/petl/test/transform/test_reshape.py index 35502237..e69048a5 100644 --- a/petl/test/transform/test_reshape.py +++ b/petl/test/transform/test_reshape.py @@ -3,7 +3,9 @@ from datetime import datetime +import pytest +from petl.errors import FieldSelectionError from petl.test.helpers import ieq from petl.transform.reshape import melt, recast, transpose, pivot, flatten, \ unflatten @@ -69,6 +71,13 @@ def test_melt_empty(): ieq(expect, actual) +def test_melt_headerless(): + table = [] + expect = [] + actual = melt(table, key='foo') + ieq(expect, actual) + + def test_melt_1_shortrow(): table = (('id', 'gender', 'age'), @@ -254,6 +263,13 @@ def test_recast_empty(): ieq(expect, actual) +def test_recast_headerless(): + table = [] + expect = [] + actual = recast(table) + ieq(expect, actual) + + def test_recast_date(): dt = datetime.now().replace @@ -383,6 +399,13 @@ def test_pivot_empty(): ieq(expect2, table2) +def test_pivot_headerless(): + table1 = [] + with pytest.raises(FieldSelectionError): + for i in pivot(table1, 'region', 'gender', 'units', sum): + pass + + def test_flatten(): table1 = (('foo', 'bar', 'baz'), @@ -406,6 +429,13 @@ def test_flatten_empty(): ieq(expect1, actual1) +def test_flatten_headerless(): + table1 = [] + expect1 = [] + actual1 = flatten(table1) + ieq(expect1, actual1) + + def test_unflatten(): table1 = (('lines',), diff --git a/petl/test/transform/test_selects.py b/petl/test/transform/test_selects.py index ec9ac913..24d1cf63 100644 --- a/petl/test/transform/test_selects.py +++ b/petl/test/transform/test_selects.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest +from petl.errors import FieldSelectionError from petl.test.helpers import ieq, eq_ from petl.comparison import Comparable from petl.transform.selects import select, selectin, selectcontains, \ @@ -113,6 +115,20 @@ def test_select_empty(): ieq(expect, actual) +def test_rowselect_headerless(): + table = [] + expect = [] + actual = select(table, 'True') + ieq(expect, actual) + + +def test_fieldselect_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + for i in select(table, 'foo', lambda v: v == 'a'): + pass + + def test_select_falsey(): table = (('foo',), ([],), diff --git a/petl/test/transform/test_sorts.py b/petl/test/transform/test_sorts.py index 24aff0b1..b962e1ea 100644 --- a/petl/test/transform/test_sorts.py +++ b/petl/test/transform/test_sorts.py @@ -11,7 +11,7 @@ from petl.compat import next - +from petl.errors import FieldSelectionError from petl.test.helpers import ieq, eq_ from petl.util import nrows from petl.transform.basics import cat @@ -361,6 +361,25 @@ def test_sort_none(): ieq(expectation, result) +def test_sort_headerless_no_keys(): + """ + Sorting a headerless table without specifying cols should be a no-op. + """ + table = [] + result = sort(table) + expectation = [] + ieq(expectation, result) + + +def test_sort_headerless_explicit(): + """ + But if you specify keys, they must exist. + """ + table = [] + with pytest.raises(FieldSelectionError): + for i in sort(table, 'foo'): + pass + # TODO test sort with native comparison diff --git a/petl/test/transform/test_unpacks.py b/petl/test/transform/test_unpacks.py index bb83e910..e913d1ff 100644 --- a/petl/test/transform/test_unpacks.py +++ b/petl/test/transform/test_unpacks.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest +from petl.errors import ArgumentError from petl.test.helpers import ieq from petl.transform.unpacks import unpack, unpackdict @@ -76,6 +78,13 @@ def test_unpack_empty(): ieq(expect2, table2) +def test_unpack_headerless(): + table = [] + with pytest.raises(ArgumentError): + for i in unpack(table, 'bar', ['baz', 'quux']): + pass + + def test_unpackdict(): table1 = (('foo', 'bar'), diff --git a/petl/test/transform/test_validation.py b/petl/test/transform/test_validation.py index a8e6dc28..cfe88121 100644 --- a/petl/test/transform/test_validation.py +++ b/petl/test/transform/test_validation.py @@ -130,3 +130,14 @@ def test_header(): ieq(expect, actual) ieq(expect, actual) + + +def test_validation_headerless(): + header = ('foo', 'bar', 'baz') + table = [] + # Expect only a missing header - no exceptions please + expect = (('name', 'row', 'field', 'value', 'error'), + ('__header__', 0, None, None, 'AssertionError')) + actual = validate(table, header=header) + ieq(expect, actual) + ieq(expect, actual) diff --git a/petl/test/util/test_base.py b/petl/test/util/test_base.py index 65431e2d..8cd54d1d 100644 --- a/petl/test/util/test_base.py +++ b/petl/test/util/test_base.py @@ -1,7 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest -from petl.errors import ArgumentError +from petl.errors import FieldSelectionError from petl.test.helpers import ieq, eq_ from petl.compat import next from petl.util.base import header, fieldnames, data, dicts, records, \ @@ -52,6 +53,13 @@ def test_data(): ieq(expect, actual) +def test_data_headerless(): + table = [] + actual = data(table) + expect = [] + ieq(expect, actual) + + def test_dicts(): table = (('foo', 'bar'), ('a', 1), ('b', 2)) actual = dicts(table) @@ -59,6 +67,13 @@ def test_dicts(): ieq(expect, actual) +def test_dicts_headerless(): + table = [] + actual = dicts(table) + expect = [] + ieq(expect, actual) + + def test_dicts_shortrows(): table = (('foo', 'bar'), ('a', 1), ('b',)) actual = dicts(table) @@ -94,6 +109,13 @@ def test_records(): eq_('qux', o.get('baz', default='qux')) +def test_records_headerless(): + table = [] + actual = records(table) + expect = [] + ieq(expect, actual) + + def test_records_errors(): table = (('foo', 'bar'), ('a', 1), ('b', 2)) actual = records(table) @@ -147,6 +169,13 @@ def test_namedtuples(): eq_(2, o.bar) +def test_namedtuples_headerless(): + table = [] + actual = namedtuples(table) + expect = [] + ieq(expect, actual) + + def test_namedtuples_unevenrows(): table = (('foo', 'bar'), ('a', 1, True), ('b',)) actual = namedtuples(table) @@ -187,6 +216,14 @@ def test_itervalues(): ieq(expect, actual) +def test_itervalues_headerless(): + table = [] + actual = itervalues(table, 'foo') + with pytest.raises(FieldSelectionError): + for i in actual: + pass + + def test_values(): table = (('foo', 'bar', 'baz'), @@ -222,6 +259,14 @@ def test_values(): ieq(expect, actual) +def test_values_headerless(): + table = [] + actual = values(table, 'foo') + with pytest.raises(FieldSelectionError): + for i in actual: + pass + + def test_rowgroupby(): table = (('foo', 'bar', 'baz'), @@ -279,3 +324,9 @@ def test_rowgroupby(): eq_(2, len(vals)) eq_(True, vals[0]) eq_(None, vals[1]) # gets padded + + +def test_rowgroupby_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + rowgroupby(table, 'foo') diff --git a/petl/test/util/test_lookups.py b/petl/test/util/test_lookups.py index 2a8c54bf..06002d21 100644 --- a/petl/test/util/test_lookups.py +++ b/petl/test/util/test_lookups.py @@ -1,7 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest -from petl.errors import DuplicateKeyError +from petl.errors import DuplicateKeyError, FieldSelectionError from petl.test.helpers import eq_ from petl import cut, lookup, lookupone, dictlookup, dictlookupone, \ recordlookup, recordlookupone @@ -42,6 +43,12 @@ def test_lookup(): eq_(expect, actual) +def test_lookup_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + lookup(table, 'foo', 'bar') + + def test_lookupone(): t1 = (('foo', 'bar'), ('a', 1), ('b', 2), ('b', 3)) @@ -85,6 +92,12 @@ def test_lookupone(): eq_(expect, actual) +def test_lookupone_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + lookupone(table, 'foo', 'bar') + + def test_dictlookup(): t1 = (('foo', 'bar'), ('a', 1), ('b', 2), ('b', 3)) diff --git a/petl/test/util/test_materialise.py b/petl/test/util/test_materialise.py index ae723720..b09be7f0 100644 --- a/petl/test/util/test_materialise.py +++ b/petl/test/util/test_materialise.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division +import pytest +from petl.errors import FieldSelectionError from petl.test.helpers import eq_ from petl.util.materialise import columns, facetcolumns @@ -13,6 +15,19 @@ def test_columns(): eq_([1, 2, 3], cols['bar']) +def test_columns_empty(): + table = [('foo', 'bar')] + cols = columns(table) + eq_([], cols['foo']) + eq_([], cols['bar']) + + +def test_columns_headerless(): + table = [] + cols = columns(table) + eq_({}, cols) + + def test_facetcolumns(): table = [['foo', 'bar', 'baz'], @@ -27,3 +42,9 @@ def test_facetcolumns(): eq_(['b', 'b'], fc['b']['foo']) eq_([2, 3], fc['b']['bar']) eq_([True, None], fc['b']['baz']) + + +def test_facetcolumns_headerless(): + table = [] + with pytest.raises(FieldSelectionError): + facetcolumns(table, 'foo') diff --git a/petl/test/util/test_vis.py b/petl/test/util/test_vis.py index b34b089e..b6227acc 100644 --- a/petl/test/util/test_vis.py +++ b/petl/test/util/test_vis.py @@ -189,3 +189,10 @@ def test_lookstr(): +-----+-----+ """ eq_(expect, actual) + + +def test_look_headerless(): + table = [] + actual = repr(look(table)) + expect = "" + eq_(expect, actual) diff --git a/petl/transform/basics.py b/petl/transform/basics.py index 8627e81a..faa3e4c0 100644 --- a/petl/transform/basics.py +++ b/petl/transform/basics.py @@ -130,7 +130,10 @@ def itercut(source, spec, missing=None): spec = tuple(spec) # make sure no-one can change midstream # convert field selection into field indices - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] indices = asindices(hdr, spec) # define a function to transform each row in the source data @@ -202,7 +205,10 @@ def itercutout(source, spec, missing=None): spec = tuple(spec) # make sure no-one can change midstream # convert field selection into field indices - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] indicesout = asindices(hdr, spec) indices = [i for i in range(len(hdr)) if i not in indicesout] @@ -340,7 +346,12 @@ def __iter__(self): def itercat(sources, missing, header): its = [iter(t) for t in sources] - hdrs = [list(next(it)) for it in its] + hdrs = [] + for it in its: + try: + hdrs.append(list(next(it))) + except StopIteration: + hdrs.append([]) if header is None: # determine output fields by gathering all fields found in the sources @@ -451,7 +462,12 @@ def __iter__(self): def iterstack(sources, missing, trim, pad): its = [iter(t) for t in sources] - hdrs = [next(it) for it in its] + hdrs = [] + for it in its: + try: + hdrs.append(next(it)) + except StopIteration: + hdrs.append([]) hdr = hdrs[0] n = len(hdr) yield tuple(hdr) @@ -526,7 +542,10 @@ def __iter__(self): def iteraddfield(source, field, value, index): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) # determine index of new field @@ -615,7 +634,10 @@ def __iter__(self): def iteraddfields(source, field_defs): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) # initialize output fields and indices @@ -826,7 +848,10 @@ def __iter__(self): def itertail(source, n): it = iter(source) - yield tuple(next(it)) # fields + try: + yield tuple(next(it)) # fields + except StopIteration: + return # stop generating cache = deque() for row in it: cache.append(row) @@ -910,7 +935,10 @@ def __iter__(self): it = iter(self.table) # determine output fields - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] outhdr = [f for f in hdr if f != self.field] outhdr.insert(self.index, self.field) yield tuple(outhdr) @@ -977,7 +1005,12 @@ def __iter__(self): def iterannex(tables, missing): its = [iter(t) for t in tables] - hdrs = [next(it) for it in its] + hdrs = [] + for it in its: + try: + hdrs.append(next(it)) + except StopIteration: + hdrs.append([]) outhdr = tuple(chain(*hdrs)) yield outhdr for rows in izip_longest(*its): @@ -1042,7 +1075,10 @@ def __iter__(self): def iteraddrownumbers(table, start, step, field): it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] outhdr = [field] outhdr.extend(hdr) yield tuple(outhdr) @@ -1097,7 +1133,10 @@ def __iter__(self): def iteraddcolumn(table, field, col, index, missing): it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] # determine position of new column if index is None: @@ -1186,13 +1225,19 @@ def __iter__(self): def iteraddfieldusingcontext(table, field, query): it = iter(table) - hdr = tuple(next(it)) + try: + hdr = tuple(next(it)) + except StopIteration: + hdr = () flds = list(map(text_type, hdr)) yield hdr + (field,) flds.append(field) it = (Record(row, flds) for row in it) prv = None - cur = next(it) + try: + cur = next(it) + except StopIteration: + return # no more items for nxt in it: v = query(prv, cur, nxt) yield tuple(cur) + (v,) diff --git a/petl/transform/conversions.py b/petl/transform/conversions.py index c699a795..5b952f10 100644 --- a/petl/transform/conversions.py +++ b/petl/transform/conversions.py @@ -354,9 +354,12 @@ def iterfieldconvert(source, converters, failonerror, errorvalue, where, # grab the fields in the source table it = iter(source) - hdr = next(it) - flds = list(map(text_type, hdr)) - yield tuple(hdr) # these are not modified + try: + hdr = next(it) + flds = list(map(text_type, hdr)) + yield tuple(hdr) # these are not modified + except StopIteration: + hdr = flds = [] # converters will fail selecting a field # build converter functions converter_functions = dict() diff --git a/petl/transform/dedup.py b/petl/transform/dedup.py index 5c64e5ca..3d27e525 100644 --- a/petl/transform/dedup.py +++ b/petl/transform/dedup.py @@ -89,7 +89,12 @@ def iterduplicates(source, key): # first need to sort the data it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + if key is None: + return # nothing to do on a table without headers + hdr = [] yield tuple(hdr) # convert field selection into field indices @@ -189,7 +194,10 @@ def iterunique(source, key): # first need to sort the data it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return yield tuple(hdr) # convert field selection into field indices @@ -326,7 +334,10 @@ def iterconflicts(source, key, missing, exclude, include): include = None it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) yield tuple(hdr) @@ -407,7 +418,10 @@ def __init__(self, table, key=None, count=None, presorted=False, def __iter__(self): it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return # convert field selection into field indices if self.key is None: diff --git a/petl/transform/fills.py b/petl/transform/fills.py index a3c8f8e4..8dcf28c0 100644 --- a/petl/transform/fills.py +++ b/petl/transform/fills.py @@ -104,7 +104,10 @@ def __iter__(self): def iterfilldown(table, fillfields, missing): it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return yield tuple(hdr) if not fillfields: # fill down all fields fillfields = hdr @@ -177,7 +180,10 @@ def __iter__(self): def iterfillright(table, missing): it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return yield tuple(hdr) for row in it: outrow = list(row) @@ -243,7 +249,10 @@ def __iter__(self): def iterfillleft(table, missing): it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return yield tuple(hdr) for row in it: outrow = list(reversed(row)) diff --git a/petl/transform/headers.py b/petl/transform/headers.py index 641acffe..d5b81cb6 100644 --- a/petl/transform/headers.py +++ b/petl/transform/headers.py @@ -79,7 +79,10 @@ def __setitem__(self, key, value): def iterrename(source, spec, strict): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) if strict: for x in spec: @@ -138,7 +141,10 @@ def __iter__(self): def itersetheader(source, header): it = iter(source) - next(it) # discard source header + try: + next(it) # discard source header + except StopIteration: + pass # no previous header yield tuple(header) for row in it: yield tuple(row) @@ -185,7 +191,10 @@ def __iter__(self): def iterextendheader(source, fields): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] outhdr = list(hdr) outhdr.extend(fields) yield tuple(outhdr) @@ -308,7 +317,10 @@ def __init__(self, table, prefix): def __iter__(self): it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return outhdr = tuple((text_type(self.prefix) + text_type(f)) for f in hdr) yield outhdr for row in it: @@ -332,7 +344,10 @@ def __init__(self, table, suffix): def __iter__(self): it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return outhdr = tuple((text_type(f) + text_type(self.suffix)) for f in hdr) yield outhdr for row in it: @@ -361,7 +376,10 @@ def __init__(self, table, reverse, missing): def __iter__(self): it = iter(self.table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return shdr = sorted(hdr) indices = asindices(hdr, shdr) transform = rowgetter(*indices) diff --git a/petl/transform/maps.py b/petl/transform/maps.py index ff64afe0..c268de02 100644 --- a/petl/transform/maps.py +++ b/petl/transform/maps.py @@ -88,7 +88,10 @@ def __iter__(self): def iterfieldmap(source, mappings, failonerror, errorvalue): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) outhdr = mappings.keys() yield tuple(outhdr) @@ -214,7 +217,10 @@ def __iter__(self): def iterrowmap(source, rowmapper, header, failonerror): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) yield tuple(header) it = (Record(row, flds) for row in it) @@ -308,7 +314,10 @@ def __iter__(self): def iterrowmapmany(source, rowgenerator, header, failonerror): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) yield tuple(header) it = (Record(row, flds) for row in it) diff --git a/petl/transform/regex.py b/petl/transform/regex.py index 112cbfd4..2efe00de 100644 --- a/petl/transform/regex.py +++ b/petl/transform/regex.py @@ -100,7 +100,10 @@ def itercapture(source, field, pattern, newfields, include_original, flags, it = iter(source) prog = re.compile(pattern, flags) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) if isinstance(field, int) and field < len(hdr): field_index = field @@ -197,7 +200,10 @@ def itersplit(source, field, pattern, newfields, include_original, maxsplit, it = iter(source) prog = re.compile(pattern, flags) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) if isinstance(field, int) and field < len(hdr): field_index = field @@ -312,7 +318,10 @@ def __iter__(self): def itersearch(table, pattern, field, flags, complement): prog = re.compile(pattern, flags) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) yield tuple(hdr) @@ -439,7 +448,10 @@ def itersplitdown(table, field, pattern, maxsplit, flags): prog = re.compile(pattern, flags) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) if isinstance(field, int) and field < len(hdr): diff --git a/petl/transform/reshape.py b/petl/transform/reshape.py index 055b1abc..c3f1e821 100644 --- a/petl/transform/reshape.py +++ b/petl/transform/reshape.py @@ -110,7 +110,10 @@ def itermelt(source, key, variables, variablefield, valuefield): raise ValueError('either key or variables must be specified') it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return # determine key and variable field indices key_indices = variables_indices = None @@ -298,7 +301,10 @@ def iterrecast(source, key, variablefield, valuefield, # TODO only make one pass through the data it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) # normalise some stuff @@ -538,7 +544,10 @@ def iterpivot(source, f1, f2, f3, aggfun, missing): # second pass - generate output it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) f1i = flds.index(f1) f2i = flds.index(f2) diff --git a/petl/transform/selects.py b/petl/transform/selects.py index 87456aef..77e45d29 100644 --- a/petl/transform/selects.py +++ b/petl/transform/selects.py @@ -112,8 +112,11 @@ def __iter__(self): def iterfieldselect(source, field, where, complement, missing): it = iter(source) - hdr = next(it) - yield tuple(hdr) + try: + hdr = next(it) + yield tuple(hdr) + except StopIteration: + hdr = [] # will raise FieldSelectionError below indices = asindices(hdr, field) getv = operator.itemgetter(*indices) for row in it: @@ -127,7 +130,10 @@ def iterfieldselect(source, field, where, complement, missing): def iterrowselect(source, where, missing, complement): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return # will yield nothing flds = list(map(text_type, hdr)) yield tuple(hdr) it = (Record(row, flds, missing=missing) for row in it) @@ -421,7 +427,10 @@ def __iter__(self): def iterselectusingcontext(table, query): it = iter(table) - hdr = tuple(next(it)) + try: + hdr = tuple(next(it)) + except StopIteration: + return # will yield nothing flds = list(map(text_type, hdr)) yield hdr it = (Record(row, flds) for row in it) diff --git a/petl/transform/sorts.py b/petl/transform/sorts.py index d591132e..801866b8 100755 --- a/petl/transform/sorts.py +++ b/petl/transform/sorts.py @@ -286,7 +286,12 @@ def _iternocache(self, source, key, reverse): self.clearcache() it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + if key is None: + return # nothing to do on a table without headers + hdr = [] yield tuple(hdr) if key is not None: @@ -480,7 +485,12 @@ def itermergesort(sources, key, header, missing, reverse): # borrow this from itercat - TODO remove code smells its = [iter(t) for t in sources] - src_hdrs = [next(it) for it in its] + src_hdrs = [] + for it in its: + try: + src_hdrs.append(next(it)) + except StopIteration: + src_hdrs.append([]) if header is None: # determine output fields by gathering all fields found in the sources @@ -562,7 +572,10 @@ def issorted(table, key=None, reverse=False, strict=False): op = operator.ge it = iter(table) - flds = [text_type(f) for f in next(it)] + try: + flds = [text_type(f) for f in next(it)] + except StopIteration: + flds = [] if key is None: prev = next(it) for curr in it: diff --git a/petl/transform/unpacks.py b/petl/transform/unpacks.py index 00d39b2e..5de74c11 100644 --- a/petl/transform/unpacks.py +++ b/petl/transform/unpacks.py @@ -64,7 +64,10 @@ def __iter__(self): def iterunpack(source, field, newfields, include_original, missing): it = iter(source) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) if field in flds: field_index = flds.index(field) @@ -164,7 +167,10 @@ def iterunpackdict(table, field, keys, includeoriginal, samplesize, missing): # set up it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) fidx = flds.index(field) outhdr = list(flds) diff --git a/petl/transform/validation.py b/petl/transform/validation.py index 76cde4d3..26821f0f 100644 --- a/petl/transform/validation.py +++ b/petl/transform/validation.py @@ -112,7 +112,10 @@ def iterproblems(table, constraints, expected_header): yield outhdr it = iter(table) - actual_header = next(it) + try: + actual_header = next(it) + except StopIteration: + actual_header = [] if expected_header is None: flds = list(map(text_type, actual_header)) diff --git a/petl/util/base.py b/petl/util/base.py index d53b7f9e..b950d288 100644 --- a/petl/util/base.py +++ b/petl/util/base.py @@ -244,7 +244,10 @@ def itervalues(table, field, **kwargs): missing = kwargs.get('missing', None) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] indices = asindices(hdr, field) assert len(indices) > 0, 'no field selected' @@ -445,7 +448,10 @@ def __repr__(self): def iterdicts(table, *sliceargs, **kwargs): missing = kwargs.get('missing', None) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return if sliceargs: it = islice(it, *sliceargs) for row in it: @@ -517,7 +523,10 @@ def iternamedtuples(table, *sliceargs, **kwargs): missing = kwargs.get('missing', None) name = kwargs.get('name', 'row') it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) nt = namedtuple(name, tuple(flds)) if sliceargs: @@ -639,7 +648,10 @@ def __repr__(self): def iterrecords(table, *sliceargs, **kwargs): missing = kwargs.get('missing', None) it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return flds = list(map(text_type, hdr)) if sliceargs: it = islice(it, *sliceargs) @@ -695,7 +707,10 @@ def rowgroupby(table, key, value=None): """ it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) # wrap rows as records it = (Record(row, flds) for row in it) diff --git a/petl/util/lookups.py b/petl/util/lookups.py index 921b45e4..54b1f086 100644 --- a/petl/util/lookups.py +++ b/petl/util/lookups.py @@ -13,7 +13,10 @@ def _setup_lookup(table, key, value): # obtain iterator and header row it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] # prepare key getter keyindices = asindices(hdr, key) @@ -225,7 +228,10 @@ def dictlookup(table, key, dictionary=None): dictionary = dict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) keyindices = asindices(hdr, key) assert len(keyindices) > 0, 'no key selected' @@ -303,7 +309,10 @@ def dictlookupone(table, key, dictionary=None, strict=False): dictionary = dict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) keyindices = asindices(hdr, key) assert len(keyindices) > 0, 'no key selected' @@ -331,7 +340,10 @@ def recordlookup(table, key, dictionary=None): dictionary = dict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) keyindices = asindices(hdr, key) assert len(keyindices) > 0, 'no key selected' @@ -363,7 +375,10 @@ def recordlookupone(table, key, dictionary=None, strict=False): dictionary = dict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) keyindices = asindices(hdr, key) assert len(keyindices) > 0, 'no key selected' diff --git a/petl/util/materialise.py b/petl/util/materialise.py index 0728d77c..074dfb37 100644 --- a/petl/util/materialise.py +++ b/petl/util/materialise.py @@ -60,7 +60,10 @@ def columns(table, missing=None): cols = OrderedDict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) for f in flds: cols[f] = list() @@ -94,7 +97,10 @@ def facetcolumns(table, key, missing=None): fct = dict() it = iter(table) - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + hdr = [] flds = list(map(text_type, hdr)) indices = asindices(hdr, key) assert len(indices) > 0, 'no key field selected' diff --git a/petl/util/vis.py b/petl/util/vis.py index e1d830a2..a75baf70 100644 --- a/petl/util/vis.py +++ b/petl/util/vis.py @@ -194,7 +194,10 @@ def _look_grid(table, vrepr, index_header, truncate, width): it = iter(table) # fields representation - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return '' flds = list(map(text_type, hdr)) if index_header: fldsrepr = ['%s|%s' % (i, r) for (i, r) in enumerate(flds)] @@ -294,7 +297,10 @@ def _look_simple(table, vrepr, index_header, truncate, width): it = iter(table) # fields representation - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return '' flds = list(map(text_type, hdr)) if index_header: fldsrepr = ['%s|%s' % (i, r) for (i, r) in enumerate(flds)] @@ -377,7 +383,10 @@ def _look_minimal(table, vrepr, index_header, truncate, width): it = iter(table) # fields representation - hdr = next(it) + try: + hdr = next(it) + except StopIteration: + return '' flds = list(map(text_type, hdr)) if index_header: fldsrepr = ['%s|%s' % (i, r) for (i, r) in enumerate(flds)] @@ -495,7 +504,10 @@ def __repr__(self): # construct output output = '' it = iter(table) - flds = next(it) + try: + flds = next(it) + except StopIteration: + return '' cols = defaultdict(list) for row in it: for i, f in enumerate(flds):