Skip to content

Commit

Permalink
avoid doctest failing when missing bcolz or pytables (hard setup) (#493)
Browse files Browse the repository at this point in the history 
* avoid doctest fail when missing bcolz or pytables (hard setup)

This is just a quality of life improvement.
Changed docstring for skipping doctests for bcolz and pytables.
It's hard to setup bcolz and pytables because they need local packages installed.
Every time that included code in docstrings they failed leading and one need to erase the docstring for getting the test to pass.

* Apply suggestions from code review of avro support in pull #493

Good catch!

Co-authored-by: Alistair Miles <alimanfoo@googlemail.com>

Co-authored-by: Juarez Rudsatz <juarez.rudsatz@ceabs.com.br>
Co-authored-by: Alistair Miles <alimanfoo@googlemail.com>
  • Loading branch information
@juarezr @alimanfoo
3 people committed Jun 25, 2020
1 parent ee769a7 commit 65139a5
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 88 deletions.
7 changes: 4 additions & 3 deletions optional_requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
Cython==0.29.13
numpy==1.16.4
numexpr==2.6.9
bcolz==1.2.1
tables==3.5.2
intervaltree==3.0.2
lxml==4.4.0
openpyxl==2.6.2
pandas==0.24.2
psycopg2==2.8.3
PyMySQL==0.9.3
SQLAlchemy==1.3.6
Whoosh==2.7.4
Expand All @@ -18,3 +15,7 @@ lz4
zstandard
smbprotocol>=1.0.1
s3fs>=0.2.2
# packages bellow need complex local setup
psycopg2==2.8.3
bcolz==1.2.1
tables==3.5.2
47 changes: 27 additions & 20 deletions petl/io/bcolz.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,19 @@ def frombcolz(source, expression=None, outcols=None, limit=None, skip=0):
"""Extract a table from a bcolz ctable, e.g.::
>>> import petl as etl
>>> import bcolz
>>> cols = [
... ['apples', 'oranges', 'pears'],
... [1, 3, 7],
... [2.5, 4.4, .1]
... ]
>>> names = ('foo', 'bar', 'baz')
>>> ctbl = bcolz.ctable(cols, names=names)
>>> tbl = etl.frombcolz(ctbl)
>>> tbl
>>>
>>> def example_from_bcolz():
... import bcolz
... cols = [
... ['apples', 'oranges', 'pears'],
... [1, 3, 7],
... [2.5, 4.4, .1]
... ]
... names = ('foo', 'bar', 'baz')
... ctbl = bcolz.ctable(cols, names=names)
... return etl.frombcolz(ctbl)
>>>
>>> example_from_bcolz() # doctest: +SKIP
+-----------+-----+-----+
| foo | bar | baz |
+===========+=====+=====+
Expand All @@ -35,8 +38,8 @@ def frombcolz(source, expression=None, outcols=None, limit=None, skip=0):
If `expression` is provided it will be executed by bcolz and only
matching rows returned, e.g.::
>>> tbl2 = etl.frombcolz(ctbl, expression='bar > 1')
>>> tbl2
>>> tbl2 = etl.frombcolz(ctbl, expression='bar > 1') # doctest: +SKIP
>>> tbl2 # doctest: +SKIP
+-----------+-----+-----+
| foo | bar | baz |
+===========+=====+=====+
Expand Down Expand Up @@ -97,19 +100,23 @@ def tobcolz(table, dtype=None, sample=1000, **kwargs):
"""Load data into a bcolz ctable, e.g.::
>>> import petl as etl
>>> table = [('foo', 'bar', 'baz'),
... ('apples', 1, 2.5),
... ('oranges', 3, 4.4),
... ('pears', 7, .1)]
>>> ctbl = etl.tobcolz(table)
>>> ctbl
>>>
>>> def example_to_bcolz():
... table = [('foo', 'bar', 'baz'),
... ('apples', 1, 2.5),
... ('oranges', 3, 4.4),
... ('pears', 7, .1)]
... return etl.tobcolz(table)
>>>
>>> ctbl = example_to_bcolz() # doctest: +SKIP
>>> ctbl # doctest: +SKIP
ctable((3,), [('foo', '<U7'), ('bar', '<i8'), ('baz', '<f8')])
nbytes: 132; cbytes: 1023.98 KB; ratio: 0.00
cparams := cparams(clevel=5, shuffle=1, cname='lz4', quantize=0)
[('apples', 1, 2.5) ('oranges', 3, 4.4) ('pears', 7, 0.1)]
>>> ctbl.names
>>> ctbl.names # doctest: +SKIP
['foo', 'bar', 'baz']
>>> ctbl['foo']
>>> ctbl['foo'] # doctest: +SKIP
carray((3,), <U7)
nbytes := 84; cbytes := 511.98 KB; ratio: 0.00
cparams := cparams(clevel=5, shuffle=1, cname='lz4', quantize=0)
Expand Down
132 changes: 67 additions & 65 deletions petl/io/pytables.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,35 +17,36 @@ def fromhdf5(source, where=None, name=None, condition=None,
Provides access to an HDF5 table. E.g.::
>>> import petl as etl
>>> import tables
>>>
>>> # set up a new hdf5 table to demonstrate with
... h5file = tables.open_file('example.h5', mode='w',
... title='Example file')
>>> h5file.create_group('/', 'testgroup', 'Test Group')
/testgroup (Group) 'Test Group'
children := []
>>> class FooBar(tables.IsDescription):
... foo = tables.Int32Col(pos=0)
... bar = tables.StringCol(6, pos=2)
...
>>> h5table = h5file.create_table('/testgroup', 'testtable', FooBar,
... 'Test Table')
>>> # load some data into the table
... table1 = (('foo', 'bar'),
... (1, b'asdfgh'),
... (2, b'qwerty'),
... (3, b'zxcvbn'))
>>> for row in table1[1:]:
... for i, f in enumerate(table1[0]):
... h5table.row[f] = row[i]
... h5table.row.append()
...
>>> h5file.flush()
>>> h5file.close()
>>> class FooBar(tables.IsDescription): # doctest: +SKIP
... foo = tables.Int32Col(pos=0) # doctest: +SKIP
... bar = tables.StringCol(6, pos=2) # doctest: +SKIP
>>> #
... # now demonstrate use of fromhdf5
... table1 = etl.fromhdf5('example.h5', '/testgroup', 'testtable')
>>> table1
>>> def setup_hdf5_table():
... import tables
... h5file = tables.open_file('example.h5', mode='w',
... title='Example file')
... h5file.create_group('/', 'testgroup', 'Test Group')
... h5table = h5file.create_table('/testgroup', 'testtable', FooBar,
... 'Test Table')
... # load some data into the table
... table1 = (('foo', 'bar'),
... (1, b'asdfgh'),
... (2, b'qwerty'),
... (3, b'zxcvbn'))
... for row in table1[1:]:
... for i, f in enumerate(table1[0]):
... h5table.row[f] = row[i]
... h5table.row.append()
... h5file.flush()
... h5file.close()
>>>
>>> setup_hdf5_table() # doctest: +SKIP
>>>
>>> # now demonstrate use of fromhdf5
>>> table1 = etl.fromhdf5('example.h5', '/testgroup', 'testtable') # doctest: +SKIP
>>> table1 # doctest: +SKIP
+-----+-----------+
| foo | bar |
+=====+===========+
Expand All @@ -57,16 +58,16 @@ def fromhdf5(source, where=None, name=None, condition=None,
+-----+-----------+
>>> # alternatively just specify path to table node
... table1 = etl.fromhdf5('example.h5', '/testgroup/testtable')
... table1 = etl.fromhdf5('example.h5', '/testgroup/testtable') # doctest: +SKIP
>>> # ...or use an existing tables.File object
... h5file = tables.open_file('example.h5')
>>> table1 = etl.fromhdf5(h5file, '/testgroup/testtable')
... h5file = tables.open_file('example.h5') # doctest: +SKIP
>>> table1 = etl.fromhdf5(h5file, '/testgroup/testtable') # doctest: +SKIP
>>> # ...or use an existing tables.Table object
... h5tbl = h5file.get_node('/testgroup/testtable')
>>> table1 = etl.fromhdf5(h5tbl)
... h5tbl = h5file.get_node('/testgroup/testtable') # doctest: +SKIP
>>> table1 = etl.fromhdf5(h5tbl) # doctest: +SKIP
>>> # use a condition to filter data
... table2 = etl.fromhdf5(h5tbl, condition='foo < 3')
>>> table2
... table2 = etl.fromhdf5(h5tbl, condition='foo < 3') # doctest: +SKIP
>>> table2 # doctest: +SKIP
+-----+-----------+
| foo | bar |
+=====+===========+
Expand All @@ -75,7 +76,7 @@ def fromhdf5(source, where=None, name=None, condition=None,
| 2 | b'qwerty' |
+-----+-----------+
>>> h5file.close()
>>> h5file.close() # doctest: +SKIP
"""

Expand Down Expand Up @@ -201,36 +202,37 @@ def fromhdf5sorted(source, where=None, name=None, sortby=None, checkCSI=False,
Provides access to an HDF5 table, sorted by an indexed column, e.g.::
>>> import petl as etl
>>> import tables
>>>
>>> # set up a new hdf5 table to demonstrate with
... h5file = tables.open_file('example.h5', mode='w', title='Test file')
>>> h5file.create_group('/', 'testgroup', 'Test Group')
/testgroup (Group) 'Test Group'
children := []
>>> class FooBar(tables.IsDescription):
... foo = tables.Int32Col(pos=0)
... bar = tables.StringCol(6, pos=2)
...
>>> h5table = h5file.create_table('/testgroup', 'testtable', FooBar, 'Test Table')
>>> # load some data into the table
... table1 = (('foo', 'bar'),
... (3, b'asdfgh'),
... (2, b'qwerty'),
... (1, b'zxcvbn'))
>>> for row in table1[1:]:
... for i, f in enumerate(table1[0]):
... h5table.row[f] = row[i]
... h5table.row.append()
...
>>> h5table.cols.foo.create_csindex() # CS index is required
0
>>> h5file.flush()
>>> h5file.close()
>>> #
>>> class FooBar(tables.IsDescription): # doctest: +SKIP
... foo = tables.Int32Col(pos=0) # doctest: +SKIP
... bar = tables.StringCol(6, pos=2) # doctest: +SKIP
>>>
>>> def setup_hdf5_index():
... import tables
... h5file = tables.open_file('example.h5', mode='w',
... title='Example file')
... h5file.create_group('/', 'testgroup', 'Test Group')
... h5table = h5file.create_table('/testgroup', 'testtable', FooBar,
... 'Test Table')
... # load some data into the table
... table1 = (('foo', 'bar'),
... (1, b'asdfgh'),
... (2, b'qwerty'),
... (3, b'zxcvbn'))
... for row in table1[1:]:
... for i, f in enumerate(table1[0]):
... h5table.row[f] = row[i]
... h5table.row.append()
... h5table.cols.foo.create_csindex() # CS index is required
... h5file.flush()
... h5file.close()
>>>
>>> setup_hdf5_index() # doctest: +SKIP
>>>
... # access the data, sorted by the indexed column
... table2 = etl.fromhdf5sorted('example.h5', '/testgroup', 'testtable',
... sortby='foo')
>>> table2
... table2 = etl.fromhdf5sorted('example.h5', '/testgroup', 'testtable', sortby='foo') # doctest: +SKIP
>>> table2 # doctest: +SKIP
+-----+-----------+
| foo | bar |
+=====+===========+
Expand Down Expand Up @@ -301,8 +303,8 @@ def tohdf5(table, source, where=None, name=None, create=False, drop=False,
... (2, b'qwerty'),
... (3, b'zxcvbn'))
>>> etl.tohdf5(table1, 'example.h5', '/testgroup', 'testtable',
... drop=True, create=True, createparents=True)
>>> etl.fromhdf5('example.h5', '/testgroup', 'testtable')
... drop=True, create=True, createparents=True) # doctest: +SKIP
>>> etl.fromhdf5('example.h5', '/testgroup', 'testtable') # doctest: +SKIP
+-----+-----------+
| foo | bar |
+=====+===========+
Expand Down

0 comments on commit 65139a5

Please sign in to comment.