diff --git a/README.md b/README.md index 42950e0a4..3f05c4dd9 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ > An [AWS Professional Service](https://aws.amazon.com/professional-services/) open source initiative | aws-proserve-opensource@amazon.com -[![Release](https://img.shields.io/badge/release-1.8.0-brightgreen.svg)](https://pypi.org/project/awswrangler/) +[![Release](https://img.shields.io/badge/release-1.8.1-brightgreen.svg)](https://pypi.org/project/awswrangler/) [![Python Version](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-brightgreen.svg)](https://anaconda.org/conda-forge/awswrangler) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) diff --git a/awswrangler/__metadata__.py b/awswrangler/__metadata__.py index 8ee60f0e7..b92119688 100644 --- a/awswrangler/__metadata__.py +++ b/awswrangler/__metadata__.py @@ -7,5 +7,5 @@ __title__: str = "awswrangler" __description__: str = "Pandas on AWS." -__version__: str = "1.8.0" +__version__: str = "1.8.1" __license__: str = "Apache License 2.0" diff --git a/awswrangler/athena/_read.py b/awswrangler/athena/_read.py index 433f79cd5..c04cfcac3 100644 --- a/awswrangler/athena/_read.py +++ b/awswrangler/athena/_read.py @@ -265,7 +265,7 @@ def _fetch_csv_result( converters=query_metadata.converters, quoting=csv.QUOTE_ALL, keep_default_na=False, - na_values=[""], + na_values=["", "NaN"], chunksize=_chunksize, skip_blank_lines=False, use_threads=False, diff --git a/docs/source/install.rst b/docs/source/install.rst index b4fb7e253..e462b88d5 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -47,8 +47,8 @@ the *Python library path* field. `Official Glue Python Shell Reference `_ -AWS Glue Python PySpark Jobs ----------------------------- +AWS Glue PySpark Jobs +--------------------- .. note:: AWS Data Wrangler has compiled dependencies (C/C++) so there is only support for ``Glue PySpark Jobs >= 2.0``. diff --git a/tests/test_athena.py b/tests/test_athena.py index f2762104d..88c9958a9 100644 --- a/tests/test_athena.py +++ b/tests/test_athena.py @@ -2,6 +2,7 @@ import logging import boto3 +import numpy as np import pandas as pd import pytest @@ -772,3 +773,17 @@ def test_parse_describe_table(): def test_describe_table(path, glue_database, glue_table): wr.catalog.create_parquet_table(database=glue_database, table=glue_table, path=path, columns_types={"c0": "int"}) assert wr.athena.describe_table(database=glue_database, table=glue_table).shape == (1, 4) + + +@pytest.mark.parametrize("ctas_approach", [False, True]) +def test_athena_nan_inf(glue_database, ctas_approach): + sql = "SELECT nan() AS nan, infinity() as inf, -infinity() as inf_n, 1.2 as regular" + df = wr.athena.read_sql_query(sql, glue_database, ctas_approach) + print(df) + print(df.dtypes) + assert df.shape == (1, 4) + assert df.dtypes.to_list() == ["float64", "float64", "float64", "float64"] + assert np.isnan(df.nan.iloc[0]) + assert df.inf.iloc[0] == np.PINF + assert df.inf_n.iloc[0] == np.NINF + assert df.regular.iloc[0] == 1.2 diff --git a/tests/test_metadata.py b/tests/test_metadata.py index d644ee4aa..be7685063 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -2,7 +2,7 @@ def test_metadata(): - assert wr.__version__ == "1.8.0" + assert wr.__version__ == "1.8.1" assert wr.__title__ == "awswrangler" assert wr.__description__ == "Pandas on AWS." assert wr.__license__ == "Apache License 2.0"