From 63eb23682e98b2d9fddf0bf61eb80996b760a920 Mon Sep 17 00:00:00 2001 From: Bryan Yang Date: Tue, 11 Aug 2020 01:52:39 +0800 Subject: [PATCH 1/5] Add who use wr (#347) * update who use * remove underscore --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0ffa08711..e0fb95bfd 100644 --- a/README.md +++ b/README.md @@ -124,3 +124,4 @@ Please send a PR with your company name and @githubhandle if you may. 2. [Pier](https://www.pier.digital/) [[@flaviomax](https://github.com/flaviomax)] 3. [M4U](https://www.m4u.com.br/) [[@Thiago-Dantas](https://github.com/Thiago-Dantas)] 4. [Serasa Experian](https://www.serasaexperian.com.br/) [[@andre-marcos-perez](https://github.com/andre-marcos-perez)] +5. [LINE TV](https://www.linetv.tw/) [[@bryanyang0528](https://github.com/bryanyang0528)] From 39b4d46a9e5c509c28e5925e423a2ccf18ff0784 Mon Sep 17 00:00:00 2001 From: Igor Tavares Date: Mon, 10 Aug 2020 18:26:58 -0300 Subject: [PATCH 2/5] Add e-mail. --- README.md | 2 +- docs/source/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e0fb95bfd..7b18bf38a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ![AWS Data Wrangler](docs/source/_static/logo2.png?raw=true "AWS Data Wrangler") -> An [AWS Professional Service](https://aws.amazon.com/professional-services/) open source initiative +> An [AWS Professional Service](https://aws.amazon.com/professional-services/) open source initiative | aws-proserve-opensource@amazon.com [![Release](https://img.shields.io/badge/release-1.8.0-brightgreen.svg)](https://pypi.org/project/awswrangler/) [![Python Version](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-brightgreen.svg)](https://anaconda.org/conda-forge/awswrangler) diff --git a/docs/source/index.rst b/docs/source/index.rst index 9d457b769..9325f83d6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,4 +1,4 @@ -An `AWS Professional Service `_ open source initiative. +An `AWS Professional Service `_ open source initiative | aws-proserve-opensource@amazon.com Quick Start ----------- From 249743fe3c50e8e919e2d7d041199831973e4e51 Mon Sep 17 00:00:00 2001 From: Igor Tavares Date: Tue, 11 Aug 2020 10:28:35 -0300 Subject: [PATCH 3/5] Fix Athena NaN values when ctas_approach is False. #351 --- awswrangler/athena/_read.py | 2 +- tests/test_athena.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/awswrangler/athena/_read.py b/awswrangler/athena/_read.py index 433f79cd5..c04cfcac3 100644 --- a/awswrangler/athena/_read.py +++ b/awswrangler/athena/_read.py @@ -265,7 +265,7 @@ def _fetch_csv_result( converters=query_metadata.converters, quoting=csv.QUOTE_ALL, keep_default_na=False, - na_values=[""], + na_values=["", "NaN"], chunksize=_chunksize, skip_blank_lines=False, use_threads=False, diff --git a/tests/test_athena.py b/tests/test_athena.py index f2762104d..576c77bbf 100644 --- a/tests/test_athena.py +++ b/tests/test_athena.py @@ -4,6 +4,7 @@ import boto3 import pandas as pd import pytest +import numpy as np import awswrangler as wr @@ -772,3 +773,17 @@ def test_parse_describe_table(): def test_describe_table(path, glue_database, glue_table): wr.catalog.create_parquet_table(database=glue_database, table=glue_table, path=path, columns_types={"c0": "int"}) assert wr.athena.describe_table(database=glue_database, table=glue_table).shape == (1, 4) + + +@pytest.mark.parametrize("ctas_approach", [False, True]) +def test_athena_nan_inf(glue_database, ctas_approach): + sql = "SELECT nan() AS nan, infinity() as inf, -infinity() as inf_n, 1.2 as regular" + df = wr.athena.read_sql_query(sql, glue_database, ctas_approach) + print(df) + print(df.dtypes) + assert df.shape == (1, 4) + assert df.dtypes.to_list() == ["float64", "float64", "float64", "float64"] + assert np.isnan(df.nan.iloc[0]) + assert df.inf.iloc[0] == np.PINF + assert df.inf_n.iloc[0] == np.NINF + assert df.regular.iloc[0] == 1.2 From bbdf425232bc9de012aabc06fe703c8ea4a8ae1e Mon Sep 17 00:00:00 2001 From: Igor Tavares Date: Tue, 11 Aug 2020 10:59:00 -0300 Subject: [PATCH 4/5] Applying isort. #351 --- tests/test_athena.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_athena.py b/tests/test_athena.py index 576c77bbf..88c9958a9 100644 --- a/tests/test_athena.py +++ b/tests/test_athena.py @@ -2,9 +2,9 @@ import logging import boto3 +import numpy as np import pandas as pd import pytest -import numpy as np import awswrangler as wr From bfb547a42995ac3a2ad1cce1378bf27514b4aca5 Mon Sep 17 00:00:00 2001 From: Igor Tavares Date: Tue, 11 Aug 2020 13:57:12 -0300 Subject: [PATCH 5/5] Bumping version to 1.8.1 --- README.md | 2 +- awswrangler/__metadata__.py | 2 +- docs/source/install.rst | 4 ++-- tests/test_metadata.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 42950e0a4..3f05c4dd9 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ > An [AWS Professional Service](https://aws.amazon.com/professional-services/) open source initiative | aws-proserve-opensource@amazon.com -[![Release](https://img.shields.io/badge/release-1.8.0-brightgreen.svg)](https://pypi.org/project/awswrangler/) +[![Release](https://img.shields.io/badge/release-1.8.1-brightgreen.svg)](https://pypi.org/project/awswrangler/) [![Python Version](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-brightgreen.svg)](https://anaconda.org/conda-forge/awswrangler) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) diff --git a/awswrangler/__metadata__.py b/awswrangler/__metadata__.py index 8ee60f0e7..b92119688 100644 --- a/awswrangler/__metadata__.py +++ b/awswrangler/__metadata__.py @@ -7,5 +7,5 @@ __title__: str = "awswrangler" __description__: str = "Pandas on AWS." -__version__: str = "1.8.0" +__version__: str = "1.8.1" __license__: str = "Apache License 2.0" diff --git a/docs/source/install.rst b/docs/source/install.rst index b4fb7e253..e462b88d5 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -47,8 +47,8 @@ the *Python library path* field. `Official Glue Python Shell Reference `_ -AWS Glue Python PySpark Jobs ----------------------------- +AWS Glue PySpark Jobs +--------------------- .. note:: AWS Data Wrangler has compiled dependencies (C/C++) so there is only support for ``Glue PySpark Jobs >= 2.0``. diff --git a/tests/test_metadata.py b/tests/test_metadata.py index d644ee4aa..be7685063 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -2,7 +2,7 @@ def test_metadata(): - assert wr.__version__ == "1.8.0" + assert wr.__version__ == "1.8.1" assert wr.__title__ == "awswrangler" assert wr.__description__ == "Pandas on AWS." assert wr.__license__ == "Apache License 2.0"