From 326107c48bf6c506c2e75e623330c221b8f10a1a Mon Sep 17 00:00:00 2001
From: Kenneth <24253983+aerymilts@users.noreply.github.com>
Date: Sun, 18 Mar 2018 18:44:40 +0800
Subject: [PATCH] BUG: fixed json_normalize for subrecords with NoneTypes
 (#20030)

TST: additional coverage for the test cases from (#20030)

DOC: added changes to whatsnew/v0.23.0.txt (#20030)
---
 doc/source/whatsnew/v0.23.0.txt        |  1 +
 pandas/io/json/normalize.py            |  5 ++-
 pandas/tests/io/json/test_normalize.py | 50 ++++++++++++++++++++++++++
 3 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt
index cfe28edd175b64..f0fc62f455fd11 100644
--- a/doc/source/whatsnew/v0.23.0.txt
+++ b/doc/source/whatsnew/v0.23.0.txt
@@ -979,6 +979,7 @@ I/O
 - :class:`Timedelta` now supported in :func:`DataFrame.to_excel` for all Excel file types (:issue:`19242`, :issue:`9155`, :issue:`19900`)
 - Bug in :meth:`pandas.io.stata.StataReader.value_labels` raising an ``AttributeError`` when called on very old files. Now returns an empty dict (:issue:`19417`)
 - Bug in :func:`read_pickle` when unpickling objects with :class:`TimedeltaIndex` or :class:`Float64Index` created with pandas prior to version 0.20 (:issue:`19939`)
+- Bug in :meth:`pandas.io.json.json_normalize` where subrecords are not properly normalized if any subrecords values are NoneType (:issue:`20030`)
 
 Plotting
 ^^^^^^^^
diff --git a/pandas/io/json/normalize.py b/pandas/io/json/normalize.py
index c7901f4352d001..549204abd3caff 100644
--- a/pandas/io/json/normalize.py
+++ b/pandas/io/json/normalize.py
@@ -80,6 +80,8 @@ def nested_to_record(ds, prefix="", sep=".", level=0):
                 if level != 0:  # so we skip copying for top level, common case
                     v = new_d.pop(k)
                     new_d[newkey] = v
+                if v is None:  # pop the key if the value is None
+                    new_d.pop(k)
                 continue
             else:
                 v = new_d.pop(k)
@@ -189,7 +191,8 @@ def _pull_field(js, spec):
         data = [data]
 
     if record_path is None:
-        if any(isinstance(x, dict) for x in compat.itervalues(data[0])):
+        if any([[isinstance(x, dict)
+                for x in compat.itervalues(y)] for y in data]):
             # naive normalization, this is idempotent for flat records
             # and potentially will inflate the data considerably for
             # deeply nested structures:
diff --git a/pandas/tests/io/json/test_normalize.py b/pandas/tests/io/json/test_normalize.py
index 1cceae32cd748d..09d52066f84db5 100644
--- a/pandas/tests/io/json/test_normalize.py
+++ b/pandas/tests/io/json/test_normalize.py
@@ -1,6 +1,7 @@
 import pytest
 import numpy as np
 import json
+import math
 
 import pandas.util.testing as tm
 from pandas import compat, Index, DataFrame
@@ -54,6 +55,17 @@ def state_data():
          'state': 'Ohio'}]
 
 
+@pytest.fixture
+def author_missing_data():
+    return [
+        {'info': None},
+        {'info':
+            {'created_at': '11/08/1993', 'last_updated': '26/05/2012'},
+            'author_name':
+         {'first': 'Jane', 'last_name': 'Doe'}
+         }]
+
+
 class TestJSONNormalize(object):
 
     def test_simple_records(self):
@@ -226,6 +238,21 @@ def test_non_ascii_key(self):
         result = json_normalize(json.loads(testjson))
         tm.assert_frame_equal(result, expected)
 
+    def test_missing_field(self, author_missing_data):
+        result = json_normalize(author_missing_data)
+        ex_data = [
+            {'author_name.first': math.nan,
+             'author_name.last_name': math.nan,
+             'info.created_at': math.nan,
+             'info.last_updated': math.nan},
+            {'author_name.first': 'Jane',
+             'author_name.last_name': 'Doe',
+             'info.created_at': '11/08/1993',
+             'info.last_updated': '26/05/2012'}
+        ]
+        expected = DataFrame(ex_data)
+        tm.assert_frame_equal(result, expected)
+
 
 class TestNestedToRecord(object):
 
@@ -322,3 +349,26 @@ def test_json_normalize_errors(self):
                             ['general', 'trade_version']],
                       errors='raise'
                       )
+
+    def test_nonetype_dropping(self):
+        data = [
+            {'info': None,
+             'author_name':
+             {'first': 'Smith', 'last_name': 'Appleseed'}
+             },
+            {'info':
+                {'created_at': '11/08/1993', 'last_updated': '26/05/2012'},
+             'author_name':
+                {'first': 'Jane', 'last_name': 'Doe'}
+             }
+        ]
+        result = nested_to_record(data)
+        expected = [
+            {'author_name.first': 'Smith',
+             'author_name.last_name': 'Appleseed'},
+            {'author_name.first': 'Jane',
+             'author_name.last_name': 'Doe',
+             'info.created_at': '11/08/1993',
+             'info.last_updated': '26/05/2012'}]
+
+        assert result == expected