From 119bf9b4179c9b8aea25e621e2e0d1c42776d5de Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 00:54:26 +0800 Subject: [PATCH 01/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- .../tests/indexing/multiindex/test_setitem.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index d732cb4d7fbbc..071e49a9506b3 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -490,6 +490,50 @@ def test_setitem_enlargement_keep_index_names(self): ) tm.assert_frame_equal(df, expected) + def test_setitem_enlargement_with_none_key(self): + # GH#59153 + # Test that enlarging a MultiIndex DataFrame works when one or more + # level keys are None + index = MultiIndex.from_tuples( + [("A", "a1"), ("A", "a2"), ("B", "b1"), ("B", None)] + ) + df = DataFrame([(0, 6), (1, 5), (2, 4), (3, 7)], index=index) + + # Test 1: Enlarge with a new index entry where second key is None + df.loc[("A", None), :] = [12, 13] + expected_index = MultiIndex.from_tuples( + [ + ("A", "a1"), + ("A", "a2"), + ("B", "b1"), + ("B", None), + ("A", None), + ] + ) + expected = DataFrame( + [[0, 6], [1, 5], [2, 4], [3, 7], [12, 13]], + index=expected_index, + ) + tm.assert_frame_equal(df, expected) + + # Test 2: Enlarge with None in first level key + df.loc[(None, "c1"), :] = [14, 15] + expected_index = MultiIndex.from_tuples( + [ + ("A", "a1"), + ("A", "a2"), + ("B", "b1"), + ("B", None), + ("A", None), + (None, "c1"), + ] + ) + expected = DataFrame( + [[0, 6], [1, 5], [2, 4], [3, 7], [12, 13], [14, 15]], + index=expected_index, + ) + tm.assert_frame_equal(df, expected) + def test_frame_setitem_view_direct(multiindex_dataframe_random_data): # this works because we are modifying the underlying array From 1199d445407099e0a0ba2403228863d847a634d9 Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 01:08:18 +0800 Subject: [PATCH 02/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- .../tests/indexing/multiindex/test_setitem.py | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index 071e49a9506b3..940d5455706c3 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -492,14 +492,13 @@ def test_setitem_enlargement_keep_index_names(self): def test_setitem_enlargement_with_none_key(self): # GH#59153 - # Test that enlarging a MultiIndex DataFrame works when one or more - # level keys are None + # Test basic enlargement with None keys in different levels index = MultiIndex.from_tuples( [("A", "a1"), ("A", "a2"), ("B", "b1"), ("B", None)] ) df = DataFrame([(0, 6), (1, 5), (2, 4), (3, 7)], index=index) - # Test 1: Enlarge with a new index entry where second key is None + # Enlarge with None in second level df.loc[("A", None), :] = [12, 13] expected_index = MultiIndex.from_tuples( [ @@ -516,7 +515,7 @@ def test_setitem_enlargement_with_none_key(self): ) tm.assert_frame_equal(df, expected) - # Test 2: Enlarge with None in first level key + # Enlarge with None in first level df.loc[(None, "c1"), :] = [14, 15] expected_index = MultiIndex.from_tuples( [ @@ -534,6 +533,45 @@ def test_setitem_enlargement_with_none_key(self): ) tm.assert_frame_equal(df, expected) + def test_setitem_enlargement_none_key_indexslice_retrieval(self): + # GH#59153 + # Test IndexSlice functionality and retrieval with None keys + + # Test IndexSlice selection with None + idx = pd.IndexSlice + df = DataFrame( + [[10], [20], [30]], + index=MultiIndex.from_tuples([("A", "a1"), ("B", None), ("C", "c1")]), + columns=["val"] + ) + + result = df.loc[idx[:, None], :] + expected = DataFrame([[20]], index=MultiIndex.from_tuples([("B", None)]), columns=["val"]) + tm.assert_frame_equal(result, expected) + + # Enlarge using IndexSlice with None + df.loc[idx["D", None], :] = [40] + assert df.loc[("D", None), "val"] == 40 + + # Test retrieval after enlargement + df2 = DataFrame( + [[100, 200]], + index=MultiIndex.from_tuples([("A", "a1")]), + columns=["col1", "col2"] + ) + + df2.loc[("A", None), :] = [300, 400] + + # Retrieve using full tuple key + result = df2.loc[("A", None), :] + expected = Series([300, 400], index=["col1", "col2"], name=("A", None)) + tm.assert_series_equal(result, expected) + + # Retrieve using xs + result_xs = df2.xs("A") + assert None in result_xs.index + assert result_xs.loc[None, "col1"] == 300 + def test_frame_setitem_view_direct(multiindex_dataframe_random_data): # this works because we are modifying the underlying array From 0092a7cd2364557bf170e0c802ab95d159db343a Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 01:13:06 +0800 Subject: [PATCH 03/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- pandas/tests/indexing/multiindex/test_setitem.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index 940d5455706c3..36f9a3ba62f2f 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -546,7 +546,11 @@ def test_setitem_enlargement_none_key_indexslice_retrieval(self): ) result = df.loc[idx[:, None], :] - expected = DataFrame([[20]], index=MultiIndex.from_tuples([("B", None)]), columns=["val"]) + expected = DataFrame( + [[20]], + index=MultiIndex.from_tuples([("B", None)]), + columns=["val"], + ) tm.assert_frame_equal(result, expected) # Enlarge using IndexSlice with None From 0004ec64f50f3200eb1d78d46a3cd7dba8f82c23 Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 09:56:07 +0800 Subject: [PATCH 04/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- .../tests/indexing/multiindex/test_setitem.py | 87 ++++++------------- 1 file changed, 28 insertions(+), 59 deletions(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index 36f9a3ba62f2f..2bf6eca05f1d9 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -490,92 +490,61 @@ def test_setitem_enlargement_keep_index_names(self): ) tm.assert_frame_equal(df, expected) - def test_setitem_enlargement_with_none_key(self): + def test_setitem_enlargement_multiindex_with_none(self): # GH#59153 - # Test basic enlargement with None keys in different levels + # Test that we can enlarge a DataFrame with a MultiIndex + # when one or more level keys are None index = MultiIndex.from_tuples( [("A", "a1"), ("A", "a2"), ("B", "b1"), ("B", None)] ) df = DataFrame([(0, 6), (1, 5), (2, 4), (3, 7)], index=index) - # Enlarge with None in second level + # Enlarge with a new index entry where one key is None df.loc[("A", None), :] = [12, 13] + expected_index = MultiIndex.from_tuples( - [ - ("A", "a1"), - ("A", "a2"), - ("B", "b1"), - ("B", None), - ("A", None), - ] + [("A", "a1"), ("A", "a2"), ("B", "b1"), ("B", None), ("A", None)] ) expected = DataFrame( [[0, 6], [1, 5], [2, 4], [3, 7], [12, 13]], index=expected_index, + columns=[0, 1], ) tm.assert_frame_equal(df, expected) - # Enlarge with None in first level - df.loc[(None, "c1"), :] = [14, 15] + # Test retrieval of the newly added row + result = df.loc[("A", None), :] + expected_row = Series([12, 13], index=[0, 1], name=("A", np.nan)) + tm.assert_series_equal(result, expected_row) + + def test_setitem_enlargement_multiindex_multiple_none(self): + # GH#59153 + # Test enlarging with multiple None keys in different levels + index = MultiIndex.from_tuples([("A", "a1"), ("B", "b1")]) + df = DataFrame([[1, 2], [3, 4]], index=index, columns=["x", "y"]) + + # Add row with None in first level + df.loc[(None, "c1"), :] = [5, 6] + + # Add row with None in second level + df.loc[("C", None), :] = [7, 8] + + # (None, None) case removed as requested expected_index = MultiIndex.from_tuples( [ ("A", "a1"), - ("A", "a2"), ("B", "b1"), - ("B", None), - ("A", None), (None, "c1"), + ("C", None), ] ) expected = DataFrame( - [[0, 6], [1, 5], [2, 4], [3, 7], [12, 13], [14, 15]], + [[1, 2], [3, 4], [5, 6], [7, 8]], index=expected_index, + columns=["x", "y"], ) tm.assert_frame_equal(df, expected) - def test_setitem_enlargement_none_key_indexslice_retrieval(self): - # GH#59153 - # Test IndexSlice functionality and retrieval with None keys - - # Test IndexSlice selection with None - idx = pd.IndexSlice - df = DataFrame( - [[10], [20], [30]], - index=MultiIndex.from_tuples([("A", "a1"), ("B", None), ("C", "c1")]), - columns=["val"] - ) - - result = df.loc[idx[:, None], :] - expected = DataFrame( - [[20]], - index=MultiIndex.from_tuples([("B", None)]), - columns=["val"], - ) - tm.assert_frame_equal(result, expected) - - # Enlarge using IndexSlice with None - df.loc[idx["D", None], :] = [40] - assert df.loc[("D", None), "val"] == 40 - - # Test retrieval after enlargement - df2 = DataFrame( - [[100, 200]], - index=MultiIndex.from_tuples([("A", "a1")]), - columns=["col1", "col2"] - ) - - df2.loc[("A", None), :] = [300, 400] - - # Retrieve using full tuple key - result = df2.loc[("A", None), :] - expected = Series([300, 400], index=["col1", "col2"], name=("A", None)) - tm.assert_series_equal(result, expected) - - # Retrieve using xs - result_xs = df2.xs("A") - assert None in result_xs.index - assert result_xs.loc[None, "col1"] == 300 - def test_frame_setitem_view_direct(multiindex_dataframe_random_data): # this works because we are modifying the underlying array From 93cf66c7057df09213f86a54319b5fe71a313356 Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 10:16:58 +0800 Subject: [PATCH 05/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- pandas/tests/indexing/multiindex/test_setitem.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index 2bf6eca05f1d9..ee7743fb30bf5 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -509,6 +509,7 @@ def test_setitem_enlargement_multiindex_with_none(self): [[0, 6], [1, 5], [2, 4], [3, 7], [12, 13]], index=expected_index, columns=[0, 1], + dtype=float, ) tm.assert_frame_equal(df, expected) @@ -542,6 +543,7 @@ def test_setitem_enlargement_multiindex_multiple_none(self): [[1, 2], [3, 4], [5, 6], [7, 8]], index=expected_index, columns=["x", "y"], + dtype=float, ) tm.assert_frame_equal(df, expected) From 9f127a0e6e1e975079cc13aafe971028199d85fa Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 10:37:51 +0800 Subject: [PATCH 06/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- pandas/tests/indexing/multiindex/test_setitem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index ee7743fb30bf5..e2c8f021319e7 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -515,7 +515,7 @@ def test_setitem_enlargement_multiindex_with_none(self): # Test retrieval of the newly added row result = df.loc[("A", None), :] - expected_row = Series([12, 13], index=[0, 1], name=("A", np.nan)) + expected_row = Series([12, 13], index=[0, 1], name=("A", np.nan), dtype=float) tm.assert_series_equal(result, expected_row) def test_setitem_enlargement_multiindex_multiple_none(self): From 94aa1f9df32d63b2cb5b5d3c37eb117e3879e71d Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 11:30:00 +0800 Subject: [PATCH 07/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- pandas/tests/indexing/multiindex/test_setitem.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index e2c8f021319e7..bbf13dd0eef76 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -537,7 +537,8 @@ def test_setitem_enlargement_multiindex_multiple_none(self): ("B", "b1"), (None, "c1"), ("C", None), - ] + ], + dtype="object" ) expected = DataFrame( [[1, 2], [3, 4], [5, 6], [7, 8]], From 1513f66aa491dfcfbd6c8298b485fcbe38e0eb6d Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 13:26:52 +0800 Subject: [PATCH 08/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- .../tests/indexing/multiindex/test_setitem.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index bbf13dd0eef76..62a786d7f6731 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -497,58 +497,55 @@ def test_setitem_enlargement_multiindex_with_none(self): index = MultiIndex.from_tuples( [("A", "a1"), ("A", "a2"), ("B", "b1"), ("B", None)] ) - df = DataFrame([(0, 6), (1, 5), (2, 4), (3, 7)], index=index) + df = DataFrame([(0.0, 6.0), (1.0, 5.0), (2.0, 4.0), (3.0, 7.0)], index=index) # Enlarge with a new index entry where one key is None - df.loc[("A", None), :] = [12, 13] + df.loc[("A", None), :] = [12.0, 13.0] expected_index = MultiIndex.from_tuples( [("A", "a1"), ("A", "a2"), ("B", "b1"), ("B", None), ("A", None)] ) expected = DataFrame( - [[0, 6], [1, 5], [2, 4], [3, 7], [12, 13]], + [[0.0, 6.0], [1.0, 5.0], [2.0, 4.0], [3.0, 7.0], [12.0, 13.0]], index=expected_index, columns=[0, 1], - dtype=float, ) tm.assert_frame_equal(df, expected) # Test retrieval of the newly added row result = df.loc[("A", None), :] - expected_row = Series([12, 13], index=[0, 1], name=("A", np.nan), dtype=float) + expected_row = Series([12.0, 13.0], index=[0, 1], name=("A", np.nan)) tm.assert_series_equal(result, expected_row) def test_setitem_enlargement_multiindex_multiple_none(self): # GH#59153 # Test enlarging with multiple None keys in different levels index = MultiIndex.from_tuples([("A", "a1"), ("B", "b1")]) - df = DataFrame([[1, 2], [3, 4]], index=index, columns=["x", "y"]) + df = DataFrame([[1.0, 2.0], [3.0, 4.0]], index=index, columns=["x", "y"]) # Add row with None in first level - df.loc[(None, "c1"), :] = [5, 6] + df.loc[(None, "c1"), :] = [5.0, 6.0] # Add row with None in second level - df.loc[("C", None), :] = [7, 8] + df.loc[("C", None), :] = [7.0, 8.0] - # (None, None) case removed as requested expected_index = MultiIndex.from_tuples( [ ("A", "a1"), ("B", "b1"), (None, "c1"), ("C", None), - ], - dtype="object" + ] ) expected = DataFrame( - [[1, 2], [3, 4], [5, 6], [7, 8]], + [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]], index=expected_index, columns=["x", "y"], - dtype=float, ) tm.assert_frame_equal(df, expected) + def test_frame_setitem_view_direct(multiindex_dataframe_random_data): # this works because we are modifying the underlying array # really a no-no From 2ecbc5007abbbea3f94abeb7db3602fa8a6f7c97 Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 14:32:30 +0800 Subject: [PATCH 09/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- pandas/tests/indexing/multiindex/test_setitem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index 62a786d7f6731..0cccf6bc09a3e 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -173,11 +173,11 @@ def test_multiindex_setitem2(self): idx = pd.IndexSlice df = df_orig.copy() df.loc[idx[:, :, "Stock"], :] *= 2 - tm.assert_frame_equal(df, expected) + tm.assert_frame_equal(df, expected, check_index_type=False) df = df_orig.copy() df.loc[idx[:, :, "Stock"], "price"] *= 2 - tm.assert_frame_equal(df, expected) + tm.assert_frame_equal(df, expected, check_index_type=False) def test_multiindex_assignment(self): # GH3777 part 2 From 70f7204d45034f418761593ba2627c7eb4acc632 Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 17:45:25 +0800 Subject: [PATCH 10/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- pandas/tests/indexing/multiindex/test_setitem.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index 0cccf6bc09a3e..05939d17c4ebd 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -510,7 +510,8 @@ def test_setitem_enlargement_multiindex_with_none(self): index=expected_index, columns=[0, 1], ) - tm.assert_frame_equal(df, expected) + + tm.assert_frame_equal(df, expected, check_exact=False) # Test retrieval of the newly added row result = df.loc[("A", None), :] @@ -542,7 +543,8 @@ def test_setitem_enlargement_multiindex_multiple_none(self): index=expected_index, columns=["x", "y"], ) - tm.assert_frame_equal(df, expected) + + tm.assert_frame_equal(df, expected, check_exact=False) From 7adc2913a2d3918f155f4ec3fcce22d89d0f5ca5 Mon Sep 17 00:00:00 2001 From: wdyy20041223 <2795352227@qq,com> Date: Fri, 7 Nov 2025 18:42:41 +0800 Subject: [PATCH 11/12] TST: Add regression tests for enlarging MultiIndex with None keys (GH#59153) --- pandas/tests/indexing/multiindex/test_setitem.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index 05939d17c4ebd..c085b5acde94f 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -510,8 +510,8 @@ def test_setitem_enlargement_multiindex_with_none(self): index=expected_index, columns=[0, 1], ) - - tm.assert_frame_equal(df, expected, check_exact=False) + # check_index_type=False because None in MultiIndex causes mixed inferred_type + tm.assert_frame_equal(df, expected, check_index_type=False) # Test retrieval of the newly added row result = df.loc[("A", None), :] @@ -543,8 +543,8 @@ def test_setitem_enlargement_multiindex_multiple_none(self): index=expected_index, columns=["x", "y"], ) - - tm.assert_frame_equal(df, expected, check_exact=False) + # check_index_type=False because None in MultiIndex causes mixed inferred_type + tm.assert_frame_equal(df, expected, check_index_type=False) From bdc1a0a93b776ddaaf2c08f91277e803221274d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Nov 2025 11:59:48 +0000 Subject: [PATCH 12/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pandas/tests/indexing/multiindex/test_setitem.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index c085b5acde94f..4e8af3f7a7759 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -547,7 +547,6 @@ def test_setitem_enlargement_multiindex_multiple_none(self): tm.assert_frame_equal(df, expected, check_index_type=False) - def test_frame_setitem_view_direct(multiindex_dataframe_random_data): # this works because we are modifying the underlying array # really a no-no