Merge pull request #1357 from burnash/1354-get-records-with-blank-header

Many fixes for `get_records`

gspread/worksheet.py

@@ -676,16 +676,48 @@ def get_records(  # noqa: C901 # this comment disables the complexity check for
 
         if last_index is None:
             last_index = self.row_count
+            last_index_set = False
         elif last_index < first_index:
             raise ValueError("last_index must be greater than or equal to first_index")
         elif last_index > self.row_count:
             raise ValueError(
                 "last_index must be an integer less than or equal to the number of rows in the worksheet"
             )
+        else:
+            last_index_set = True
 
-        keys = self.get_values(
+        values = self.get_values(
+            "{first_index}:{last_index}".format(
+                first_index=first_index, last_index=last_index
+            ),
+            value_render_option=value_render_option,
+        )
+        if values == []:
+            # see test_get_records_with_all_values_blank
+            #  if last index is not asked for,
+            #  we don't know the length of the sheet so we return []
+            if last_index_set is False:
+                return []
+            # otherwise values will later be padded to be the size of keys + sheet size
+            values = [[]]
+
+        keys_row = self.get_values(
             "{head}:{head}".format(head=head), value_render_option=value_render_option
-        )[0]
+        )
+        keys = keys_row[0] if len(keys_row) > 0 else []
+
+        values_width = len(values[0])
+        keys_width = len(keys)
+        values_wider_than_keys_by = values_width - keys_width
+
+        # pad keys and values to be the same WIDTH
+        if values_wider_than_keys_by > 0:
+            keys.extend([default_blank] * values_wider_than_keys_by)
+        elif values_wider_than_keys_by < 0:
+            values = fill_gaps(values, cols=keys_width, padding_value=default_blank)
+
+        # pad values to be the HEIGHT of last_index - first_index + 1
+        values = fill_gaps(values, rows=last_index - first_index + 1)
 
         if expected_headers is None:
             # all headers must be unique
@@ -707,29 +739,6 @@ def get_records(  # noqa: C901 # this comment disables the complexity check for
                     )
                 )
 
-        values = self.get_values(
-            "{first_index}:{last_index}".format(
-                first_index=first_index, last_index=last_index
-            ),
-            value_render_option=value_render_option,
-        )
-
-        values_len = len(values[0])
-        keys_len = len(keys)
-        values_wider_than_keys_by = values_len - keys_len
-        default_blank_in_keys = default_blank in keys
-
-        if ((values_wider_than_keys_by > 0) and default_blank_in_keys) or (
-            values_wider_than_keys_by > 1
-        ):
-            raise GSpreadException(
-                "the header row in the worksheet contains multiple empty cells"
-            )
-        elif values_wider_than_keys_by == 1:
-            keys.append(default_blank)
-        elif values_wider_than_keys_by < 0:
-            values = fill_gaps(values, cols=keys_len, padding_value=default_blank)
-
         if numericise_ignore == ["all"]:
             pass
         else:

tests/worksheet_test.py

@@ -805,10 +805,7 @@ def test_get_all_records(self):
             ["", "", "", ""],
             ["A4", 0.4, "", 4],
         ]
-        cell_list = self.sheet.range("A1:D4")
-        for cell, value in zip(cell_list, itertools.chain(*rows)):
-            cell.value = value
-        self.sheet.update_cells(cell_list)
+        self.sheet.update("A1:D4", rows)
 
         # first, read empty strings to empty strings
         read_records = self.sheet.get_all_records()
@@ -846,10 +843,7 @@ def test_get_all_records_different_header(self):
             ["", "", "", ""],
             ["A4", 0.4, "", 4],
         ]
-        cell_list = self.sheet.range("A1:D6")
-        for cell, value in zip(cell_list, itertools.chain(*rows)):
-            cell.value = value
-        self.sheet.update_cells(cell_list)
+        self.sheet.update("A1:D6", rows)
 
         # first, read empty strings to empty strings
         read_records = self.sheet.get_all_records(head=3)
@@ -920,35 +914,17 @@ def test_get_all_records_duplicate_keys(self):
         self.sheet.resize(4, 4)
         # put in new values
         rows = [
-            ["A1", "A1", "", "D1"],
+            ["A1", "faff", "C3", "faff"],
             [1, "b2", 1.45, ""],
             ["", "", "", ""],
             ["A4", 0.4, "", 4],
         ]
-        cell_list = self.sheet.range("A1:D4")
-        for cell, value in zip(cell_list, itertools.chain(*rows)):
-            cell.value = value
-        self.sheet.update_cells(cell_list)
+        self.sheet.update("A1:D4", rows)
 
+        # check no expected headers
         with pytest.raises(GSpreadException):
             self.sheet.get_all_records()
 
-    @pytest.mark.vcr(allow_playback_repeats=True)
-    def test_get_all_records_expected_headers(self):
-        self.sheet.resize(4, 4)
-
-        # put in new values
-        rows = [
-            ["A1", "faff", "C3", "faff"],
-            [1, "b2", 1.45, ""],
-            ["", "", "", ""],
-            ["A4", 0.4, "", 4],
-        ]
-        cell_list = self.sheet.range("A1:D4")
-        for cell, value in zip(cell_list, itertools.chain(*rows)):
-            cell.value = value
-        self.sheet.update_cells(cell_list)
-
         # check non uniques expected headers
         expected_headers = ["A1", "A1"]
         with pytest.raises(GSpreadException):
@@ -972,6 +948,109 @@ def test_get_all_records_expected_headers(self):
         self.assertDictEqual(expected_values_2, read_records[1])
         self.assertDictEqual(expected_values_3, read_records[2])
 
+    @pytest.mark.vcr()
+    def test_get_all_records_with_blank_final_headers(self):
+        # regression test for #590, #629, #1354
+        self.sheet.resize(4, 4)
+
+        # put in new values
+        rows = [
+            ["A1", "faff", "", ""],
+            [1, "b2", 1.45, ""],
+            ["", "", "", ""],
+            ["A4", 0.4, "", 4],
+        ]
+        self.sheet.update("A1:D4", rows)
+
+        with pytest.raises(GSpreadException):
+            self.sheet.get_all_records()
+
+        expected_headers = []
+        read_records = self.sheet.get_all_records(
+            expected_headers=expected_headers,
+        )
+
+        expected_values_1 = dict(zip(rows[0], rows[1]))
+        expected_values_2 = dict(zip(rows[0], rows[2]))
+        expected_values_3 = dict(zip(rows[0], rows[3]))
+        self.assertDictEqual(expected_values_1, read_records[0])
+        self.assertDictEqual(expected_values_2, read_records[1])
+        self.assertDictEqual(expected_values_3, read_records[2])
+
+    @pytest.mark.vcr()
+    def test_get_all_records_with_keys_blank(self):
+        # regression test for #1355
+        self.sheet.resize(4, 4)
+
+        rows = [
+            ["", "", "", ""],
+            ["c", "d", "e", "f"],
+            ["g", "h", "i", "j"],
+            ["k", "l", "m", ""],
+        ]
+        cell_list = self.sheet.range("A1:D4")
+        for cell, value in zip(cell_list, itertools.chain(*rows)):
+            cell.value = value
+        self.sheet.update_cells(cell_list)
+
+        # duplicate headers
+        with pytest.raises(GSpreadException):
+            self.sheet.get_all_records()
+
+        # ignore duplicate headers
+        read_records = self.sheet.get_all_records(expected_headers=[])
+
+        expected_values_1 = dict(zip(rows[0], rows[1]))
+        expected_values_2 = dict(zip(rows[0], rows[2]))
+        expected_values_3 = dict(zip(rows[0], rows[3]))
+        self.assertDictEqual(expected_values_1, read_records[0])
+        self.assertDictEqual(expected_values_2, read_records[1])
+        self.assertDictEqual(expected_values_3, read_records[2])
+
+    @pytest.mark.vcr()
+    def test_get_records_with_all_values_blank(self):
+        # regression test for #1355
+        self.sheet.resize(4, 4)
+
+        rows = [
+            ["a", "b", "c", "d"],
+            ["", "", "", ""],
+            ["", "", "", ""],
+            ["", "", "", ""],
+        ]
+        self.sheet.update("A1:D4", rows)
+
+        expected_values_1 = dict(zip(rows[0], rows[1]))
+        expected_values_2 = dict(zip(rows[0], rows[2]))
+        expected_values_3 = dict(zip(rows[0], rows[3]))
+
+        # I ask for get_records(first_index=2, last_index=4)
+        # I want [{...}, {...}, {...}]
+
+        read_records_first_last = self.sheet.get_records(first_index=2, last_index=4)
+        self.assertEqual(len(read_records_first_last), 3)
+        self.assertDictEqual(expected_values_1, read_records_first_last[0])
+        self.assertDictEqual(expected_values_2, read_records_first_last[1])
+        self.assertDictEqual(expected_values_3, read_records_first_last[2])
+
+        # I ask for get_records()
+        # I want []
+        read_records_nofirst_nolast = self.sheet.get_records()
+        self.assertEqual(len(read_records_nofirst_nolast), 0)
+
+        # I ask for get_records(first_index=1)
+        # I want []
+        read_records_first_nolast = self.sheet.get_records(first_index=2)
+        self.assertEqual(len(read_records_first_nolast), 0)
+
+        # I ask for get_records(last_index=4)
+        # I want [{...}, {...}, {...}]
+        read_records_nofirst_last = self.sheet.get_records(last_index=4)
+        self.assertEqual(len(read_records_nofirst_last), 3)
+        self.assertDictEqual(expected_values_1, read_records_nofirst_last[0])
+        self.assertDictEqual(expected_values_2, read_records_nofirst_last[1])
+        self.assertDictEqual(expected_values_3, read_records_nofirst_last[2])
+
     @pytest.mark.vcr()
     def test_get_all_records_numericise_unformatted(self):
         self.sheet.resize(2, 4)
@@ -1006,10 +1085,7 @@ def test_get_records(self):
             [7, 8, 9],
             [10, 11, 12],
         ]
-        cell_list = self.sheet.range("A1:C5")
-        for cell, value in zip(cell_list, itertools.chain(*rows)):
-            cell.value = value
-        self.sheet.update_cells(cell_list)
+        self.sheet.update("A1:C5", rows)
 
         # test1 - set last_index only
         read_records = self.sheet.get_records(last_index=3)
@@ -1056,14 +1132,7 @@ def test_get_records_pad_one_key(self):
             ["A1", "B1", "C1"],
             [1, 2, 3, 4],
         ]
-        cell_list = self.sheet.range("A1:C1")
-        for cell in cell_list:
-            cell.value = rows[0][cell.col - 1]
-        self.sheet.update_cells(cell_list)
-        cell_list = self.sheet.range("A2:D2")
-        for cell in cell_list:
-            cell.value = rows[1][cell.col - 1]
-        self.sheet.update_cells(cell_list)
+        self.sheet.update("A1:D2", rows)
 
         read_records = self.sheet.get_records(head=1, first_index=2, last_index=2)
         rows[0].append("")
@@ -1076,19 +1145,9 @@ def test_get_records_pad_values(self):
         self.sheet.resize(2, 4)
         rows = [
             ["A1", "B1", "C1"],
-            [
-                1,
-                2,
-            ],
+            [1, 2],
         ]
-        cell_list = self.sheet.range("A1:C1")
-        for cell in cell_list:
-            cell.value = rows[0][cell.col - 1]
-        self.sheet.update_cells(cell_list)
-        cell_list = self.sheet.range("A2:B2")
-        for cell in cell_list:
-            cell.value = rows[1][cell.col - 1]
-        self.sheet.update_cells(cell_list)
+        self.sheet.update("A1:C2", rows)
 
         read_records = self.sheet.get_records(head=1, first_index=2, last_index=2)
         rows[1].append("")
@@ -1100,20 +1159,10 @@ def test_get_records_pad_values(self):
     def test_get_records_pad_more_than_one_key(self):
         self.sheet.resize(2, 4)
         rows = [
-            [
-                "A1",
-                "B1",
-            ],
+            ["A1", "B1"],
             [1, 2, 3, 4],
         ]
-        cell_list = self.sheet.range("A1:B1")
-        for cell in cell_list:
-            cell.value = rows[0][cell.col - 1]
-        self.sheet.update_cells(cell_list)
-        cell_list = self.sheet.range("A2:D2")
-        for cell in cell_list:
-            cell.value = rows[1][cell.col - 1]
-        self.sheet.update_cells(cell_list)
+        self.sheet.update("A1:D2", rows)
 
         with pytest.raises(GSpreadException):
             self.sheet.get_records(head=1, first_index=2, last_index=2)