|
15 | 15 | # specific language governing permissions and limitations |
16 | 16 | # under the License. |
17 | 17 | """ |
18 | | -Streaming, constant-memory XLSX writer for multi-sheet dashboard exports. |
19 | | -
|
20 | | -Unlike :mod:`superset.utils.excel`, which materializes a whole DataFrame in |
21 | | -memory, this writer streams rows one at a time into an ``xlsxwriter`` workbook |
22 | | -opened in ``constant_memory`` mode, so a dashboard with many large charts never |
23 | | -holds more than one row per sheet in memory at once. |
| 18 | +Streaming XLSX writer for multi-sheet dashboard exports. |
| 19 | +
|
| 20 | +Unlike :mod:`superset.utils.excel`, which builds an in-memory DataFrame per |
| 21 | +sheet and hands the whole thing to ``xlsxwriter`` at once, this writer opens the |
| 22 | +workbook in ``constant_memory`` mode and writes rows one at a time, so |
| 23 | +``xlsxwriter`` keeps at most one row per sheet buffered on the writer side. The |
| 24 | +source records may still be materialized upstream (e.g. by the chart query |
| 25 | +response); this bounds only the writer's own footprint, not the caller's. |
24 | 26 | """ |
25 | 27 |
|
26 | 28 | from __future__ import annotations |
|
54 | 56 | _MAX_EXCEL_INT = 10**15 |
55 | 57 |
|
56 | 58 |
|
| 59 | +def _quote_if_formula(text: str) -> str: |
| 60 | + """ |
| 61 | + Prefix formula-like text with an apostrophe so spreadsheet apps treat it as |
| 62 | + literal text (defense against formula injection). |
| 63 | +
|
| 64 | + Leading whitespace is ignored when detecting a formula, because spreadsheet |
| 65 | + apps still evaluate a cell whose formula prefix is preceded by spaces or |
| 66 | + tabs (e.g. ``" =cmd"`` or ``"\\t=cmd"``). |
| 67 | + """ |
| 68 | + stripped = text.lstrip() |
| 69 | + return f"'{text}" if stripped and stripped[0] in _FORMULA_PREFIXES else text |
| 70 | + |
| 71 | + |
| 72 | +def _coerce_float_cell(value: Any) -> Any: |
| 73 | + """ |
| 74 | + Convert a ``Decimal``/real value to something ``xlsxwriter`` accepts. |
| 75 | +
|
| 76 | + ``float()`` on a non-finite ``Decimal`` ("NaN"/"Infinity") yields a value |
| 77 | + xlsxwriter rejects, and an over-large value can raise ``OverflowError``; |
| 78 | + blank the former and stringify the latter, and stringify magnitudes Excel |
| 79 | + cannot represent precisely. |
| 80 | + """ |
| 81 | + try: |
| 82 | + number = float(value) |
| 83 | + except (OverflowError, ValueError): |
| 84 | + return str(value) |
| 85 | + if not math.isfinite(number): |
| 86 | + return "" |
| 87 | + return str(number) if abs(number) > _MAX_EXCEL_INT else number |
| 88 | + |
| 89 | + |
57 | 90 | def sanitize_sheet_name(raw: str, used: set[str]) -> str: |
58 | 91 | """ |
59 | 92 | Produce a valid, unique Excel sheet name from ``raw``. |
@@ -104,24 +137,19 @@ def _sanitize_cell(value: Any) -> Any: |
104 | 137 | if isinstance(value, bool): |
105 | 138 | return value |
106 | 139 | if isinstance(value, str): |
107 | | - return f"'{value}" if value and value[0] in _FORMULA_PREFIXES else value |
| 140 | + return _quote_if_formula(value) |
108 | 141 | if isinstance(value, (datetime, date)): |
109 | 142 | return value.isoformat() |
110 | 143 | if isinstance(value, Decimal): |
111 | | - number = float(value) |
112 | | - return str(number) if abs(number) > _MAX_EXCEL_INT else number |
| 144 | + return _coerce_float_cell(value) |
113 | 145 | if isinstance(value, numbers.Integral): |
114 | 146 | number = int(value) |
115 | 147 | return str(number) if abs(number) > _MAX_EXCEL_INT else number |
116 | 148 | if isinstance(value, numbers.Real): |
117 | | - number = float(value) |
118 | | - if not math.isfinite(number): |
119 | | - return "" |
120 | | - return str(number) if abs(number) > _MAX_EXCEL_INT else number |
| 149 | + return _coerce_float_cell(value) |
121 | 150 | # Anything else (lists, dicts, custom objects) is stringified, still guarding |
122 | 151 | # against formula injection on the resulting text. |
123 | | - text = str(value) |
124 | | - return f"'{text}" if text and text[0] in _FORMULA_PREFIXES else text |
| 152 | + return _quote_if_formula(str(value)) |
125 | 153 |
|
126 | 154 |
|
127 | 155 | class StreamingXlsxWriter: |
@@ -153,20 +181,34 @@ def add_sheet( |
153 | 181 | :param name: Desired sheet name (sanitized/de-duplicated automatically) |
154 | 182 | :param columns: Column headers |
155 | 183 | :param rows: Iterable of row sequences, streamed one at a time |
156 | | - :returns: The number of data rows actually written (capped at Excel's |
157 | | - per-sheet limit) |
| 184 | + :returns: The number of data rows actually written (capped just below |
| 185 | + Excel's per-sheet limit; when the data is larger a final notice row |
| 186 | + is appended and the dropped rows are not counted) |
158 | 187 | """ |
159 | 188 | sheet_name = sanitize_sheet_name(name, self._used_sheet_names) |
160 | 189 | worksheet = self._workbook.add_worksheet(sheet_name) |
161 | 190 | worksheet.write_row(0, 0, [_sanitize_cell(col) for col in columns]) |
162 | 191 |
|
| 192 | + # Reserve the final row for a truncation notice, so when the data |
| 193 | + # exceeds the sheet's capacity the user can see rows were dropped |
| 194 | + # instead of silently losing them. |
| 195 | + row_cap = MAX_DATA_ROWS_PER_SHEET - 1 |
163 | 196 | written = 0 |
| 197 | + truncated = False |
164 | 198 | for row in rows: |
165 | | - if written >= MAX_DATA_ROWS_PER_SHEET: |
| 199 | + if written >= row_cap: |
| 200 | + truncated = True |
166 | 201 | break |
167 | 202 | worksheet.write_row(written + 1, 0, [_sanitize_cell(cell) for cell in row]) |
168 | 203 | written += 1 |
169 | 204 |
|
| 205 | + if truncated: |
| 206 | + worksheet.write_string( |
| 207 | + written + 1, |
| 208 | + 0, |
| 209 | + f"[Truncated: only first {written:,} rows exported]", |
| 210 | + ) |
| 211 | + |
170 | 212 | self.sheet_count += 1 |
171 | 213 | return written |
172 | 214 |
|
|
0 commit comments