From 7de47a657c9f53e8796af28e1b2f839a94c020e3 Mon Sep 17 00:00:00 2001 From: Matija Obreza Date: Wed, 23 Jan 2019 02:29:26 +0100 Subject: [PATCH 1/8] Super-streaming SXSSF --- .../poi/xssf/streaming/IRowGenerator.java | 32 ++ .../xssf/streaming/SXSSFFormulaEvaluator.java | 12 +- .../poi/xssf/streaming/SXSSFPicture.java | 4 +- .../apache/poi/xssf/streaming/SXSSFSheet.java | 13 +- .../poi/xssf/streaming/SXSSFWorkbook.java | 10 +- .../poi/xssf/streaming/SheetDataWriter.java | 7 +- .../xssf/streaming/StreamingSheetWriter.java | 82 ++++ .../poi/xssf/streaming/SuperSXSSFSheet.java | 63 ++++ .../xssf/streaming/SuperSXSSFWorkbook.java | 349 ++++++++++++++++++ 9 files changed, 557 insertions(+), 15 deletions(-) create mode 100644 src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java create mode 100644 src/ooxml/java/org/apache/poi/xssf/streaming/StreamingSheetWriter.java create mode 100644 src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java create mode 100644 src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java b/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java new file mode 100644 index 00000000000..4435afdaca2 --- /dev/null +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java @@ -0,0 +1,32 @@ +/* ==================================================================== + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +==================================================================== */ + +package org.apache.poi.xssf.streaming; + +/** + * IRowGenerator for Super-streaming XSSF sheets + */ +public interface IRowGenerator { + + /** + * Generate and add rows to the sheet + * + * @param sheet the sheet + * @throws Exception the exception + */ + void generateRows(SXSSFSheet sheet) throws Exception; +} diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFFormulaEvaluator.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFFormulaEvaluator.java index 088694054bb..69f704ef788 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFFormulaEvaluator.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFFormulaEvaluator.java @@ -106,11 +106,13 @@ public static void evaluateAllFormulaCells(SXSSFWorkbook wb, boolean skipOutOfWi // Process the sheets as best we can for (Sheet sheet : wb) { - // Check if any rows have already been flushed out - int lastFlushedRowNum = ((SXSSFSheet) sheet).getLastFlushedRowNum(); - if (lastFlushedRowNum > -1) { - if (! skipOutOfWindow) throw new RowFlushedException(0); - logger.log(POILogger.INFO, "Rows up to " + lastFlushedRowNum + " have already been flushed, skipping"); + if (sheet instanceof SXSSFSheet) { + // Check if any rows have already been flushed out + int lastFlushedRowNum = ((SXSSFSheet) sheet).getLastFlushedRowNum(); + if (lastFlushedRowNum > -1) { + if (! skipOutOfWindow) throw new RowFlushedException(0); + logger.log(POILogger.INFO, "Rows up to " + lastFlushedRowNum + " have already been flushed, skipping"); + } } // Evaluate what we have diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFPicture.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFPicture.java index 1d83b7c7d63..23a43dc3425 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFPicture.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFPicture.java @@ -24,6 +24,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more import org.apache.poi.ss.usermodel.Picture; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Shape; +import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.util.ImageUtils; import org.apache.poi.util.Internal; @@ -201,7 +202,8 @@ private float getRowHeightInPixels(int rowIndex) { // THE FOLLOWING THREE LINES ARE THE MAIN CHANGE compared to the non-streaming version: use the SXSSF sheet, // not the XSSF sheet (which never contais rows when using SXSSF) XSSFSheet xssfSheet = getSheet(); - SXSSFSheet sheet = _wb.getSXSSFSheet(xssfSheet); + SXSSFSheet sxSheet = _wb.getSXSSFSheet(xssfSheet); + Sheet sheet = sxSheet == null ? xssfSheet : sxSheet; Row row = sheet.getRow(rowIndex); float height = row != null ? row.getHeightInPoints() : sheet.getDefaultRowHeightInPoints(); return height * Units.PIXEL_DPI / Units.POINT_DPI; diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java index f639dc12e7b..6d373519e18 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java @@ -65,15 +65,22 @@ Licensed to the Apache Software Foundation (ASF) under one or more public class SXSSFSheet implements Sheet { /*package*/ final XSSFSheet _sh; - private final SXSSFWorkbook _workbook; + protected final SXSSFWorkbook _workbook; private final TreeMap _rows = new TreeMap<>(); - private final SheetDataWriter _writer; + protected SheetDataWriter _writer; private int _randomAccessWindowSize = SXSSFWorkbook.DEFAULT_WINDOW_SIZE; - private final AutoSizeColumnTracker _autoSizeColumnTracker; + protected final AutoSizeColumnTracker _autoSizeColumnTracker; private int outlineLevelRow; private int lastFlushedRowNumber = -1; private boolean allFlushed; + protected SXSSFSheet(SXSSFWorkbook workbook, XSSFSheet xSheet, int randomAccessWindowSize) { + _workbook = workbook; + _sh = xSheet; + setRandomAccessWindowSize(randomAccessWindowSize); + _autoSizeColumnTracker = new AutoSizeColumnTracker(this); + } + public SXSSFSheet(SXSSFWorkbook workbook, XSSFSheet xSheet) throws IOException { _workbook = workbook; _sh = xSheet; diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java index 2074a70b4ac..2ddc8b3091d 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java @@ -96,7 +96,7 @@ public class SXSSFWorkbook implements Workbook { public static final int DEFAULT_WINDOW_SIZE = 100; private static final POILogger logger = POILogFactory.getLogger(SXSSFWorkbook.class); - private final XSSFWorkbook _wb; + protected final XSSFWorkbook _wb; private final Map _sxFromXHash = new HashMap<>(); private final Map _xFromSxHash = new HashMap<>(); @@ -111,12 +111,12 @@ public class SXSSFWorkbook implements Workbook { /** * shared string table - a cache of strings in this workbook */ - private final SharedStringsTable _sharedStringSource; + protected final SharedStringsTable _sharedStringSource; /** * controls whether Zip64 mode is used - Always became the default in POI 5.0.0 */ - private Zip64Mode zip64Mode = Zip64Mode.Always; + protected Zip64Mode zip64Mode = Zip64Mode.Always; /** * Construct a new workbook with default row window size @@ -285,7 +285,7 @@ public int getRandomAccessWindowSize() { return _randomAccessWindowSize; } - private void setRandomAccessWindowSize(int rowAccessWindowSize) { + protected void setRandomAccessWindowSize(int rowAccessWindowSize) { if(rowAccessWindowSize == 0 || rowAccessWindowSize < -1) { throw new IllegalArgumentException("rowAccessWindowSize must be greater than 0 or -1"); } @@ -377,7 +377,7 @@ void deregisterSheetMapping(XSSFSheet xSheet) _xFromSxHash.remove(xSheet); } - private XSSFSheet getSheetFromZipEntryName(String sheetRef) + protected XSSFSheet getSheetFromZipEntryName(String sheetRef) { for(XSSFSheet sheet : _sxFromXHash.values()) { diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SheetDataWriter.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SheetDataWriter.java index cbac3b8822d..8cf5cbc0e2b 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SheetDataWriter.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SheetDataWriter.java @@ -55,7 +55,7 @@ public class SheetDataWriter implements Closeable { private static final POILogger logger = POILogFactory.getLogger(SheetDataWriter.class); private final File _fd; - private final Writer _out; + protected final Writer _out; private int _rownum; private int _numberOfFlushedRows; private int _lowestIndexOfFlushedRows; // meaningful only of _numberOfFlushedRows>0 @@ -72,6 +72,11 @@ public SheetDataWriter() throws IOException { _fd = createTempFile(); _out = createWriter(_fd); } + + public SheetDataWriter(Writer writer) throws IOException { + _fd = null; + _out = writer; + } public SheetDataWriter(SharedStringsTable sharedStringsTable) throws IOException { this(); diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/StreamingSheetWriter.java b/src/ooxml/java/org/apache/poi/xssf/streaming/StreamingSheetWriter.java new file mode 100644 index 00000000000..2dce1fed128 --- /dev/null +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/StreamingSheetWriter.java @@ -0,0 +1,82 @@ +/* ==================================================================== + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +==================================================================== */ + +package org.apache.poi.xssf.streaming; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; + +import org.apache.poi.util.POILogFactory; +import org.apache.poi.util.POILogger; + +/** + * Unlike SheetDataWriter, this writer does not create a temporary file, it writes data directly + * to the provided OutputStream. + * @since 4.1.0 + */ +public class StreamingSheetWriter extends SheetDataWriter { + private static final POILogger logger = POILogFactory.getLogger(StreamingSheetWriter.class); + + public StreamingSheetWriter() throws IOException { + throw new RuntimeException("StreamingSheetWriter requires OutputStream"); + } + + public StreamingSheetWriter(OutputStream out) throws IOException { + super(createWriter(out)); + logger.log(POILogger.DEBUG, "Preparing SSXSSF sheet writer"); + } + + @Override + public File createTempFile() throws IOException { + throw new RuntimeException("Not supported with StreamingSheetWriter"); + } + + @Override + public Writer createWriter(File fd) throws IOException { + throw new RuntimeException("Not supported with StreamingSheetWriter"); + } + + /** + * Create a writer for the sheet data. + * + * @param out the output stream to write to + */ + protected static Writer createWriter(OutputStream out) throws IOException { + return new BufferedWriter(new OutputStreamWriter(out, "UTF-8")); + } + + @Override + public void close() throws IOException { + _out.flush(); + } + + @Override + public InputStream getWorksheetXMLInputStream() throws IOException { + throw new RuntimeException("Not supported with StreamingSheetWriter"); + } + + @Override + boolean dispose() throws IOException { + _out.close(); + return true; + } +} diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java new file mode 100644 index 00000000000..f5441a690b3 --- /dev/null +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java @@ -0,0 +1,63 @@ +/* ==================================================================== + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +==================================================================== */ + +package org.apache.poi.xssf.streaming; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.poi.xssf.usermodel.XSSFSheet; + +/** + * An update of SXSSFSheet that uses IRowGenerator to create rows. + * @since 4.1.0 + */ +public class SuperSXSSFSheet extends SXSSFSheet { + private IRowGenerator rowGenerator; + + public SuperSXSSFSheet(SuperSXSSFWorkbook workbook, XSSFSheet xSheet) throws IOException { + super(workbook, xSheet, workbook.getRandomAccessWindowSize()); + } + + @Override + public InputStream getWorksheetXMLInputStream() throws IOException { + throw new RuntimeException("Not supported by SuperSXSSFSheet"); + } + + public void setRowGenerator(IRowGenerator rowGenerator) { + this.rowGenerator = rowGenerator; + } + + public void writeRows(OutputStream out) throws IOException { + // delayed creation of SheetDataWriter + _writer = ((SuperSXSSFWorkbook) _workbook).createSheetDataWriter(out); + try { + if (this.rowGenerator != null) { + this.rowGenerator.generateRows(this); + } + } catch (Exception e) { + throw new IOException("Error generating Excel rows", e); + } finally { + // flush buffered rows + flushRows(0); + // flush writer buffer + _writer.close(); + out.flush(); + } + } +} diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java new file mode 100644 index 00000000000..e37c7dc01e8 --- /dev/null +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java @@ -0,0 +1,349 @@ +/* ==================================================================== + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +==================================================================== */ + +package org.apache.poi.xssf.streaming; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.poi.openxml4j.util.ZipArchiveThresholdInputStream; +import org.apache.poi.openxml4j.util.ZipEntrySource; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.util.IOUtils; +import org.apache.poi.util.POILogFactory; +import org.apache.poi.util.POILogger; +import org.apache.poi.xssf.usermodel.XSSFChartSheet; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +/** + * An update of SXSSFWorkbook avoids generating a temporary file and writes data directly to + * the provided OutputStream. + * @since 4.1.0 + */ +public class SuperSXSSFWorkbook extends SXSSFWorkbook { + private static final POILogger logger = POILogFactory.getLogger(SuperSXSSFWorkbook.class); + + public SuperSXSSFWorkbook() { + this(null); + } + + public SuperSXSSFWorkbook(XSSFWorkbook workbook) { + this(workbook, SXSSFWorkbook.DEFAULT_WINDOW_SIZE); + } + + public SuperSXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize) { + setRandomAccessWindowSize(rowAccessWindowSize); + _sharedStringSource = null; + if (workbook == null) { + _wb = new XSSFWorkbook(); + } else { + _wb = workbook; + } + } + + @Override + public void setCompressTempFiles(boolean compress) { + // NOOP + } + + @Override + protected SheetDataWriter createSheetDataWriter() throws IOException { + throw new RuntimeException("Not supported by SuperSXSSFWorkbook"); + } + + protected StreamingSheetWriter createSheetDataWriter(OutputStream out) throws IOException { + return new StreamingSheetWriter(out); + } + + @Override + protected void injectData(ZipEntrySource zipEntrySource, OutputStream out) throws IOException { + ZipArchiveOutputStream zos = new ZipArchiveOutputStream(out); + zos.setUseZip64(zip64Mode); + try { + Enumeration en = zipEntrySource.getEntries(); + while (en.hasMoreElements()) { + ZipArchiveEntry ze = en.nextElement(); + ZipArchiveEntry zeOut = new ZipArchiveEntry(ze.getName()); + zeOut.setSize(ze.getSize()); + zeOut.setTime(ze.getTime()); + zos.putArchiveEntry(zeOut); + try (final InputStream is = zipEntrySource.getInputStream(ze)) { + if (is instanceof ZipArchiveThresholdInputStream) { + // #59743 - disable Threshold handling for SXSSF copy + // as users tend to put too much repetitive data in when using SXSSF :) + ((ZipArchiveThresholdInputStream) is).setGuardState(false); + } + XSSFSheet xSheet = getSheetFromZipEntryName(ze.getName()); + // See bug 56557, we should not inject data into the special ChartSheets + if (xSheet != null && !(xSheet instanceof XSSFChartSheet)) { + SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xSheet); + if (sxSheet != null) { + copyStreamAndInjectWorksheet(is, zos, sxSheet); + } else { + IOUtils.copy(is, zos); + } + } else { + IOUtils.copy(is, zos); + } + } finally { + zos.closeArchiveEntry(); + } + } + } finally { + zos.finish(); + zipEntrySource.close(); + } + } + + private static void copyStreamAndInjectWorksheet(InputStream in, OutputStream out, SuperSXSSFSheet sxSheet) + throws IOException { + InputStreamReader inReader = new InputStreamReader(in, "UTF-8"); + OutputStreamWriter outWriter = new OutputStreamWriter(out, "UTF-8"); + boolean needsStartTag = true; + int c; + int pos = 0; + String s = "" or "" + // (excluding). + while (((c = inReader.read()) != -1)) { + if (c == s.charAt(pos)) { + pos++; + if (pos == n) { + if ("') { + // Found + outWriter.write(s); + outWriter.write(c); + s = ""; + n = s.length(); + pos = 0; + needsStartTag = false; + continue; + } + if (c == '/') { + // Found ') { + // Found + break; + } + + outWriter.write(s); + outWriter.write('/'); + outWriter.write(c); + pos = 0; + continue; + } + + outWriter.write(s); + outWriter.write('/'); + outWriter.write(c); + pos = 0; + continue; + } else { + // Found + break; + } + } + } else { + if (pos > 0) { + outWriter.write(s, 0, pos); + } + if (c == s.charAt(0)) { + pos = 1; + } else { + outWriter.write(c); + pos = 0; + } + } + } + outWriter.flush(); + if (needsStartTag) { + outWriter.write("\n"); + outWriter.flush(); + } + + // Invoke row generation in SXSSFSheet + sxSheet.writeRows(out); + + outWriter.write(""); + outWriter.flush(); + // Copy the rest of "in" to "out". + while (((c = inReader.read()) != -1)) { + outWriter.write(c); + } + outWriter.flush(); + } + + @Override + SXSSFSheet createAndRegisterSXSSFSheet(XSSFSheet xSheet) { + final SuperSXSSFSheet sxSheet; + try { + sxSheet = new SuperSXSSFSheet(this, xSheet); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + registerSheetMapping(sxSheet, xSheet); + return sxSheet; + } + + /** + * Returns an iterator of the sheets in the workbook in sheet order. Includes hidden and very hidden sheets. + * + * @return an iterator of the sheets. + */ + @Override + public Iterator sheetIterator() { + return new SheetIterator(); + } + + private final class SheetIterator implements Iterator { + final private Iterator it; + + @SuppressWarnings("unchecked") + public SheetIterator() { + it = (Iterator) (Iterator) _wb.iterator(); + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + @SuppressWarnings("unchecked") + public T next() throws NoSuchElementException { + final XSSFSheet xssfSheet = it.next(); + SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xssfSheet); + return (T) (sxSheet == null ? xssfSheet : sxSheet); + } + + /** + * Unexpected behavior may occur if sheets are reordered after iterator has been created. Support for the remove + * method may be added in the future if someone can figure out a reliable implementation. + */ + @Override + public void remove() throws IllegalStateException { + throw new UnsupportedOperationException("remove method not supported on XSSFWorkbook.iterator(). " + + "Use Sheet.removeSheetAt(int) instead."); + } + } + + /** + * Alias for {@link #sheetIterator()} to allow foreach loops + */ + @Override + public Iterator iterator() { + return sheetIterator(); + } + + @Override + public SXSSFSheet getSheetAt(int index) { + throw new RuntimeException("Not supported by SuperSXSSFWorkbook"); + } + + public XSSFSheet getXSSFSheetAt(int index) { + return _wb.getSheetAt(index); + } + + /** + * Gets the sheet at the given index for streaming. + * + * @param index the index + * @return the streaming sheet at + */ + public SXSSFSheet getStreamingSheetAt(int index) { + XSSFSheet xSheet = _wb.getSheetAt(index); + SXSSFSheet sxSheet = getSXSSFSheet(xSheet); + if (sxSheet == null) { + return createAndRegisterSXSSFSheet(xSheet); + } else { + return sxSheet; + } + } + + @Override + public SXSSFSheet getSheet(String name) { + throw new RuntimeException("Not supported by SuperSXSSFWorkbook"); + } + + public XSSFSheet getXSSFSheet(String name) { + return _wb.getSheet(name); + } + + /** + * Gets sheet with the given name for streaming. + * + * @param name the name + * @return the streaming sheet + */ + public SuperSXSSFSheet getStreamingSheet(String name) { + XSSFSheet xSheet = _wb.getSheet(name); + SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xSheet); + if (sxSheet == null) { + return (SuperSXSSFSheet) createAndRegisterSXSSFSheet(xSheet); + } else { + return sxSheet; + } + } + + /** + * Removes sheet at the given index + * + * @param index of the sheet to remove (0-based) + */ + @Override + public void removeSheetAt(int index) { + // Get the sheet to be removed + XSSFSheet xSheet = _wb.getSheetAt(index); + SXSSFSheet sxSheet = getSXSSFSheet(xSheet); + + // De-register it + _wb.removeSheetAt(index); + + // The sheet may not be a streaming sheet and is not mapped + if (sxSheet != null) { + deregisterSheetMapping(xSheet); + + // Clean up temporary resources + try { + sxSheet.dispose(); + } catch (IOException e) { + logger.log(POILogger.WARN, e); + } + } + } +} From 9e603b70cc966fca02442b5a544a98fec46bb09a Mon Sep 17 00:00:00 2001 From: Matija Obreza Date: Wed, 23 Jan 2019 16:41:20 +0100 Subject: [PATCH 2/8] Copy data to output stream using `ISheetInjector` - SXSSF and SuperSXSSF use different approaches - Reduces code duplication in SuperSXSSFWorkbook - Added unit tests --- .../apache/poi/xssf/streaming/SXSSFSheet.java | 1 + .../poi/xssf/streaming/SXSSFWorkbook.java | 25 +- .../xssf/streaming/SuperSXSSFWorkbook.java | 150 +----- .../streaming/TestSuperSXSSFWorkbook.java | 500 ++++++++++++++++++ 4 files changed, 539 insertions(+), 137 deletions(-) create mode 100644 src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java index 6d373519e18..f819014d279 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java @@ -1911,6 +1911,7 @@ boolean dispose() throws IOException { if (!allFlushed) { flushRows(); } + if (_writer == null) return true; // SuperSXSSFSheet initializes the writer only on write return _writer.dispose(); } diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java index 2ddc8b3091d..4e3b0bc7b5e 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java @@ -103,6 +103,10 @@ public class SXSSFWorkbook implements Workbook { private int _randomAccessWindowSize = DEFAULT_WINDOW_SIZE; + protected static interface ISheetInjector { + void writeSheetData(OutputStream out) throws IOException; + } + /** * whether temp files should be compressed. */ @@ -409,7 +413,8 @@ protected void injectData(ZipEntrySource zipEntrySource, OutputStream out) throw if (xSheet != null && !(xSheet instanceof XSSFChartSheet)) { SXSSFSheet sxSheet = getSXSSFSheet(xSheet); try (InputStream xis = sxSheet.getWorksheetXMLInputStream()) { - copyStreamAndInjectWorksheet(is, zos, xis); + // copyStreamAndInjectWorksheet(is, zos, xis); + copyStreamAndInjectWorksheet(is, zos, createSheetInjector(sxSheet)); } } else { IOUtils.copy(is, zos); @@ -434,7 +439,17 @@ protected ZipArchiveOutputStream createArchiveOutputStream(OutputStream out) { } } - private static void copyStreamAndInjectWorksheet(InputStream in, OutputStream out, InputStream worksheetData) throws IOException { + protected ISheetInjector createSheetInjector(SXSSFSheet sxSheet) throws IOException { + return (output) -> { + try (InputStream xis = sxSheet.getWorksheetXMLInputStream()) { + // Copy the worksheet data to "output". + IOUtils.copy(xis, output); + } + }; + } + + // private static void copyStreamAndInjectWorksheet(InputStream in, OutputStream out, InputStream worksheetData) throws IOException { + private static void copyStreamAndInjectWorksheet(InputStream in, OutputStream out, ISheetInjector sheetInjector) throws IOException { InputStreamReader inReader = new InputStreamReader(in, StandardCharsets.UTF_8); OutputStreamWriter outWriter = new OutputStreamWriter(out, StandardCharsets.UTF_8); boolean needsStartTag = true; @@ -526,8 +541,7 @@ private static void copyStreamAndInjectWorksheet(InputStream in, OutputStream ou outWriter.write("\n"); outWriter.flush(); } - //Copy the worksheet data to "out". - IOUtils.copy(worksheetData,out); + sheetInjector.writeSheetData(out); outWriter.write(""); outWriter.flush(); //Copy the rest of "in" to "out". @@ -918,7 +932,8 @@ public void close() throws IOException { for (SXSSFSheet sheet : _xFromSxHash.values()) { try { - sheet.getSheetDataWriter().close(); + SheetDataWriter _writer = sheet.getSheetDataWriter(); + if (_writer != null) _writer.close(); } catch (IOException e) { logger.log(POILogger.WARN, "An exception occurred while closing sheet data writer for sheet " diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java index e37c7dc01e8..34b14450d65 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java @@ -79,133 +79,11 @@ protected StreamingSheetWriter createSheetDataWriter(OutputStream out) throws IO } @Override - protected void injectData(ZipEntrySource zipEntrySource, OutputStream out) throws IOException { - ZipArchiveOutputStream zos = new ZipArchiveOutputStream(out); - zos.setUseZip64(zip64Mode); - try { - Enumeration en = zipEntrySource.getEntries(); - while (en.hasMoreElements()) { - ZipArchiveEntry ze = en.nextElement(); - ZipArchiveEntry zeOut = new ZipArchiveEntry(ze.getName()); - zeOut.setSize(ze.getSize()); - zeOut.setTime(ze.getTime()); - zos.putArchiveEntry(zeOut); - try (final InputStream is = zipEntrySource.getInputStream(ze)) { - if (is instanceof ZipArchiveThresholdInputStream) { - // #59743 - disable Threshold handling for SXSSF copy - // as users tend to put too much repetitive data in when using SXSSF :) - ((ZipArchiveThresholdInputStream) is).setGuardState(false); - } - XSSFSheet xSheet = getSheetFromZipEntryName(ze.getName()); - // See bug 56557, we should not inject data into the special ChartSheets - if (xSheet != null && !(xSheet instanceof XSSFChartSheet)) { - SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xSheet); - if (sxSheet != null) { - copyStreamAndInjectWorksheet(is, zos, sxSheet); - } else { - IOUtils.copy(is, zos); - } - } else { - IOUtils.copy(is, zos); - } - } finally { - zos.closeArchiveEntry(); - } - } - } finally { - zos.finish(); - zipEntrySource.close(); - } - } - - private static void copyStreamAndInjectWorksheet(InputStream in, OutputStream out, SuperSXSSFSheet sxSheet) - throws IOException { - InputStreamReader inReader = new InputStreamReader(in, "UTF-8"); - OutputStreamWriter outWriter = new OutputStreamWriter(out, "UTF-8"); - boolean needsStartTag = true; - int c; - int pos = 0; - String s = "" or "" - // (excluding). - while (((c = inReader.read()) != -1)) { - if (c == s.charAt(pos)) { - pos++; - if (pos == n) { - if ("') { - // Found - outWriter.write(s); - outWriter.write(c); - s = ""; - n = s.length(); - pos = 0; - needsStartTag = false; - continue; - } - if (c == '/') { - // Found ') { - // Found - break; - } - - outWriter.write(s); - outWriter.write('/'); - outWriter.write(c); - pos = 0; - continue; - } - - outWriter.write(s); - outWriter.write('/'); - outWriter.write(c); - pos = 0; - continue; - } else { - // Found - break; - } - } - } else { - if (pos > 0) { - outWriter.write(s, 0, pos); - } - if (c == s.charAt(0)) { - pos = 1; - } else { - outWriter.write(c); - pos = 0; - } - } - } - outWriter.flush(); - if (needsStartTag) { - outWriter.write("\n"); - outWriter.flush(); - } - - // Invoke row generation in SXSSFSheet - sxSheet.writeRows(out); - - outWriter.write(""); - outWriter.flush(); - // Copy the rest of "in" to "out". - while (((c = inReader.read()) != -1)) { - outWriter.write(c); - } - outWriter.flush(); + protected ISheetInjector createSheetInjector(SXSSFSheet sxSheet) throws IOException { + SuperSXSSFSheet ssxSheet = (SuperSXSSFSheet) sxSheet; + return (output) -> { + ssxSheet.writeRows(output); + }; } @Override @@ -220,6 +98,14 @@ SXSSFSheet createAndRegisterSXSSFSheet(XSSFSheet xSheet) { return sxSheet; } + public SuperSXSSFSheet createSheet() { + return (SuperSXSSFSheet) super.createSheet(); + } + + public SuperSXSSFSheet createSheet(String sheetname) { + return (SuperSXSSFSheet) super.createSheet(sheetname); + } + /** * Returns an iterator of the sheets in the workbook in sheet order. Includes hidden and very hidden sheets. * @@ -285,13 +171,13 @@ public XSSFSheet getXSSFSheetAt(int index) { * @param index the index * @return the streaming sheet at */ - public SXSSFSheet getStreamingSheetAt(int index) { + public SuperSXSSFSheet getStreamingSheetAt(int index) { XSSFSheet xSheet = _wb.getSheetAt(index); SXSSFSheet sxSheet = getSXSSFSheet(xSheet); - if (sxSheet == null) { - return createAndRegisterSXSSFSheet(xSheet); + if (sxSheet == null && xSheet != null) { + return (SuperSXSSFSheet) createAndRegisterSXSSFSheet(xSheet); } else { - return sxSheet; + return (SuperSXSSFSheet) sxSheet; } } @@ -313,7 +199,7 @@ public XSSFSheet getXSSFSheet(String name) { public SuperSXSSFSheet getStreamingSheet(String name) { XSSFSheet xSheet = _wb.getSheet(name); SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xSheet); - if (sxSheet == null) { + if (sxSheet == null && xSheet != null) { return (SuperSXSSFSheet) createAndRegisterSXSSFSheet(xSheet); } else { return sxSheet; diff --git a/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java b/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java new file mode 100644 index 00000000000..6a9c2a37e98 --- /dev/null +++ b/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java @@ -0,0 +1,500 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.apache.poi.xssf.streaming; + +import static org.apache.poi.POITestCase.assertStartsWith; +import static org.apache.poi.POITestCase.assertEndsWith; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.poi.POIDataSamples; +import org.apache.poi.POITestCase; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.openxml4j.opc.PackageAccess; +import org.apache.poi.ss.usermodel.BaseTestXWorkbook; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.ss.util.CellReference; +import org.apache.poi.util.NullOutputStream; +import org.apache.poi.xssf.SXSSFITestDataProvider; +import org.apache.poi.xssf.XSSFTestDataSamples; +import org.apache.poi.xssf.model.SharedStringsTable; +import org.apache.poi.xssf.usermodel.XSSFCell; +import org.apache.poi.xssf.usermodel.XSSFRow; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.After; +import org.junit.Ignore; +import org.junit.Test; + +public final class TestSuperSXSSFWorkbook extends BaseTestXWorkbook { + + public TestSuperSXSSFWorkbook() { + super(SXSSFITestDataProvider.instance); + } + + @After + public void tearDown(){ + ((SXSSFITestDataProvider)_testDataProvider).cleanup(); + } + + /** + * cloning of sheets is not supported in SXSSF + */ + @Override + @Test + public void cloneSheet() throws IOException { + try { + super.cloneSheet(); + fail("expected exception"); + } catch (RuntimeException e){ + assertEquals("Not Implemented", e.getMessage()); + } + } + + /** + * cloning of sheets is not supported in SXSSF + */ + @Override + @Test + public void sheetClone() throws IOException { + try { + super.sheetClone(); + fail("expected exception"); + } catch (RuntimeException e){ + assertEquals("Not Implemented", e.getMessage()); + } + } + + /** + * Skip this test, as SXSSF doesn't update formulas on sheet name + * changes. + */ + @Override + @Ignore("SXSSF doesn't update formulas on sheet name changes, as most cells probably aren't in memory at the time") + @Test + public void setSheetName() { + } + + @Test + public void existingWorkbook() throws IOException { + XSSFWorkbook xssfWb1 = new XSSFWorkbook(); + xssfWb1.createSheet("S1"); + SuperSXSSFWorkbook wb1 = new SuperSXSSFWorkbook(xssfWb1); + XSSFWorkbook xssfWb2 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb1); + assertTrue(wb1.dispose()); + + SuperSXSSFWorkbook wb2 = new SuperSXSSFWorkbook(xssfWb2); + assertEquals(1, wb2.getNumberOfSheets()); + Sheet sheet = wb2.getStreamingSheetAt(0); + assertNotNull(sheet); + assertEquals("S1", sheet.getSheetName()); + assertTrue(wb2.dispose()); + xssfWb2.close(); + xssfWb1.close(); + + wb2.close(); + wb1.close(); + } + + @Test + public void useSharedStringsTable() throws Exception { + // not supported with SuperSXSSF + } + + @Test + public void addToExistingWorkbook() throws IOException { + XSSFWorkbook xssfWb1 = new XSSFWorkbook(); + xssfWb1.createSheet("S1"); + Sheet sheet = xssfWb1.createSheet("S2"); + Row row = sheet.createRow(1); + Cell cell = row.createCell(1); + cell.setCellValue("value 2_1_1"); + SuperSXSSFWorkbook wb1 = new SuperSXSSFWorkbook(xssfWb1); + XSSFWorkbook xssfWb2 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb1); + assertTrue(wb1.dispose()); + xssfWb1.close(); + + SuperSXSSFWorkbook wb2 = new SuperSXSSFWorkbook(xssfWb2); + // Add a row to the existing empty sheet + SuperSXSSFSheet ssheet1 = wb2.getStreamingSheetAt(0); + ssheet1.setRowGenerator((ssxSheet) -> { + Row row1_1 = ssxSheet.createRow(1); + Cell cell1_1_1 = row1_1.createCell(1); + cell1_1_1.setCellValue("value 1_1_1"); + }); + + // Add a row to the existing non-empty sheet + SuperSXSSFSheet ssheet2 = wb2.getStreamingSheetAt(1); + ssheet2.setRowGenerator((ssxSheet) -> { + Row row2_2 = ssxSheet.createRow(2); + Cell cell2_2_1 = row2_2.createCell(1); + cell2_2_1.setCellValue("value 2_2_1"); + }); + // Add a sheet with one row + SuperSXSSFSheet ssheet3 = wb2.createSheet("S3"); + ssheet3.setRowGenerator((ssxSheet) -> { + Row row3_1 = ssxSheet.createRow(1); + Cell cell3_1_1 = row3_1.createCell(1); + cell3_1_1.setCellValue("value 3_1_1"); + }); + + XSSFWorkbook xssfWb3 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb2); + wb2.close(); + + assertEquals(3, xssfWb3.getNumberOfSheets()); + // Verify sheet 1 + XSSFSheet sheet1 = xssfWb3.getSheetAt(0); + assertEquals("S1", sheet1.getSheetName()); + assertEquals(1, sheet1.getPhysicalNumberOfRows()); + XSSFRow row1_1 = sheet1.getRow(1); + assertNotNull(row1_1); + XSSFCell cell1_1_1 = row1_1.getCell(1); + assertNotNull(cell1_1_1); + assertEquals("value 1_1_1", cell1_1_1.getStringCellValue()); + // Verify sheet 2 + XSSFSheet sheet2 = xssfWb3.getSheetAt(1); + assertEquals("S2", sheet2.getSheetName()); + assertEquals(2, sheet2.getPhysicalNumberOfRows()); + Row row2_1 = sheet2.getRow(1); + assertNotNull(row2_1); + Cell cell2_1_1 = row2_1.getCell(1); + assertNotNull(cell2_1_1); + assertEquals("value 2_1_1", cell2_1_1.getStringCellValue()); + XSSFRow row2_2 = sheet2.getRow(2); + assertNotNull(row2_2); + XSSFCell cell2_2_1 = row2_2.getCell(1); + assertNotNull(cell2_2_1); + assertEquals("value 2_2_1", cell2_2_1.getStringCellValue()); + // Verify sheet 3 + XSSFSheet sheet3 = xssfWb3.getSheetAt(2); + assertEquals("S3", sheet3.getSheetName()); + assertEquals(1, sheet3.getPhysicalNumberOfRows()); + XSSFRow row3_1 = sheet3.getRow(1); + assertNotNull(row3_1); + XSSFCell cell3_1_1 = row3_1.getCell(1); + assertNotNull(cell3_1_1); + assertEquals("value 3_1_1", cell3_1_1.getStringCellValue()); + + xssfWb2.close(); + xssfWb3.close(); + wb1.close(); + } + + @Test + public void sheetdataWriter() throws IOException{ + SuperSXSSFWorkbook wb = new SuperSXSSFWorkbook(); + SXSSFSheet sh = wb.createSheet(); + assertSame(sh.getClass(), SuperSXSSFSheet.class); + SheetDataWriter wr = sh.getSheetDataWriter(); + assertNull(wr); + } + + @Test + public void gzipSheetdataWriter() throws IOException { + SuperSXSSFWorkbook wb = new SuperSXSSFWorkbook(); + wb.setCompressTempFiles(true); + + final int rowNum = 1000; + final int sheetNum = 5; + populateData(wb, 1000, 5); + + XSSFWorkbook xwb = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb); + for(int i = 0; i < sheetNum; i++){ + Sheet sh = xwb.getSheetAt(i); + assertEquals("sheet" + i, sh.getSheetName()); + for(int j = 0; j < rowNum; j++){ + Row row = sh.getRow(j); + assertNotNull("row[" + j + "]", row); + Cell cell1 = row.getCell(0); + assertEquals(new CellReference(cell1).formatAsString(), cell1.getStringCellValue()); + + Cell cell2 = row.getCell(1); + assertEquals(i, (int)cell2.getNumericCellValue()); + + Cell cell3 = row.getCell(2); + assertEquals(j, (int)cell3.getNumericCellValue()); + } + } + + assertTrue(wb.dispose()); + xwb.close(); + wb.close(); + } + + private static void assertWorkbookDispose(SuperSXSSFWorkbook wb) + { + populateData(wb, 1000, 5); + + for (Sheet sheet : wb) { + SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) sheet; + assertNull(sxSheet.getSheetDataWriter()); + } + + assertTrue(wb.dispose()); + + for (Sheet sheet : wb) { + SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) sheet; + assertNull(sxSheet.getSheetDataWriter()); + } + } + + private static void populateData(SuperSXSSFWorkbook wb, final int rowNum, final int sheetNum) { + for(int i = 0; i < sheetNum; i++){ + SuperSXSSFSheet sheet = wb.createSheet("sheet" + i); + int index = i; + sheet.setRowGenerator((sh) -> { + for(int j = 0; j < rowNum; j++){ + Row row = sh.createRow(j); + Cell cell1 = row.createCell(0); + cell1.setCellValue(new CellReference(cell1).formatAsString()); + + Cell cell2 = row.createCell(1); + cell2.setCellValue(index); + + Cell cell3 = row.createCell(2); + cell3.setCellValue(j); + } + }); + } + } + + @Test + public void workbookDispose() throws IOException { + SuperSXSSFWorkbook wb1 = new SuperSXSSFWorkbook(); + // the underlying writer is SheetDataWriter + assertWorkbookDispose(wb1); + wb1.close(); + + SuperSXSSFWorkbook wb2 = new SuperSXSSFWorkbook(); + wb2.setCompressTempFiles(true); + // the underlying writer is GZIPSheetDataWriter + assertWorkbookDispose(wb2); + wb2.close(); + } + + @Ignore("currently writing the same sheet multiple times is not supported...") + @Test + public void bug53515() throws Exception { + Workbook wb1 = new SXSSFWorkbook(10); + populateWorkbook(wb1); + saveTwice(wb1); + Workbook wb2 = new XSSFWorkbook(); + populateWorkbook(wb2); + saveTwice(wb2); + wb2.close(); + wb1.close(); + } + + @Ignore("Crashes the JVM because of documented JVM behavior with concurrent writing/reading of zip-files, " + + "see http://www.oracle.com/technetwork/java/javase/documentation/overview-156328.html") + @Test + public void bug53515a() throws Exception { + File out = new File("Test.xlsx"); + assertTrue(!out.exists() || out.delete()); + for (int i = 0; i < 2; i++) { + final SXSSFWorkbook wb; + if (out.exists()) { + wb = new SXSSFWorkbook( + (XSSFWorkbook) WorkbookFactory.create(out)); + } else { + wb = new SXSSFWorkbook(10); + } + + try { + FileOutputStream outSteam = new FileOutputStream(out); + if (i == 0) { + populateWorkbook(wb); + } else { + System.gc(); + System.gc(); + System.gc(); + } + + wb.write(outSteam); + // assertTrue(wb.dispose()); + outSteam.close(); + } finally { + assertTrue(wb.dispose()); + } + wb.close(); + } + assertTrue(out.exists()); + assertTrue(out.delete()); + } + + private static void populateWorkbook(Workbook wb) { + Sheet sh = wb.createSheet(); + for (int rownum = 0; rownum < 100; rownum++) { + Row row = sh.createRow(rownum); + for (int cellnum = 0; cellnum < 10; cellnum++) { + Cell cell = row.createCell(cellnum); + String address = new CellReference(cell).formatAsString(); + cell.setCellValue(address); + } + } + } + + private static void saveTwice(Workbook wb) throws Exception { + for (int i = 0; i < 2; i++) { + try { + NullOutputStream out = new NullOutputStream(); + wb.write(out); + out.close(); + } catch (Exception e) { + throw new Exception("ERROR: failed on " + (i + 1) + + "th time calling " + wb.getClass().getName() + + ".write() with exception " + e.getMessage(), e); + } + } + } + + @Test + public void closeDoesNotModifyWorkbook() throws IOException { + final String filename = "SampleSS.xlsx"; + final File file = POIDataSamples.getSpreadSheetInstance().getFile(filename); + + // Some tests commented out because close() modifies the file + // See bug 58779 + + // String + //wb = new SXSSFWorkbook(new XSSFWorkbook(file.getPath())); + //assertCloseDoesNotModifyFile(filename, wb); + + // File + //wb = new SXSSFWorkbook(new XSSFWorkbook(file)); + //assertCloseDoesNotModifyFile(filename, wb); + + // InputStream + + try (FileInputStream fis = new FileInputStream(file); + XSSFWorkbook xwb = new XSSFWorkbook(fis); + SXSSFWorkbook wb = new SXSSFWorkbook(xwb)) { + assertCloseDoesNotModifyFile(filename, wb); + } + + // OPCPackage + //wb = new SXSSFWorkbook(new XSSFWorkbook(OPCPackage.open(file))); + //assertCloseDoesNotModifyFile(filename, wb); + } + + /** + * Bug #59743 + * + * this is only triggered on other files apart of sheet[1,2,...].xml + * as those are either copied uncompressed or with the use of GZIPInputStream + * so we use shared strings + */ + @Test + public void testZipBombNotTriggeredOnUselessContent() throws IOException { + SXSSFWorkbook swb = new SXSSFWorkbook(null, 1, true, true); + SXSSFSheet s = swb.createSheet(); + char[] useless = new char[32767]; + Arrays.fill(useless, ' '); + + for (int row=0; row<1; row++) { + Row r = s.createRow(row); + for (int col=0; col<10; col++) { + char[] prefix = Integer.toHexString(row * 1000 + col).toCharArray(); + Arrays.fill(useless, 0, 10, ' '); + System.arraycopy(prefix, 0, useless, 0, prefix.length); + String ul = new String(useless); + r.createCell(col, CellType.STRING).setCellValue(ul); + } + } + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + swb.write(bos); + swb.dispose(); + swb.close(); + } + + /** + * To avoid accident changes to the template, you should be able + * to create a SXSSFWorkbook from a read-only XSSF one, then + * change + save that (only). See bug #60010 + * TODO Fix this to work! + */ + @Test + @Ignore + public void createFromReadOnlyWorkbook() throws Exception { + String sheetName = "Test SXSSF"; + File input = XSSFTestDataSamples.getSampleFile("sample.xlsx"); + + try (OPCPackage pkg = OPCPackage.open(input, PackageAccess.READ)) { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (XSSFWorkbook xssf = new XSSFWorkbook(pkg)) { + try (SXSSFWorkbook wb = new SXSSFWorkbook(xssf, 2)) { + Sheet s = wb.createSheet(sheetName); + for (int i = 0; i < 10; i++) { + Row r = s.createRow(i); + r.createCell(0).setCellValue(true); + r.createCell(1).setCellValue(2.4); + r.createCell(2).setCellValue("Test Row " + i); + } + assertEquals(10, s.getLastRowNum()); + + wb.write(bos); + wb.dispose(); + } + } + + try (XSSFWorkbook xssf = new XSSFWorkbook(new ByteArrayInputStream(bos.toByteArray()))) { + Sheet s = xssf.getSheet(sheetName); + assertEquals(10, s.getLastRowNum()); + assertTrue(s.getRow(0).getCell(0).getBooleanCellValue()); + assertEquals("Test Row 9", s.getRow(9).getCell(2).getStringCellValue()); + } + } + } + + + @Test + public void test56557() throws IOException { + Workbook wb = XSSFTestDataSamples.openSampleWorkbook("56557.xlsx"); + + // Using streaming XSSFWorkbook makes the output file invalid + wb = new SXSSFWorkbook(((XSSFWorkbook) wb)); + + // Should not throw POIXMLException: java.io.IOException: Unable to parse xml bean when reading back + Workbook wbBack = XSSFTestDataSamples.writeOutAndReadBack(wb); + assertNotNull(wbBack); + wbBack.close(); + + wb.close(); + } +} From 3b4c452530f7c9e47f0454719165fa3b2e5ada02 Mon Sep 17 00:00:00 2001 From: Matija Obreza Date: Wed, 23 Jan 2019 18:40:50 +0100 Subject: [PATCH 3/8] Added @Beta annotation --- .../org/apache/poi/xssf/streaming/IRowGenerator.java | 3 +++ .../poi/xssf/streaming/StreamingSheetWriter.java | 2 ++ .../apache/poi/xssf/streaming/SuperSXSSFSheet.java | 2 ++ .../poi/xssf/streaming/SuperSXSSFWorkbook.java | 12 ++---------- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java b/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java index 4435afdaca2..c006683f3de 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java @@ -17,9 +17,12 @@ Licensed to the Apache Software Foundation (ASF) under one or more package org.apache.poi.xssf.streaming; +import org.apache.poi.util.Beta; + /** * IRowGenerator for Super-streaming XSSF sheets */ +@Beta public interface IRowGenerator { /** diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/StreamingSheetWriter.java b/src/ooxml/java/org/apache/poi/xssf/streaming/StreamingSheetWriter.java index 2dce1fed128..1f6e2b49537 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/StreamingSheetWriter.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/StreamingSheetWriter.java @@ -25,6 +25,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more import java.io.OutputStreamWriter; import java.io.Writer; +import org.apache.poi.util.Beta; import org.apache.poi.util.POILogFactory; import org.apache.poi.util.POILogger; @@ -33,6 +34,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more * to the provided OutputStream. * @since 4.1.0 */ +@Beta public class StreamingSheetWriter extends SheetDataWriter { private static final POILogger logger = POILogFactory.getLogger(StreamingSheetWriter.class); diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java index f5441a690b3..13a317fc4b8 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java @@ -21,12 +21,14 @@ Licensed to the Apache Software Foundation (ASF) under one or more import java.io.InputStream; import java.io.OutputStream; +import org.apache.poi.util.Beta; import org.apache.poi.xssf.usermodel.XSSFSheet; /** * An update of SXSSFSheet that uses IRowGenerator to create rows. * @since 4.1.0 */ +@Beta public class SuperSXSSFSheet extends SXSSFSheet { private IRowGenerator rowGenerator; diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java index 34b14450d65..b19c4337e34 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java @@ -18,23 +18,14 @@ Licensed to the Apache Software Foundation (ASF) under one or more package org.apache.poi.xssf.streaming; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.util.Enumeration; import java.util.Iterator; import java.util.NoSuchElementException; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; -import org.apache.poi.openxml4j.util.ZipArchiveThresholdInputStream; -import org.apache.poi.openxml4j.util.ZipEntrySource; import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.util.IOUtils; +import org.apache.poi.util.Beta; import org.apache.poi.util.POILogFactory; import org.apache.poi.util.POILogger; -import org.apache.poi.xssf.usermodel.XSSFChartSheet; import org.apache.poi.xssf.usermodel.XSSFSheet; import org.apache.poi.xssf.usermodel.XSSFWorkbook; @@ -43,6 +34,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more * the provided OutputStream. * @since 4.1.0 */ +@Beta public class SuperSXSSFWorkbook extends SXSSFWorkbook { private static final POILogger logger = POILogFactory.getLogger(SuperSXSSFWorkbook.class); From 6f715c2a836f17bb89ad7e65afe92a62723566a8 Mon Sep 17 00:00:00 2001 From: Matija Obreza Date: Thu, 24 Jan 2019 09:37:59 +0100 Subject: [PATCH 4/8] Code formatted --- .../streaming/TestSuperSXSSFWorkbook.java | 318 +++++++++--------- 1 file changed, 153 insertions(+), 165 deletions(-) diff --git a/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java b/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java index 6a9c2a37e98..6637dd0bde9 100644 --- a/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java +++ b/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java @@ -19,10 +19,7 @@ package org.apache.poi.xssf.streaming; -import static org.apache.poi.POITestCase.assertStartsWith; -import static org.apache.poi.POITestCase.assertEndsWith; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; @@ -38,7 +35,6 @@ import java.util.Arrays; import org.apache.poi.POIDataSamples; -import org.apache.poi.POITestCase; import org.apache.poi.openxml4j.opc.OPCPackage; import org.apache.poi.openxml4j.opc.PackageAccess; import org.apache.poi.ss.usermodel.BaseTestXWorkbook; @@ -52,7 +48,6 @@ import org.apache.poi.util.NullOutputStream; import org.apache.poi.xssf.SXSSFITestDataProvider; import org.apache.poi.xssf.XSSFTestDataSamples; -import org.apache.poi.xssf.model.SharedStringsTable; import org.apache.poi.xssf.usermodel.XSSFCell; import org.apache.poi.xssf.usermodel.XSSFRow; import org.apache.poi.xssf.usermodel.XSSFSheet; @@ -62,16 +57,16 @@ import org.junit.Test; public final class TestSuperSXSSFWorkbook extends BaseTestXWorkbook { - + public TestSuperSXSSFWorkbook() { super(SXSSFITestDataProvider.instance); } - + @After - public void tearDown(){ - ((SXSSFITestDataProvider)_testDataProvider).cleanup(); + public void tearDown() { + ((SXSSFITestDataProvider) _testDataProvider).cleanup(); } - + /** * cloning of sheets is not supported in SXSSF */ @@ -81,11 +76,11 @@ public void cloneSheet() throws IOException { try { super.cloneSheet(); fail("expected exception"); - } catch (RuntimeException e){ + } catch (RuntimeException e) { assertEquals("Not Implemented", e.getMessage()); } } - + /** * cloning of sheets is not supported in SXSSF */ @@ -95,218 +90,217 @@ public void sheetClone() throws IOException { try { super.sheetClone(); fail("expected exception"); - } catch (RuntimeException e){ + } catch (RuntimeException e) { assertEquals("Not Implemented", e.getMessage()); } } - + /** - * Skip this test, as SXSSF doesn't update formulas on sheet name - * changes. + * Skip this test, as SXSSF doesn't update formulas on sheet name changes. */ @Override @Ignore("SXSSF doesn't update formulas on sheet name changes, as most cells probably aren't in memory at the time") @Test public void setSheetName() { } - + @Test public void existingWorkbook() throws IOException { - XSSFWorkbook xssfWb1 = new XSSFWorkbook(); - xssfWb1.createSheet("S1"); + XSSFWorkbook xssfWb1 = new XSSFWorkbook(); + xssfWb1.createSheet("S1"); SuperSXSSFWorkbook wb1 = new SuperSXSSFWorkbook(xssfWb1); - XSSFWorkbook xssfWb2 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb1); - assertTrue(wb1.dispose()); - - SuperSXSSFWorkbook wb2 = new SuperSXSSFWorkbook(xssfWb2); - assertEquals(1, wb2.getNumberOfSheets()); - Sheet sheet = wb2.getStreamingSheetAt(0); - assertNotNull(sheet); - assertEquals("S1", sheet.getSheetName()); - assertTrue(wb2.dispose()); - xssfWb2.close(); - xssfWb1.close(); - - wb2.close(); - wb1.close(); + XSSFWorkbook xssfWb2 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb1); + assertTrue(wb1.dispose()); + + SuperSXSSFWorkbook wb2 = new SuperSXSSFWorkbook(xssfWb2); + assertEquals(1, wb2.getNumberOfSheets()); + Sheet sheet = wb2.getStreamingSheetAt(0); + assertNotNull(sheet); + assertEquals("S1", sheet.getSheetName()); + assertTrue(wb2.dispose()); + xssfWb2.close(); + xssfWb1.close(); + + wb2.close(); + wb1.close(); } - + @Test public void useSharedStringsTable() throws Exception { // not supported with SuperSXSSF } - + @Test public void addToExistingWorkbook() throws IOException { - XSSFWorkbook xssfWb1 = new XSSFWorkbook(); - xssfWb1.createSheet("S1"); - Sheet sheet = xssfWb1.createSheet("S2"); - Row row = sheet.createRow(1); - Cell cell = row.createCell(1); - cell.setCellValue("value 2_1_1"); - SuperSXSSFWorkbook wb1 = new SuperSXSSFWorkbook(xssfWb1); - XSSFWorkbook xssfWb2 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb1); + XSSFWorkbook xssfWb1 = new XSSFWorkbook(); + xssfWb1.createSheet("S1"); + Sheet sheet = xssfWb1.createSheet("S2"); + Row row = sheet.createRow(1); + Cell cell = row.createCell(1); + cell.setCellValue("value 2_1_1"); + SuperSXSSFWorkbook wb1 = new SuperSXSSFWorkbook(xssfWb1); + XSSFWorkbook xssfWb2 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb1); assertTrue(wb1.dispose()); xssfWb1.close(); - + SuperSXSSFWorkbook wb2 = new SuperSXSSFWorkbook(xssfWb2); - // Add a row to the existing empty sheet - SuperSXSSFSheet ssheet1 = wb2.getStreamingSheetAt(0); - ssheet1.setRowGenerator((ssxSheet) -> { - Row row1_1 = ssxSheet.createRow(1); - Cell cell1_1_1 = row1_1.createCell(1); - cell1_1_1.setCellValue("value 1_1_1"); - }); - - // Add a row to the existing non-empty sheet - SuperSXSSFSheet ssheet2 = wb2.getStreamingSheetAt(1); - ssheet2.setRowGenerator((ssxSheet) -> { - Row row2_2 = ssxSheet.createRow(2); - Cell cell2_2_1 = row2_2.createCell(1); - cell2_2_1.setCellValue("value 2_2_1"); - }); - // Add a sheet with one row + // Add a row to the existing empty sheet + SuperSXSSFSheet ssheet1 = wb2.getStreamingSheetAt(0); + ssheet1.setRowGenerator((ssxSheet) -> { + Row row1_1 = ssxSheet.createRow(1); + Cell cell1_1_1 = row1_1.createCell(1); + cell1_1_1.setCellValue("value 1_1_1"); + }); + + // Add a row to the existing non-empty sheet + SuperSXSSFSheet ssheet2 = wb2.getStreamingSheetAt(1); + ssheet2.setRowGenerator((ssxSheet) -> { + Row row2_2 = ssxSheet.createRow(2); + Cell cell2_2_1 = row2_2.createCell(1); + cell2_2_1.setCellValue("value 2_2_1"); + }); + // Add a sheet with one row SuperSXSSFSheet ssheet3 = wb2.createSheet("S3"); ssheet3.setRowGenerator((ssxSheet) -> { - Row row3_1 = ssxSheet.createRow(1); - Cell cell3_1_1 = row3_1.createCell(1); - cell3_1_1.setCellValue("value 3_1_1"); + Row row3_1 = ssxSheet.createRow(1); + Cell cell3_1_1 = row3_1.createCell(1); + cell3_1_1.setCellValue("value 3_1_1"); }); - XSSFWorkbook xssfWb3 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb2); - wb2.close(); - - assertEquals(3, xssfWb3.getNumberOfSheets()); - // Verify sheet 1 - XSSFSheet sheet1 = xssfWb3.getSheetAt(0); - assertEquals("S1", sheet1.getSheetName()); - assertEquals(1, sheet1.getPhysicalNumberOfRows()); - XSSFRow row1_1 = sheet1.getRow(1); - assertNotNull(row1_1); - XSSFCell cell1_1_1 = row1_1.getCell(1); - assertNotNull(cell1_1_1); - assertEquals("value 1_1_1", cell1_1_1.getStringCellValue()); - // Verify sheet 2 - XSSFSheet sheet2 = xssfWb3.getSheetAt(1); - assertEquals("S2", sheet2.getSheetName()); - assertEquals(2, sheet2.getPhysicalNumberOfRows()); - Row row2_1 = sheet2.getRow(1); - assertNotNull(row2_1); - Cell cell2_1_1 = row2_1.getCell(1); - assertNotNull(cell2_1_1); - assertEquals("value 2_1_1", cell2_1_1.getStringCellValue()); - XSSFRow row2_2 = sheet2.getRow(2); - assertNotNull(row2_2); - XSSFCell cell2_2_1 = row2_2.getCell(1); - assertNotNull(cell2_2_1); - assertEquals("value 2_2_1", cell2_2_1.getStringCellValue()); - // Verify sheet 3 - XSSFSheet sheet3 = xssfWb3.getSheetAt(2); - assertEquals("S3", sheet3.getSheetName()); - assertEquals(1, sheet3.getPhysicalNumberOfRows()); - XSSFRow row3_1 = sheet3.getRow(1); - assertNotNull(row3_1); - XSSFCell cell3_1_1 = row3_1.getCell(1); - assertNotNull(cell3_1_1); - assertEquals("value 3_1_1", cell3_1_1.getStringCellValue()); - + XSSFWorkbook xssfWb3 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb2); + wb2.close(); + + assertEquals(3, xssfWb3.getNumberOfSheets()); + // Verify sheet 1 + XSSFSheet sheet1 = xssfWb3.getSheetAt(0); + assertEquals("S1", sheet1.getSheetName()); + assertEquals(1, sheet1.getPhysicalNumberOfRows()); + XSSFRow row1_1 = sheet1.getRow(1); + assertNotNull(row1_1); + XSSFCell cell1_1_1 = row1_1.getCell(1); + assertNotNull(cell1_1_1); + assertEquals("value 1_1_1", cell1_1_1.getStringCellValue()); + // Verify sheet 2 + XSSFSheet sheet2 = xssfWb3.getSheetAt(1); + assertEquals("S2", sheet2.getSheetName()); + assertEquals(2, sheet2.getPhysicalNumberOfRows()); + Row row2_1 = sheet2.getRow(1); + assertNotNull(row2_1); + Cell cell2_1_1 = row2_1.getCell(1); + assertNotNull(cell2_1_1); + assertEquals("value 2_1_1", cell2_1_1.getStringCellValue()); + XSSFRow row2_2 = sheet2.getRow(2); + assertNotNull(row2_2); + XSSFCell cell2_2_1 = row2_2.getCell(1); + assertNotNull(cell2_2_1); + assertEquals("value 2_2_1", cell2_2_1.getStringCellValue()); + // Verify sheet 3 + XSSFSheet sheet3 = xssfWb3.getSheetAt(2); + assertEquals("S3", sheet3.getSheetName()); + assertEquals(1, sheet3.getPhysicalNumberOfRows()); + XSSFRow row3_1 = sheet3.getRow(1); + assertNotNull(row3_1); + XSSFCell cell3_1_1 = row3_1.getCell(1); + assertNotNull(cell3_1_1); + assertEquals("value 3_1_1", cell3_1_1.getStringCellValue()); + xssfWb2.close(); - xssfWb3.close(); - wb1.close(); + xssfWb3.close(); + wb1.close(); } - + @Test - public void sheetdataWriter() throws IOException{ + public void sheetdataWriter() throws IOException { SuperSXSSFWorkbook wb = new SuperSXSSFWorkbook(); SXSSFSheet sh = wb.createSheet(); assertSame(sh.getClass(), SuperSXSSFSheet.class); SheetDataWriter wr = sh.getSheetDataWriter(); assertNull(wr); + wb.close(); } - + @Test public void gzipSheetdataWriter() throws IOException { SuperSXSSFWorkbook wb = new SuperSXSSFWorkbook(); wb.setCompressTempFiles(true); - + final int rowNum = 1000; final int sheetNum = 5; populateData(wb, 1000, 5); - + XSSFWorkbook xwb = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb); - for(int i = 0; i < sheetNum; i++){ + for (int i = 0; i < sheetNum; i++) { Sheet sh = xwb.getSheetAt(i); assertEquals("sheet" + i, sh.getSheetName()); - for(int j = 0; j < rowNum; j++){ + for (int j = 0; j < rowNum; j++) { Row row = sh.getRow(j); assertNotNull("row[" + j + "]", row); Cell cell1 = row.getCell(0); assertEquals(new CellReference(cell1).formatAsString(), cell1.getStringCellValue()); - + Cell cell2 = row.getCell(1); - assertEquals(i, (int)cell2.getNumericCellValue()); - + assertEquals(i, (int) cell2.getNumericCellValue()); + Cell cell3 = row.getCell(2); - assertEquals(j, (int)cell3.getNumericCellValue()); + assertEquals(j, (int) cell3.getNumericCellValue()); } } - + assertTrue(wb.dispose()); xwb.close(); wb.close(); } - - private static void assertWorkbookDispose(SuperSXSSFWorkbook wb) - { + + private static void assertWorkbookDispose(SuperSXSSFWorkbook wb) { populateData(wb, 1000, 5); - + for (Sheet sheet : wb) { SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) sheet; assertNull(sxSheet.getSheetDataWriter()); } - + assertTrue(wb.dispose()); - + for (Sheet sheet : wb) { SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) sheet; assertNull(sxSheet.getSheetDataWriter()); } } - + private static void populateData(SuperSXSSFWorkbook wb, final int rowNum, final int sheetNum) { - for(int i = 0; i < sheetNum; i++){ + for (int i = 0; i < sheetNum; i++) { SuperSXSSFSheet sheet = wb.createSheet("sheet" + i); int index = i; sheet.setRowGenerator((sh) -> { - for(int j = 0; j < rowNum; j++){ + for (int j = 0; j < rowNum; j++) { Row row = sh.createRow(j); Cell cell1 = row.createCell(0); cell1.setCellValue(new CellReference(cell1).formatAsString()); - + Cell cell2 = row.createCell(1); cell2.setCellValue(index); - + Cell cell3 = row.createCell(2); cell3.setCellValue(j); } }); } } - + @Test public void workbookDispose() throws IOException { SuperSXSSFWorkbook wb1 = new SuperSXSSFWorkbook(); // the underlying writer is SheetDataWriter assertWorkbookDispose(wb1); wb1.close(); - + SuperSXSSFWorkbook wb2 = new SuperSXSSFWorkbook(); wb2.setCompressTempFiles(true); // the underlying writer is GZIPSheetDataWriter assertWorkbookDispose(wb2); wb2.close(); } - + @Ignore("currently writing the same sheet multiple times is not supported...") @Test public void bug53515() throws Exception { @@ -319,7 +313,7 @@ public void bug53515() throws Exception { wb2.close(); wb1.close(); } - + @Ignore("Crashes the JVM because of documented JVM behavior with concurrent writing/reading of zip-files, " + "see http://www.oracle.com/technetwork/java/javase/documentation/overview-156328.html") @Test @@ -329,12 +323,11 @@ public void bug53515a() throws Exception { for (int i = 0; i < 2; i++) { final SXSSFWorkbook wb; if (out.exists()) { - wb = new SXSSFWorkbook( - (XSSFWorkbook) WorkbookFactory.create(out)); + wb = new SXSSFWorkbook((XSSFWorkbook) WorkbookFactory.create(out)); } else { wb = new SXSSFWorkbook(10); } - + try { FileOutputStream outSteam = new FileOutputStream(out); if (i == 0) { @@ -344,7 +337,7 @@ public void bug53515a() throws Exception { System.gc(); System.gc(); } - + wb.write(outSteam); // assertTrue(wb.dispose()); outSteam.close(); @@ -356,7 +349,7 @@ public void bug53515a() throws Exception { assertTrue(out.exists()); assertTrue(out.delete()); } - + private static void populateWorkbook(Workbook wb) { Sheet sh = wb.createSheet(); for (int rownum = 0; rownum < 100; rownum++) { @@ -368,7 +361,7 @@ private static void populateWorkbook(Workbook wb) { } } } - + private static void saveTwice(Workbook wb) throws Exception { for (int i = 0; i < 2; i++) { try { @@ -376,48 +369,46 @@ private static void saveTwice(Workbook wb) throws Exception { wb.write(out); out.close(); } catch (Exception e) { - throw new Exception("ERROR: failed on " + (i + 1) - + "th time calling " + wb.getClass().getName() + throw new Exception("ERROR: failed on " + (i + 1) + "th time calling " + wb.getClass().getName() + ".write() with exception " + e.getMessage(), e); } } } - + @Test public void closeDoesNotModifyWorkbook() throws IOException { final String filename = "SampleSS.xlsx"; final File file = POIDataSamples.getSpreadSheetInstance().getFile(filename); - + // Some tests commented out because close() modifies the file // See bug 58779 // String - //wb = new SXSSFWorkbook(new XSSFWorkbook(file.getPath())); - //assertCloseDoesNotModifyFile(filename, wb); + // wb = new SXSSFWorkbook(new XSSFWorkbook(file.getPath())); + // assertCloseDoesNotModifyFile(filename, wb); // File - //wb = new SXSSFWorkbook(new XSSFWorkbook(file)); - //assertCloseDoesNotModifyFile(filename, wb); + // wb = new SXSSFWorkbook(new XSSFWorkbook(file)); + // assertCloseDoesNotModifyFile(filename, wb); // InputStream - + try (FileInputStream fis = new FileInputStream(file); - XSSFWorkbook xwb = new XSSFWorkbook(fis); - SXSSFWorkbook wb = new SXSSFWorkbook(xwb)) { + XSSFWorkbook xwb = new XSSFWorkbook(fis); + SXSSFWorkbook wb = new SXSSFWorkbook(xwb)) { assertCloseDoesNotModifyFile(filename, wb); } // OPCPackage - //wb = new SXSSFWorkbook(new XSSFWorkbook(OPCPackage.open(file))); - //assertCloseDoesNotModifyFile(filename, wb); + // wb = new SXSSFWorkbook(new XSSFWorkbook(OPCPackage.open(file))); + // assertCloseDoesNotModifyFile(filename, wb); } /** * Bug #59743 * - * this is only triggered on other files apart of sheet[1,2,...].xml - * as those are either copied uncompressed or with the use of GZIPInputStream - * so we use shared strings + * this is only triggered on other files apart of sheet[1,2,...].xml as those are either copied uncompressed or with + * the use of GZIPInputStream so we use shared strings */ @Test public void testZipBombNotTriggeredOnUselessContent() throws IOException { @@ -426,9 +417,9 @@ public void testZipBombNotTriggeredOnUselessContent() throws IOException { char[] useless = new char[32767]; Arrays.fill(useless, ' '); - for (int row=0; row<1; row++) { + for (int row = 0; row < 1; row++) { Row r = s.createRow(row); - for (int col=0; col<10; col++) { + for (int col = 0; col < 10; col++) { char[] prefix = Integer.toHexString(row * 1000 + col).toCharArray(); Arrays.fill(useless, 0, 10, ' '); System.arraycopy(prefix, 0, useless, 0, prefix.length); @@ -444,17 +435,15 @@ public void testZipBombNotTriggeredOnUselessContent() throws IOException { } /** - * To avoid accident changes to the template, you should be able - * to create a SXSSFWorkbook from a read-only XSSF one, then - * change + save that (only). See bug #60010 - * TODO Fix this to work! + * To avoid accident changes to the template, you should be able to create a SXSSFWorkbook from a read-only XSSF + * one, then change + save that (only). See bug #60010 TODO Fix this to work! */ @Test @Ignore public void createFromReadOnlyWorkbook() throws Exception { String sheetName = "Test SXSSF"; File input = XSSFTestDataSamples.getSampleFile("sample.xlsx"); - + try (OPCPackage pkg = OPCPackage.open(input, PackageAccess.READ)) { ByteArrayOutputStream bos = new ByteArrayOutputStream(); try (XSSFWorkbook xssf = new XSSFWorkbook(pkg)) { @@ -467,12 +456,12 @@ public void createFromReadOnlyWorkbook() throws Exception { r.createCell(2).setCellValue("Test Row " + i); } assertEquals(10, s.getLastRowNum()); - + wb.write(bos); wb.dispose(); } } - + try (XSSFWorkbook xssf = new XSSFWorkbook(new ByteArrayInputStream(bos.toByteArray()))) { Sheet s = xssf.getSheet(sheetName); assertEquals(10, s.getLastRowNum()); @@ -481,20 +470,19 @@ public void createFromReadOnlyWorkbook() throws Exception { } } } - - + @Test public void test56557() throws IOException { Workbook wb = XSSFTestDataSamples.openSampleWorkbook("56557.xlsx"); - + // Using streaming XSSFWorkbook makes the output file invalid wb = new SXSSFWorkbook(((XSSFWorkbook) wb)); - + // Should not throw POIXMLException: java.io.IOException: Unable to parse xml bean when reading back Workbook wbBack = XSSFTestDataSamples.writeOutAndReadBack(wb); assertNotNull(wbBack); wbBack.close(); - + wb.close(); } } From fb3428fe973e40a7011a41d7863a3e4d9f9b9509 Mon Sep 17 00:00:00 2001 From: Matija Obreza Date: Thu, 25 Jun 2020 14:18:37 +0200 Subject: [PATCH 5/8] Update for current trunk --- .../org/apache/poi/xssf/streaming/SXSSFWorkbook.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java index 4e3b0bc7b5e..28aceef9937 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java @@ -414,7 +414,7 @@ protected void injectData(ZipEntrySource zipEntrySource, OutputStream out) throw SXSSFSheet sxSheet = getSXSSFSheet(xSheet); try (InputStream xis = sxSheet.getWorksheetXMLInputStream()) { // copyStreamAndInjectWorksheet(is, zos, xis); - copyStreamAndInjectWorksheet(is, zos, createSheetInjector(sxSheet)); + copyStreamAndInjectWorksheet(is, zos, createSheetInjector(xis)); } } else { IOUtils.copy(is, zos); @@ -439,12 +439,9 @@ protected ZipArchiveOutputStream createArchiveOutputStream(OutputStream out) { } } - protected ISheetInjector createSheetInjector(SXSSFSheet sxSheet) throws IOException { + protected ISheetInjector createSheetInjector(InputStream xis) throws IOException { return (output) -> { - try (InputStream xis = sxSheet.getWorksheetXMLInputStream()) { - // Copy the worksheet data to "output". - IOUtils.copy(xis, output); - } + IOUtils.copy(xis, output); }; } From b3b72359ae3b44c301a258831009d187901498af Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Fri, 26 Jun 2020 13:15:19 +0200 Subject: [PATCH 6/8] rename classes --- ...XSSFSheet.java => EmittingSXSSFSheet.java} | 16 ++-- ...rkbook.java => EmittingSXSSFWorkbook.java} | 77 +++++++++---------- .../poi/xssf/streaming/IRowGenerator.java | 4 +- .../apache/poi/xssf/streaming/SXSSFSheet.java | 2 +- .../poi/xssf/streaming/SXSSFWorkbook.java | 8 +- ...ok.java => TestEmittingSXSSFWorkbook.java} | 40 +++++----- 6 files changed, 73 insertions(+), 74 deletions(-) rename src/ooxml/java/org/apache/poi/xssf/streaming/{SuperSXSSFSheet.java => EmittingSXSSFSheet.java} (80%) rename src/ooxml/java/org/apache/poi/xssf/streaming/{SuperSXSSFWorkbook.java => EmittingSXSSFWorkbook.java} (73%) rename src/ooxml/testcases/org/apache/poi/xssf/streaming/{TestSuperSXSSFWorkbook.java => TestEmittingSXSSFWorkbook.java} (92%) diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java b/src/ooxml/java/org/apache/poi/xssf/streaming/EmittingSXSSFSheet.java similarity index 80% rename from src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java rename to src/ooxml/java/org/apache/poi/xssf/streaming/EmittingSXSSFSheet.java index 13a317fc4b8..c0b49d5d7a3 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFSheet.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/EmittingSXSSFSheet.java @@ -25,20 +25,24 @@ Licensed to the Apache Software Foundation (ASF) under one or more import org.apache.poi.xssf.usermodel.XSSFSheet; /** - * An update of SXSSFSheet that uses IRowGenerator to create rows. - * @since 4.1.0 + * A variant of SXSSFSheet that uses IRowGenerator to create rows. + * + * This variant is experimental and APIs may change at short notice. + * + * @see EmittingSXSSFWorkbook + * @since 5.0.0 */ @Beta -public class SuperSXSSFSheet extends SXSSFSheet { +public class EmittingSXSSFSheet extends SXSSFSheet { private IRowGenerator rowGenerator; - public SuperSXSSFSheet(SuperSXSSFWorkbook workbook, XSSFSheet xSheet) throws IOException { + public EmittingSXSSFSheet(EmittingSXSSFWorkbook workbook, XSSFSheet xSheet) throws IOException { super(workbook, xSheet, workbook.getRandomAccessWindowSize()); } @Override public InputStream getWorksheetXMLInputStream() throws IOException { - throw new RuntimeException("Not supported by SuperSXSSFSheet"); + throw new RuntimeException("Not supported by EmittingSXSSFSheet"); } public void setRowGenerator(IRowGenerator rowGenerator) { @@ -47,7 +51,7 @@ public void setRowGenerator(IRowGenerator rowGenerator) { public void writeRows(OutputStream out) throws IOException { // delayed creation of SheetDataWriter - _writer = ((SuperSXSSFWorkbook) _workbook).createSheetDataWriter(out); + _writer = ((EmittingSXSSFWorkbook) _workbook).createSheetDataWriter(out); try { if (this.rowGenerator != null) { this.rowGenerator.generateRows(this); diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java b/src/ooxml/java/org/apache/poi/xssf/streaming/EmittingSXSSFWorkbook.java similarity index 73% rename from src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java rename to src/ooxml/java/org/apache/poi/xssf/streaming/EmittingSXSSFWorkbook.java index b19c4337e34..d3141840163 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SuperSXSSFWorkbook.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/EmittingSXSSFWorkbook.java @@ -30,59 +30,52 @@ Licensed to the Apache Software Foundation (ASF) under one or more import org.apache.poi.xssf.usermodel.XSSFWorkbook; /** - * An update of SXSSFWorkbook avoids generating a temporary file and writes data directly to + * An variant of SXSSFWorkbook that avoids generating a temporary file and writes data directly to * the provided OutputStream. - * @since 4.1.0 + * + * This variant is experimental and APIs may change at short notice. + * + * @since 5.0.0 */ @Beta -public class SuperSXSSFWorkbook extends SXSSFWorkbook { - private static final POILogger logger = POILogFactory.getLogger(SuperSXSSFWorkbook.class); +public class EmittingSXSSFWorkbook extends SXSSFWorkbook { + private static final POILogger logger = POILogFactory.getLogger(EmittingSXSSFWorkbook.class); - public SuperSXSSFWorkbook() { + public EmittingSXSSFWorkbook() { this(null); } - public SuperSXSSFWorkbook(XSSFWorkbook workbook) { + public EmittingSXSSFWorkbook(XSSFWorkbook workbook) { this(workbook, SXSSFWorkbook.DEFAULT_WINDOW_SIZE); } - public SuperSXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize) { - setRandomAccessWindowSize(rowAccessWindowSize); - _sharedStringSource = null; - if (workbook == null) { - _wb = new XSSFWorkbook(); - } else { - _wb = workbook; - } - } - - @Override - public void setCompressTempFiles(boolean compress) { - // NOOP + public EmittingSXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize) { + super(workbook, rowAccessWindowSize, false, false); } @Override protected SheetDataWriter createSheetDataWriter() throws IOException { - throw new RuntimeException("Not supported by SuperSXSSFWorkbook"); + throw new RuntimeException("Not supported by EmittingSXSSFWorkbook"); } protected StreamingSheetWriter createSheetDataWriter(OutputStream out) throws IOException { return new StreamingSheetWriter(out); } - @Override - protected ISheetInjector createSheetInjector(SXSSFSheet sxSheet) throws IOException { - SuperSXSSFSheet ssxSheet = (SuperSXSSFSheet) sxSheet; - return (output) -> { - ssxSheet.writeRows(output); - }; - } +// @Override +// protected ISheetInjector createSheetInjector(InputStream xis) throws IOException +// protected ISheetInjector createSheetInjector(SXSSFSheet sxSheet) throws IOException { +// EmittingSXSSFSheet ssxSheet = (EmittingSXSSFSheet) sxSheet; +// return (output) -> { +// ssxSheet.writeRows(output); +// }; +// } @Override SXSSFSheet createAndRegisterSXSSFSheet(XSSFSheet xSheet) { - final SuperSXSSFSheet sxSheet; + final EmittingSXSSFSheet sxSheet; try { - sxSheet = new SuperSXSSFSheet(this, xSheet); + sxSheet = new EmittingSXSSFSheet(this, xSheet); } catch (IOException ioe) { throw new RuntimeException(ioe); } @@ -90,12 +83,12 @@ SXSSFSheet createAndRegisterSXSSFSheet(XSSFSheet xSheet) { return sxSheet; } - public SuperSXSSFSheet createSheet() { - return (SuperSXSSFSheet) super.createSheet(); + public EmittingSXSSFSheet createSheet() { + return (EmittingSXSSFSheet) super.createSheet(); } - public SuperSXSSFSheet createSheet(String sheetname) { - return (SuperSXSSFSheet) super.createSheet(sheetname); + public EmittingSXSSFSheet createSheet(String sheetname) { + return (EmittingSXSSFSheet) super.createSheet(sheetname); } /** @@ -125,7 +118,7 @@ public boolean hasNext() { @SuppressWarnings("unchecked") public T next() throws NoSuchElementException { final XSSFSheet xssfSheet = it.next(); - SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xssfSheet); + EmittingSXSSFSheet sxSheet = (EmittingSXSSFSheet) getSXSSFSheet(xssfSheet); return (T) (sxSheet == null ? xssfSheet : sxSheet); } @@ -150,7 +143,7 @@ public Iterator iterator() { @Override public SXSSFSheet getSheetAt(int index) { - throw new RuntimeException("Not supported by SuperSXSSFWorkbook"); + throw new RuntimeException("Not supported by EmittingSXSSFWorkbook"); } public XSSFSheet getXSSFSheetAt(int index) { @@ -163,19 +156,19 @@ public XSSFSheet getXSSFSheetAt(int index) { * @param index the index * @return the streaming sheet at */ - public SuperSXSSFSheet getStreamingSheetAt(int index) { + public EmittingSXSSFSheet getStreamingSheetAt(int index) { XSSFSheet xSheet = _wb.getSheetAt(index); SXSSFSheet sxSheet = getSXSSFSheet(xSheet); if (sxSheet == null && xSheet != null) { - return (SuperSXSSFSheet) createAndRegisterSXSSFSheet(xSheet); + return (EmittingSXSSFSheet) createAndRegisterSXSSFSheet(xSheet); } else { - return (SuperSXSSFSheet) sxSheet; + return (EmittingSXSSFSheet) sxSheet; } } @Override public SXSSFSheet getSheet(String name) { - throw new RuntimeException("Not supported by SuperSXSSFWorkbook"); + throw new RuntimeException("Not supported by EmittingSXSSFWorkbook"); } public XSSFSheet getXSSFSheet(String name) { @@ -188,11 +181,11 @@ public XSSFSheet getXSSFSheet(String name) { * @param name the name * @return the streaming sheet */ - public SuperSXSSFSheet getStreamingSheet(String name) { + public EmittingSXSSFSheet getStreamingSheet(String name) { XSSFSheet xSheet = _wb.getSheet(name); - SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) getSXSSFSheet(xSheet); + EmittingSXSSFSheet sxSheet = (EmittingSXSSFSheet) getSXSSFSheet(xSheet); if (sxSheet == null && xSheet != null) { - return (SuperSXSSFSheet) createAndRegisterSXSSFSheet(xSheet); + return (EmittingSXSSFSheet) createAndRegisterSXSSFSheet(xSheet); } else { return sxSheet; } diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java b/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java index c006683f3de..c9ec4e6c5d0 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/IRowGenerator.java @@ -20,7 +20,9 @@ Licensed to the Apache Software Foundation (ASF) under one or more import org.apache.poi.util.Beta; /** - * IRowGenerator for Super-streaming XSSF sheets + * IRowGenerator for Emitting SXSSF sheets + * + * @see EmittingSXSSFWorkbook */ @Beta public interface IRowGenerator { diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java index f819014d279..b8c5d76d259 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java @@ -1911,7 +1911,7 @@ boolean dispose() throws IOException { if (!allFlushed) { flushRows(); } - if (_writer == null) return true; // SuperSXSSFSheet initializes the writer only on write + if (_writer == null) return true; // EmittingSXSSFSheet initializes the writer only on write return _writer.dispose(); } diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java index 28aceef9937..af6a9b46e3e 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java @@ -190,7 +190,7 @@ public SXSSFWorkbook(XSSFWorkbook workbook){ * @param rowAccessWindowSize the number of rows that are kept in memory until flushed out, see above. */ public SXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize){ - this(workbook,rowAccessWindowSize, false); + this(workbook,rowAccessWindowSize, false); } /** @@ -214,7 +214,7 @@ public SXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize){ * @param rowAccessWindowSize the number of rows that are kept in memory until flushed out, see above. * @param compressTmpFiles whether to use gzip compression for temporary files */ - public SXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize, boolean compressTmpFiles){ + public SXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize, boolean compressTmpFiles) { this(workbook,rowAccessWindowSize, compressTmpFiles, false); } @@ -241,11 +241,11 @@ public SXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize, boolean com * @param compressTmpFiles whether to use gzip compression for temporary files * @param useSharedStringsTable whether to use a shared strings table */ - public SXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize, boolean compressTmpFiles, boolean useSharedStringsTable){ + public SXSSFWorkbook(XSSFWorkbook workbook, int rowAccessWindowSize, boolean compressTmpFiles, boolean useSharedStringsTable) { setRandomAccessWindowSize(rowAccessWindowSize); setCompressTempFiles(compressTmpFiles); if (workbook == null) { - _wb=new XSSFWorkbook(); + _wb = new XSSFWorkbook(); _sharedStringSource = useSharedStringsTable ? _wb.getSharedStringSource() : null; } else { _wb=workbook; diff --git a/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java b/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestEmittingSXSSFWorkbook.java similarity index 92% rename from src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java rename to src/ooxml/testcases/org/apache/poi/xssf/streaming/TestEmittingSXSSFWorkbook.java index 6637dd0bde9..df526b1b541 100644 --- a/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestSuperSXSSFWorkbook.java +++ b/src/ooxml/testcases/org/apache/poi/xssf/streaming/TestEmittingSXSSFWorkbook.java @@ -56,9 +56,9 @@ import org.junit.Ignore; import org.junit.Test; -public final class TestSuperSXSSFWorkbook extends BaseTestXWorkbook { +public final class TestEmittingSXSSFWorkbook extends BaseTestXWorkbook { - public TestSuperSXSSFWorkbook() { + public TestEmittingSXSSFWorkbook() { super(SXSSFITestDataProvider.instance); } @@ -108,11 +108,11 @@ public void setSheetName() { public void existingWorkbook() throws IOException { XSSFWorkbook xssfWb1 = new XSSFWorkbook(); xssfWb1.createSheet("S1"); - SuperSXSSFWorkbook wb1 = new SuperSXSSFWorkbook(xssfWb1); + EmittingSXSSFWorkbook wb1 = new EmittingSXSSFWorkbook(xssfWb1); XSSFWorkbook xssfWb2 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb1); assertTrue(wb1.dispose()); - SuperSXSSFWorkbook wb2 = new SuperSXSSFWorkbook(xssfWb2); + EmittingSXSSFWorkbook wb2 = new EmittingSXSSFWorkbook(xssfWb2); assertEquals(1, wb2.getNumberOfSheets()); Sheet sheet = wb2.getStreamingSheetAt(0); assertNotNull(sheet); @@ -127,7 +127,7 @@ public void existingWorkbook() throws IOException { @Test public void useSharedStringsTable() throws Exception { - // not supported with SuperSXSSF + // not supported with EmittingSXSSF } @Test @@ -138,14 +138,14 @@ public void addToExistingWorkbook() throws IOException { Row row = sheet.createRow(1); Cell cell = row.createCell(1); cell.setCellValue("value 2_1_1"); - SuperSXSSFWorkbook wb1 = new SuperSXSSFWorkbook(xssfWb1); + EmittingSXSSFWorkbook wb1 = new EmittingSXSSFWorkbook(xssfWb1); XSSFWorkbook xssfWb2 = SXSSFITestDataProvider.instance.writeOutAndReadBack(wb1); assertTrue(wb1.dispose()); xssfWb1.close(); - SuperSXSSFWorkbook wb2 = new SuperSXSSFWorkbook(xssfWb2); + EmittingSXSSFWorkbook wb2 = new EmittingSXSSFWorkbook(xssfWb2); // Add a row to the existing empty sheet - SuperSXSSFSheet ssheet1 = wb2.getStreamingSheetAt(0); + EmittingSXSSFSheet ssheet1 = wb2.getStreamingSheetAt(0); ssheet1.setRowGenerator((ssxSheet) -> { Row row1_1 = ssxSheet.createRow(1); Cell cell1_1_1 = row1_1.createCell(1); @@ -153,14 +153,14 @@ public void addToExistingWorkbook() throws IOException { }); // Add a row to the existing non-empty sheet - SuperSXSSFSheet ssheet2 = wb2.getStreamingSheetAt(1); + EmittingSXSSFSheet ssheet2 = wb2.getStreamingSheetAt(1); ssheet2.setRowGenerator((ssxSheet) -> { Row row2_2 = ssxSheet.createRow(2); Cell cell2_2_1 = row2_2.createCell(1); cell2_2_1.setCellValue("value 2_2_1"); }); // Add a sheet with one row - SuperSXSSFSheet ssheet3 = wb2.createSheet("S3"); + EmittingSXSSFSheet ssheet3 = wb2.createSheet("S3"); ssheet3.setRowGenerator((ssxSheet) -> { Row row3_1 = ssxSheet.createRow(1); Cell cell3_1_1 = row3_1.createCell(1); @@ -211,9 +211,9 @@ public void addToExistingWorkbook() throws IOException { @Test public void sheetdataWriter() throws IOException { - SuperSXSSFWorkbook wb = new SuperSXSSFWorkbook(); + EmittingSXSSFWorkbook wb = new EmittingSXSSFWorkbook(); SXSSFSheet sh = wb.createSheet(); - assertSame(sh.getClass(), SuperSXSSFSheet.class); + assertSame(sh.getClass(), EmittingSXSSFSheet.class); SheetDataWriter wr = sh.getSheetDataWriter(); assertNull(wr); wb.close(); @@ -221,7 +221,7 @@ public void sheetdataWriter() throws IOException { @Test public void gzipSheetdataWriter() throws IOException { - SuperSXSSFWorkbook wb = new SuperSXSSFWorkbook(); + EmittingSXSSFWorkbook wb = new EmittingSXSSFWorkbook(); wb.setCompressTempFiles(true); final int rowNum = 1000; @@ -251,25 +251,25 @@ public void gzipSheetdataWriter() throws IOException { wb.close(); } - private static void assertWorkbookDispose(SuperSXSSFWorkbook wb) { + private static void assertWorkbookDispose(EmittingSXSSFWorkbook wb) { populateData(wb, 1000, 5); for (Sheet sheet : wb) { - SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) sheet; + EmittingSXSSFSheet sxSheet = (EmittingSXSSFSheet) sheet; assertNull(sxSheet.getSheetDataWriter()); } assertTrue(wb.dispose()); for (Sheet sheet : wb) { - SuperSXSSFSheet sxSheet = (SuperSXSSFSheet) sheet; + EmittingSXSSFSheet sxSheet = (EmittingSXSSFSheet) sheet; assertNull(sxSheet.getSheetDataWriter()); } } - private static void populateData(SuperSXSSFWorkbook wb, final int rowNum, final int sheetNum) { + private static void populateData(EmittingSXSSFWorkbook wb, final int rowNum, final int sheetNum) { for (int i = 0; i < sheetNum; i++) { - SuperSXSSFSheet sheet = wb.createSheet("sheet" + i); + EmittingSXSSFSheet sheet = wb.createSheet("sheet" + i); int index = i; sheet.setRowGenerator((sh) -> { for (int j = 0; j < rowNum; j++) { @@ -289,12 +289,12 @@ private static void populateData(SuperSXSSFWorkbook wb, final int rowNum, final @Test public void workbookDispose() throws IOException { - SuperSXSSFWorkbook wb1 = new SuperSXSSFWorkbook(); + EmittingSXSSFWorkbook wb1 = new EmittingSXSSFWorkbook(); // the underlying writer is SheetDataWriter assertWorkbookDispose(wb1); wb1.close(); - SuperSXSSFWorkbook wb2 = new SuperSXSSFWorkbook(); + EmittingSXSSFWorkbook wb2 = new EmittingSXSSFWorkbook(); wb2.setCompressTempFiles(true); // the underlying writer is GZIPSheetDataWriter assertWorkbookDispose(wb2); From 85aee4064aa42e90542568bbaf7243a995bb9a3f Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Sat, 27 Jun 2020 00:30:41 +0200 Subject: [PATCH 7/8] tidy up dispose code --- src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java index b8c5d76d259..de42c3d5ef0 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFSheet.java @@ -1911,8 +1911,7 @@ boolean dispose() throws IOException { if (!allFlushed) { flushRows(); } - if (_writer == null) return true; // EmittingSXSSFSheet initializes the writer only on write - return _writer.dispose(); + return _writer == null || _writer.dispose(); } @Override From 958e022f7c8e034012d6faa5873bc3f1d5048b95 Mon Sep 17 00:00:00 2001 From: Matija Obreza Date: Sun, 28 Jun 2020 12:51:39 +0200 Subject: [PATCH 8/8] Fixed buggy update for current trunk --- .../poi/xssf/streaming/EmittingSXSSFWorkbook.java | 15 +++++++-------- .../apache/poi/xssf/streaming/SXSSFWorkbook.java | 12 ++++++------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/EmittingSXSSFWorkbook.java b/src/ooxml/java/org/apache/poi/xssf/streaming/EmittingSXSSFWorkbook.java index d3141840163..fb2d28a6ccd 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/EmittingSXSSFWorkbook.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/EmittingSXSSFWorkbook.java @@ -62,14 +62,13 @@ protected StreamingSheetWriter createSheetDataWriter(OutputStream out) throws IO return new StreamingSheetWriter(out); } -// @Override -// protected ISheetInjector createSheetInjector(InputStream xis) throws IOException -// protected ISheetInjector createSheetInjector(SXSSFSheet sxSheet) throws IOException { -// EmittingSXSSFSheet ssxSheet = (EmittingSXSSFSheet) sxSheet; -// return (output) -> { -// ssxSheet.writeRows(output); -// }; -// } + @Override + protected ISheetInjector createSheetInjector(SXSSFSheet sxSheet) throws IOException { + EmittingSXSSFSheet ssxSheet = (EmittingSXSSFSheet) sxSheet; + return (output) -> { + ssxSheet.writeRows(output); + }; + } @Override SXSSFSheet createAndRegisterSXSSFSheet(XSSFSheet xSheet) { diff --git a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java index af6a9b46e3e..99d9c5d5e80 100644 --- a/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java +++ b/src/ooxml/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java @@ -412,10 +412,7 @@ protected void injectData(ZipEntrySource zipEntrySource, OutputStream out) throw // See bug 56557, we should not inject data into the special ChartSheets if (xSheet != null && !(xSheet instanceof XSSFChartSheet)) { SXSSFSheet sxSheet = getSXSSFSheet(xSheet); - try (InputStream xis = sxSheet.getWorksheetXMLInputStream()) { - // copyStreamAndInjectWorksheet(is, zos, xis); - copyStreamAndInjectWorksheet(is, zos, createSheetInjector(xis)); - } + copyStreamAndInjectWorksheet(is, zos, createSheetInjector(sxSheet)); } else { IOUtils.copy(is, zos); } @@ -439,9 +436,12 @@ protected ZipArchiveOutputStream createArchiveOutputStream(OutputStream out) { } } - protected ISheetInjector createSheetInjector(InputStream xis) throws IOException { + protected ISheetInjector createSheetInjector(SXSSFSheet sxSheet) throws IOException { return (output) -> { - IOUtils.copy(xis, output); + try (InputStream xis = sxSheet.getWorksheetXMLInputStream()) { + // Copy the worksheet data to "output". + IOUtils.copy(xis, output); + } }; }