Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8190753: (zipfs): Accessing a large entry (> 2^31 bytes) leads to a negative initial size for ByteArrayOutputStream #4607

Closed
wants to merge 15 commits into from
Closed
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -105,6 +105,8 @@ class ZipFileSystem extends FileSystem {
private static final String COMPRESSION_METHOD_DEFLATED = "DEFLATED";
// Value specified for compressionMethod property to not compress Zip entries
private static final String COMPRESSION_METHOD_STORED = "STORED";
// The maximum size of array to allocate. Some VMs reserve some header words in an array.
private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;

private final ZipFileSystemProvider provider;
private final Path zfpath;
@@ -1948,7 +1950,7 @@ private OutputStream getOutputStream(Entry e) throws IOException {
e.file = getTempPathForEntry(null);
os = Files.newOutputStream(e.file, WRITE);
} else {
os = new ByteArrayOutputStream((e.size > 0)? (int)e.size : 8192);
os = new ByteArrayOutputStream((e.size > 0 && e.size <= MAX_ARRAY_SIZE)? (int)e.size : 8192);
}
Copy link
Contributor

@LanceAndersen LanceAndersen Jun 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The proposed change will address the specific issue shown in the bug. As Alan points out, there could be an issue if the deflated size is > 2gb. It would be good to look into that as part of your proposed fix.

if (e.method == METHOD_DEFLATED) {
return new DeflatingEntryOutputStream(e, os);
@@ -0,0 +1,111 @@
/*
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/

import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.zip.CRC32;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

/**
* This is intentionally a manual test. The (jtreg) configurations below are here only
* for reference about runtime expectations of this test.
*
* @bug 8190753
* @summary Verify that using zip filesystem for opening an outputstream for a zip entry whose
* compressed size is large, doesn't run into "Negative initial size" exception
* @requires (sun.arch.data.model == "64" & os.maxMemory >= 8g)
* @run testng/othervm -Xmx6g LargeCompressedEntrySizeTest
*/
public class LargeCompressedEntrySizeTest {

private static final String LARGE_FILE_NAME = "LargeZipEntry.txt";
private static final String ZIP_FILE_NAME = "8190753-test-compressed-size.zip";

@BeforeMethod
public void setUp() throws IOException {
deleteFiles();
}

@AfterMethod
public void tearDown() throws IOException {
deleteFiles();
}

/**
* Delete the files created for use by the test
*
* @throws IOException if an error occurs deleting the files
*/
private static void deleteFiles() throws IOException {
Files.deleteIfExists(Path.of(ZIP_FILE_NAME));
Files.deleteIfExists(Path.of(LARGE_FILE_NAME));
}


/**
* Using zip filesystem, creates a zip file and writes out a zip entry whose compressed size is
* expected to be greater than 2gb.
*/
@Test
public void testLargeCompressedSizeWithZipFS() throws Exception {
final Path zipFile = Path.of(ZIP_FILE_NAME);
final long largeFileSize = (2L * 1024L * 1024L * 1024L) + 1L;
final Random random = new Random();
try (FileSystem fs = FileSystems.newFileSystem(zipFile, Collections.singletonMap("create", "true"))) {
try (OutputStream os = Files.newOutputStream(fs.getPath(LARGE_FILE_NAME))) {
long remaining = largeFileSize;
final int chunkSize = 102400;
for (long l = 0; l < largeFileSize; l+=chunkSize) {
final int numToWrite = (int) Math.min(remaining, chunkSize);
final byte[] b = new byte[numToWrite];
// fill with random bytes
random.nextBytes(b);
os.write(b);
remaining -= b.length;
}
}
}
}

}
@@ -0,0 +1,138 @@
/*
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/

import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
* @test
* @bug 8190753
* @summary Verify that using zip filesystem for opening an outputstream for a large zip entry doesn't
* run into "Negative initial size" exception
* @requires (sun.arch.data.model == "64" & os.maxMemory >= 5g)
* @run testng/othervm -Xmx4g LargeEntrySizeTest
*/
public class LargeEntrySizeTest {

// a value which when cast to an integer, becomes a negative value
private static final long LARGE_FILE_SIZE = Integer.MAX_VALUE + 1L;
private static final long SMALL_FILE_SIZE = 0x100000L; // 1024L x 1024L;
private static final String LARGE_FILE_NAME = "LargeZipEntry.txt";
// File that will be created with a size less than 0xFFFFFFFF
private static final String SMALL_FILE_NAME = "SmallZipEntry.txt";
// List of files to be added to the ZIP file
private static final List<String> ZIP_ENTRIES = List.of(LARGE_FILE_NAME, SMALL_FILE_NAME);
private static final String ZIP_FILE_NAME = "8190753-test.zip";

@BeforeMethod
public void setUp() throws IOException {
deleteFiles();
}

@AfterMethod
public void tearDown() throws IOException {
deleteFiles();
}

/**
* Delete the files created for use by the test
*
* @throws IOException if an error occurs deleting the files
*/
private static void deleteFiles() throws IOException {
Files.deleteIfExists(Path.of(ZIP_FILE_NAME));
Files.deleteIfExists(Path.of(LARGE_FILE_NAME));
Files.deleteIfExists(Path.of(SMALL_FILE_NAME));
}


/**
* Verifies that large entry (whose size is greater than {@link Integer#MAX_VALUE}) in a zip file
* can be opened as an {@link OutputStream} using the zip filesystem
*/
@Test
public void testLargeEntryZipFSOutputStream() throws Exception {
final Path zipFile = Path.of(ZIP_FILE_NAME);
createZipFile(zipFile);
try (FileSystem fs = FileSystems.newFileSystem(zipFile)) {
for (String entryName : ZIP_ENTRIES) {
try (OutputStream os = Files.newOutputStream(fs.getPath(entryName), StandardOpenOption.WRITE)) {
// just a dummy write
os.write(0x01);
}
}
}
}

/**
* Creates a zip file with an entry whose size is larger than {@link Integer#MAX_VALUE}
*/
private static void createZipFile(final Path zipFile) throws IOException {
createFiles();
try (OutputStream os = Files.newOutputStream(zipFile);
ZipOutputStream zos = new ZipOutputStream(os)) {
System.out.println("Creating Zip file: " + zipFile.getFileName());
for (String srcFile : ZIP_ENTRIES) {
File fileToZip = new File(srcFile);
long fileSize = fileToZip.length();
System.out.println("Adding entry " + srcFile + " of size " + fileSize + " bytes");
try (FileInputStream fis = new FileInputStream(fileToZip)) {
ZipEntry zipEntry = new ZipEntry(fileToZip.getName());
zipEntry.setSize(fileSize);
zos.putNextEntry(zipEntry);
fis.transferTo(zos);
}
}
}
}

/**
* Create the files that will be added to the ZIP file
*/
private static void createFiles() throws IOException {
try (RandomAccessFile largeFile = new RandomAccessFile(LARGE_FILE_NAME, "rw");
RandomAccessFile smallFile = new RandomAccessFile(SMALL_FILE_NAME, "rw")) {
System.out.printf("Creating %s%n", LARGE_FILE_NAME);
largeFile.setLength(LARGE_FILE_SIZE);
System.out.printf("Creating %s%n", SMALL_FILE_NAME);
smallFile.setLength(SMALL_FILE_SIZE);
}
}

}