6701: Supporting LZ4 compressed recordings

Reviewed-by: hdafgard

core/org.openjdk.jmc.common/META-INF/MANIFEST.MF

@@ -13,4 +13,4 @@ Export-Package: org.openjdk.jmc.common,
  org.openjdk.jmc.common.util,
  org.openjdk.jmc.common.version
 Automatic-Module-Name: org.openjdk.jmc.common
-Require-Bundle: org.owasp.encoder
+Require-Bundle: org.owasp.encoder, org.lz4.lz4-java

core/org.openjdk.jmc.common/pom.xml

@@ -39,12 +39,17 @@
 		<version>8.0.0-SNAPSHOT</version>
 	</parent>
 	<artifactId>common</artifactId>
-   <dependencies>
+	<dependencies>
 		<dependency>
 			<groupId>org.owasp.encoder</groupId>
 			<artifactId>encoder</artifactId>
 			<version>1.2.2</version>
 		</dependency>
+		<dependency>
+			<groupId>org.lz4</groupId>
+			<artifactId>lz4-java</artifactId>
+			<version>1.7.1</version>
+		</dependency>
 	</dependencies>
 	<properties>
 		<spotless.config.path>${basedir}/../../configuration/ide/eclipse/formatting/formatting.xml</spotless.config.path>

core/org.openjdk.jmc.common/src/main/java/org/openjdk/jmc/common/io/IOToolkit.java

@@ -58,7 +58,7 @@
 import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipInputStream;
 
-import org.openjdk.jmc.common.messages.internal.Messages;
+import net.jpountz.lz4.LZ4FrameInputStream;
 
 /**
  * Common functionality you might want when you're working with I/O.
@@ -101,8 +101,8 @@ public static void closeSilently(Closeable closeable) {
 	}
 
 	/**
-	 * Get an input stream for a optionally compressed file. If the file is compressed using either
-	 * GZip or ZIP then an appropriate unpacking will be done.
+	 * Get an input stream for a optionally compressed file. If the file is compressed using GZip,
+	 * ZIP or LZ4, then an appropriate unpacking will be done.
 	 *
 	 * @param file
 	 *            file to read from
@@ -121,8 +121,7 @@ public static InputStream openUncompressedStream(File file) throws IOException {
 				zin.getNextEntry();
 				return zin;
 			} else if (hasMagic(file, MAGIC_LZ4)) {
-				throw new UnsupportedFormatException(
-						Messages.getString(Messages.UnsupportedFormatException_LZ4_NOT_SUPPORTED)); //$NON-NLS-1$
+				return new LZ4FrameInputStream(in);
 			}
 			return in;
 		} catch (RuntimeException e) {
@@ -138,8 +137,8 @@ public static InputStream openUncompressedStream(File file) throws IOException {
 	}
 
 	/**
-	 * Get an input stream for a optionally compressed input stream. If the input stream is
-	 * compressed using either GZip or ZIP then an appropriate unpacking will be done.
+	 * Get an input stream for a optionally compressed input stream. If the file is compressed using
+	 * GZip, ZIP or LZ4, then an appropriate unpacking will be done.
 	 *
 	 * @param stream
 	 *            input stream to read from
@@ -149,27 +148,29 @@ public static InputStream openUncompressedStream(File file) throws IOException {
 	 */
 	public static InputStream openUncompressedStream(InputStream stream) throws IOException {
 		InputStream in = stream;
-		if (!in.markSupported()) {
-			in = new BufferedInputStream(stream);
-		}
-		in.mark(MAGIC_GZ.length + 1);
-		if (hasMagic(in, MAGIC_GZ)) {
+		if (in.markSupported()) {
+			in.mark(MAGIC_GZ.length + 1);
+			if (hasMagic(in, MAGIC_GZ)) {
+				in.reset();
+				return new GZIPInputStream(in);
+			}
 			in.reset();
-			return new GZIPInputStream(in);
-		}
-		in.reset();
-		in.mark(MAGIC_ZIP.length + 1);
-		if (hasMagic(in, MAGIC_ZIP)) {
+			in.mark(MAGIC_ZIP.length + 1);
+			if (hasMagic(in, MAGIC_ZIP)) {
+				in.reset();
+				ZipInputStream zin = new ZipInputStream(in);
+				zin.getNextEntry();
+				return zin;
+			}
+			in.reset();
+			in.mark(MAGIC_LZ4.length + 1);
+			if (hasMagic(in, MAGIC_LZ4)) {
+				in.reset();
+				return new LZ4FrameInputStream(in);
+			}
 			in.reset();
-			ZipInputStream zin = new ZipInputStream(in);
-			zin.getNextEntry();
-			return zin;
-		}
-		if (hasMagic(in, MAGIC_LZ4)) {
-			throw new UnsupportedFormatException(
-					Messages.getString(Messages.UnsupportedFormatException_LZ4_NOT_SUPPORTED)); //$NON-NLS-1$
 		}
-		in.reset();
+		in = new BufferedInputStream(stream);
 		return in;
 	}
 
@@ -225,6 +226,19 @@ public static boolean isGZipFile(File file) throws IOException {
 		return hasMagic(file, MAGIC_GZ);
 	}
 
+	/**
+	 * Returns true if the file is LZ4 compressed.
+	 *
+	 * @param file
+	 *            the file to examine
+	 * @return {@code true} if it is an LZ4 compressed file, {@code false} otherwise
+	 * @throws IOException
+	 *             if an error occurred when trying to read from the file
+	 */
+	public static boolean isLZ4File(File file) throws IOException {
+		return hasMagic(file, MAGIC_LZ4);
+	}
+
 	/**
 	 * Checks if the file is a ZIP archive.
 	 *
@@ -258,6 +272,16 @@ public static int[] getZipMagic() {
 		return MAGIC_ZIP.clone();
 	}
 
+	/**
+	 * Returns the magic bytes for identifying LZ4. This is a defensive copy. It's up to the user to
+	 * cache this to avoid excessive allocations.
+	 * 
+	 * @return a copy of the magic bytes for LZ4.
+	 */
+	public static int[] getLz4Magic() {
+		return MAGIC_LZ4.clone();
+	}
+
 	/**
 	 * Checks if the file is compressed in a way compatible with
 	 * {@link #openUncompressedStream(File)}.
@@ -276,7 +300,12 @@ public static boolean isCompressedFile(File file) throws IOException {
 				return true;
 			}
 			is.reset();
-			return hasMagic(is, MAGIC_ZIP);
+			if (hasMagic(is, MAGIC_ZIP)) {
+				return true;
+			}
+			;
+			is.reset();
+			return hasMagic(is, MAGIC_LZ4);
 		}
 	}
 

core/org.openjdk.jmc.common/src/main/java/org/openjdk/jmc/common/messages/internal/Messages.java

@@ -72,7 +72,6 @@ public class Messages {
 	public static final String TimestampKind_SINCE_1970_MSG = "TimestampKind_SINCE_1970_MSG"; //$NON-NLS-1$
 	public static final String TypeHandling_MESSAGE_SIZE = "TypeHandling_MESSAGE_SIZE"; //$NON-NLS-1$
 	public static final String UnitLookup_TIMESTAMP_OUT_OF_RANGE = "UnitLookup_TIMESTAMP_OUT_OF_RANGE"; //$NON-NLS-1$
-	public static final String UnsupportedFormatException_LZ4_NOT_SUPPORTED = "UnsupportedFormatException_LZ4_NOT_SUPPORTED"; //$NON-NLS-1$
 
 	private Messages() {
 	}

core/org.openjdk.jmc.common/src/main/java/org/openjdk/jmc/common/util/XmlToolkit.java

@@ -61,7 +61,6 @@
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
 
-import org.openjdk.jmc.common.io.IOToolkit;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;

core/org.openjdk.jmc.common/src/main/resources/org/openjdk/jmc/common/messages/internal/messages.properties

@@ -182,4 +182,3 @@ RangeContentType_NAME=Range of {0}
 # {0} is a string
 MISSING_VALUE=N/A
 MISSING_VALUE_TOOLTIP=N/A ({0})
-UnsupportedFormatException_LZ4_NOT_SUPPORTED=LZ4 compression is not supported in this version

core/org.openjdk.jmc.flightrecorder.rules/pom.xml

@@ -58,6 +58,11 @@
 			<artifactId>encoder</artifactId>
 			<version>1.2.2</version>
 		</dependency>
+		<dependency>
+			<groupId>org.lz4</groupId>
+			<artifactId>lz4-java</artifactId>
+			<version>1.7.1</version>
+		</dependency>
 	</dependencies>
 
 	<build>

core/tests/org.openjdk.jmc.common.test/build.properties

@@ -30,7 +30,8 @@
 #  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
 #  WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
-source.. = src/test/java/
+source.. = src/test/java/,\
+           src/test/resources/
 output.. = target/test-classes/
 bin.includes = META-INF/,\
                .

core/tests/org.openjdk.jmc.common.test/src/test/java/org/openjdk/jmc/common/util/IOToolkitTest.java

@@ -32,6 +32,9 @@
  */
 package org.openjdk.jmc.common.util;
 
+import java.io.IOException;
+import java.io.InputStream;
+
 import org.junit.Assert;
 import org.junit.Test;
 import org.openjdk.jmc.common.io.IOToolkit;
@@ -40,9 +43,62 @@ public class IOToolkitTest {
 	private static final int MAGIC_ZIP[] = new int[] {80, 75, 3, 4};
 	private static final int MAGIC_GZ[] = new int[] {31, 139};
 
+	private static final String UNCOMPRESSED = "test.txt";
+	private static final String GZ = "test.txt.gz";
+	private static final String LZ4 = "test.txt.lz4";
+	private static final String ZIP = "test.txt.zip";
+
+	private static final String GURKA = "Gurka";
+
 	@Test
 	public void testGetMagics() {
 		Assert.assertArrayEquals(MAGIC_ZIP, IOToolkit.getZipMagic());
 		Assert.assertArrayEquals(MAGIC_GZ, IOToolkit.getGzipMagic());
 	}
+
+	@Test
+	public void testUncompressUncompressed() throws IOException {
+		InputStream uncompressedStream = IOToolkit.openUncompressedStream(getStream(UNCOMPRESSED));
+		String string = readFromStream(uncompressedStream);
+		Assert.assertEquals("String should be " + GURKA, GURKA, string);
+	}
+
+	@Test
+	public void testUncompressZipped() throws IOException {
+		InputStream uncompressedStream = IOToolkit.openUncompressedStream(getStream(ZIP));
+		String string = readFromStream(uncompressedStream);
+		Assert.assertEquals("String should be " + GURKA, GURKA, string);
+	}
+
+	@Test
+	public void testUncompressGZipped() throws IOException {
+		InputStream uncompressedStream = IOToolkit.openUncompressedStream(getStream(GZ));
+		String string = readFromStream(uncompressedStream);
+		Assert.assertEquals("String should be " + GURKA, GURKA, string);
+	}
+
+	@Test
+	public void testUncompressLZ4() throws IOException {
+		InputStream uncompressedStream = IOToolkit.openUncompressedStream(getStream(LZ4));
+		String string = readFromStream(uncompressedStream);
+		Assert.assertEquals("String should be " + GURKA, GURKA, string);
+	}
+
+	public InputStream getStream(String resourceName) throws IOException {
+		InputStream stream = getClass().getClassLoader().getResourceAsStream(resourceName);
+		if (stream == null) {
+			throw new IOException("Could not find the resource " + resourceName);
+		}
+		return stream;
+	}
+
+	public String readFromStream(InputStream stream) throws IOException {
+		StringBuilder builder = new StringBuilder();
+		int c = 0;
+		while ((c = stream.read()) != -1) {
+			builder.append((char) c);
+		}
+		return builder.toString();
+	}
+
 }

core/tests/org.openjdk.jmc.common.test/src/test/resources/test.txt

@@ -0,0 +1 @@
+Gurka