Skip to content

Commit

Permalink
SOLR-16949: Handle inputStream leaks
Browse files Browse the repository at this point in the history
(cherry picked from commit c89813a)
  • Loading branch information
janhoy committed Dec 17, 2023
1 parent 734920a commit c79011e
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 60 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -214,12 +214,11 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
"Not including uploading file to config, as it is a forbidden type: {}",
file.getFileName());
} else {
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(Files.newInputStream(file))) {
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(file)) {
Files.copy(
file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
} else {
String mimeType =
FileTypeMagicUtil.INSTANCE.guessMimeType(Files.newInputStream(file));
String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(file);
log.warn(
"Not copying file {}, as it matched the MAGIC signature of a forbidden mime type {}",
file.getFileName(),
Expand Down
93 changes: 64 additions & 29 deletions solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import com.j256.simplemagic.ContentInfo;
import com.j256.simplemagic.ContentInfoUtil;
import com.j256.simplemagic.ContentType;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileVisitResult;
Expand Down Expand Up @@ -58,30 +57,23 @@ public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack {
public static void assertConfigSetFolderLegal(Path confPath) throws IOException {
Files.walkFileTree(
confPath,
new SimpleFileVisitor<Path>() {
new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
// Read first 100 bytes of the file to determine the mime type
try (InputStream fileStream = Files.newInputStream(file)) {
byte[] bytes = new byte[100];
fileStream.read(bytes);
if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
String.format(
Locale.ROOT,
"Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s",
file,
FileTypeMagicUtil.INSTANCE.guessMimeType(bytes)));
}
return FileVisitResult.CONTINUE;
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
if (FileTypeMagicUtil.isFileForbiddenInConfigset(file)) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
String.format(
Locale.ROOT,
"Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s",
file,
FileTypeMagicUtil.INSTANCE.guessMimeType(file)));
}
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
if (SKIP_FOLDERS.contains(dir.getFileName().toString()))
return FileVisitResult.SKIP_SUBTREE;

Expand All @@ -90,19 +82,29 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
});
}

/**
* Guess the mime type of file based on its magic number.
*
* @param file file to check
* @return string with content-type or "application/octet-stream" if unknown
*/
public String guessMimeType(Path file) {
try {
return guessTypeFallbackToOctetStream(util.findMatch(file.toFile()));
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}

/**
* Guess the mime type of file based on its magic number.
*
* @param stream input stream of the file
* @return string with content-type or "application/octet-stream" if unknown
*/
public String guessMimeType(InputStream stream) {
String guessMimeType(InputStream stream) {
try {
ContentInfo info = util.findMatch(stream);
if (info == null) {
return ContentType.OTHER.getMimeType();
}
return info.getContentType().getMimeType();
return guessTypeFallbackToOctetStream(util.findMatch(stream));
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
Expand All @@ -115,7 +117,7 @@ public String guessMimeType(InputStream stream) {
* @return string with content-type or "application/octet-stream" if unknown
*/
public String guessMimeType(byte[] bytes) {
return guessMimeType(new ByteArrayInputStream(bytes));
return guessTypeFallbackToOctetStream(util.findMatch(bytes));
}

@Override
Expand All @@ -126,6 +128,31 @@ public void error(String line, String details, Exception e) {
e);
}

/**
* Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types
* are:
*
* <ul>
* <li><code>application/x-java-applet</code>: java class file
* <li><code>application/zip</code>: jar or zip archives
* <li><code>application/x-tar</code>: tar archives
* <li><code>text/x-shellscript</code>: shell or bash script
* </ul>
*
* @param file file to check
* @return true if file is among the forbidden mime-types
*/
public static boolean isFileForbiddenInConfigset(Path file) {
try (InputStream fileStream = Files.newInputStream(file)) {
return isFileForbiddenInConfigset(fileStream);
} catch (IOException e) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
String.format(Locale.ROOT, "Error reading file %s", file),
e);
}
}

/**
* Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types
* are:
Expand All @@ -140,7 +167,7 @@ public void error(String line, String details, Exception e) {
* @param fileStream stream from the file content
* @return true if file is among the forbidden mime-types
*/
public static boolean isFileForbiddenInConfigset(InputStream fileStream) {
static boolean isFileForbiddenInConfigset(InputStream fileStream) {
return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream));
}

Expand All @@ -153,7 +180,7 @@ public static boolean isFileForbiddenInConfigset(InputStream fileStream) {
public static boolean isFileForbiddenInConfigset(byte[] bytes) {
if (bytes == null || bytes.length == 0)
return false; // A ZK znode may be a folder with no content
return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes));
return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(bytes));
}

private static final Set<String> forbiddenTypes =
Expand All @@ -163,4 +190,12 @@ public static boolean isFileForbiddenInConfigset(byte[] bytes) {
"solr.configset.upload.mimetypes.forbidden",
"application/x-java-applet,application/zip,application/x-tar,text/x-shellscript")
.split(",")));

private String guessTypeFallbackToOctetStream(ContentInfo contentInfo) {
if (contentInfo == null) {
return ContentType.OTHER.getMimeType();
} else {
return contentInfo.getContentType().getMimeType();
}
}
}
58 changes: 30 additions & 28 deletions solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,38 +17,40 @@

package org.apache.solr.util;

import java.io.IOException;
import java.io.InputStream;
import org.apache.solr.SolrTestCaseJ4;

public class FileTypeMagicUtilTest extends SolrTestCaseJ4 {
public void testGuessMimeType() {
assertEquals(
"application/x-java-applet",
FileTypeMagicUtil.INSTANCE.guessMimeType(
FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
assertEquals(
"application/zip",
FileTypeMagicUtil.INSTANCE.guessMimeType(
FileTypeMagicUtil.class.getResourceAsStream(
"/runtimecode/containerplugin.v.1.jar.bin")));
assertEquals(
"application/x-tar",
FileTypeMagicUtil.INSTANCE.guessMimeType(
FileTypeMagicUtil.class.getResourceAsStream("/magic/hello.tar.bin")));
assertEquals(
"text/x-shellscript",
FileTypeMagicUtil.INSTANCE.guessMimeType(
FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
public void testGuessMimeType() throws IOException {
assertResourceMimeType("application/x-java-applet", "/magic/HelloWorldJavaClass.class.bin");
assertResourceMimeType("application/zip", "/runtimecode/containerplugin.v.1.jar.bin");
assertResourceMimeType("application/x-tar", "/magic/hello.tar.bin");
assertResourceMimeType("text/x-shellscript", "/magic/shell.sh.txt");
}

public void testIsFileForbiddenInConfigset() {
assertTrue(
FileTypeMagicUtil.isFileForbiddenInConfigset(
FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
assertTrue(
FileTypeMagicUtil.isFileForbiddenInConfigset(
FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
assertFalse(
FileTypeMagicUtil.isFileForbiddenInConfigset(
FileTypeMagicUtil.class.getResourceAsStream("/magic/plain.txt")));
public void testIsFileForbiddenInConfigset() throws IOException {
assertResourceForbiddenInConfigset("/magic/HelloWorldJavaClass.class.bin");
assertResourceForbiddenInConfigset("/magic/shell.sh.txt");
assertResourceAllowedInConfigset("/magic/plain.txt");
}

private void assertResourceMimeType(String mimeType, String resourcePath) throws IOException {
try (InputStream stream =
FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")) {
assertEquals("application/x-java-applet", FileTypeMagicUtil.INSTANCE.guessMimeType(stream));
}
}

private void assertResourceForbiddenInConfigset(String resourcePath) throws IOException {
try (InputStream stream = FileTypeMagicUtil.class.getResourceAsStream(resourcePath)) {
assertTrue(FileTypeMagicUtil.isFileForbiddenInConfigset(stream));
}
}

private void assertResourceAllowedInConfigset(String resourcePath) throws IOException {
try (InputStream stream = FileTypeMagicUtil.class.getResourceAsStream(resourcePath)) {
assertFalse(FileTypeMagicUtil.isFileForbiddenInConfigset(stream));
}
}
}

0 comments on commit c79011e

Please sign in to comment.