Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix and add more logging to MessageUtils #5259

Merged
merged 4 commits into from
Aug 23, 2023
Merged

Fix and add more logging to MessageUtils #5259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
dcae9cd
Fix and add more logging
nielsm5 Aug 16, 2023
0c4ce1f
Reduce log messages
nielsm5 Aug 23, 2023
fdc70ad
Update core/src/main/java/nl/nn/adapterframework/util/MessageUtils.java
nielsm5 Aug 23, 2023
f8167db
Remove useless code
nielsm5 Aug 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/src/main/java/nl/nn/adapterframework/core/PipeLineSession.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ public void close() {
Entry<AutoCloseable, String> entry = it.next();
AutoCloseable closeable = entry.getKey();
try {
log.warn("messageId ["+getMessageId()+"] auto closing resource "+entry.getValue());
log.info("messageId ["+getMessageId()+"] auto closing resource "+entry.getValue());
closeable.close();
} catch (Exception e) {
log.warn("Exception closing resource", e);
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/nl/nn/adapterframework/scheduler/job/LoadDatabaseSchedulesJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public void execute() {
try {
adapter = findAdapter(adapterName);
} catch (IllegalStateException e) {
getMessageKeeper().add("unable to add schedule ["+key+"]", e);
getMessageKeeper().add("unable to add schedule ["+key+"]: " + e.getMessage());
continue;
}

Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/nl/nn/adapterframework/util/MessageDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public MessageDataSource(Message message) throws IOException {
}

public MessageDataSource(Message message, String newContentType) throws IOException {
if(message.isNull()) {
if(Message.isNull(message)) {
throw new IllegalArgumentException("message may not be null");
}
if(message.getContext() == null) {
Expand Down
41 changes: 22 additions & 19 deletions core/src/main/java/nl/nn/adapterframework/util/MessageUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@

public abstract class MessageUtils {
private static final Logger LOG = LogUtil.getLogger(MessageUtils.class);
private static int charsetConfidenceLevel = AppConstants.getInstance().getInt("charset.confidenceLevel", 65);
private static final int CHARSET_CONFIDENCE_LEVEL = AppConstants.getInstance().getInt("charset.confidenceLevel", 65);

/**
* Fetch metadata from the {@link HttpServletRequest} such as Content-Length, Content-Type (mimetype + charset)
Expand All @@ -56,7 +56,10 @@ public static MessageContext getContext(HttpServletRequest request) {
result.withCharset(request.getCharacterEncoding());
int contentLength = request.getContentLength();
result.withSize(contentLength);
result.withMimeType(request.getContentType());
String contentType = request.getContentType();
if(StringUtils.isNotEmpty(contentType)) {
result.withMimeType(contentType);
}

Enumeration<String> names = request.getHeaderNames();
while(names.hasMoreElements()) {
Expand All @@ -72,14 +75,7 @@ public static MessageContext getContext(Iterator<MimeHeader> mimeHeaders) {
while (mimeHeaders.hasNext()) {
MimeHeader header = mimeHeaders.next();
String name = header.getName();
if("Content-Transfer-Encoding".equals(name)) {
try {
Charset charset = Charset.forName(header.getValue());
result.withCharset(charset);
} catch (Exception e) {
LOG.warn("Could not determine charset", e);
}
} else if("Content-Type".equals(name)) {
if("Content-Type".equals(name)) {
result.withMimeType(header.getValue());
} else {
result.put(MessageContext.HEADER_PREFIX + name, header.getValue());
Expand All @@ -98,6 +94,7 @@ public static Message parseContentAsMessage(HttpServletRequest request) throws I
if(request.getContentLength() > -1 || request.getHeader("transfer-encoding") != null) {
return new Message(request.getInputStream(), getContext(request));
} else {
// We want the context because of the request headers
return Message.nullMessage(getContext(request));
}
}
Expand All @@ -112,7 +109,7 @@ public static Message parse(AttachmentPart soapAttachment) throws SOAPException
* @throws IOException when it cannot read the first 10k bytes.
*/
public static Charset computeDecodingCharset(Message message) throws IOException {
return computeDecodingCharset(message, charsetConfidenceLevel);
return computeDecodingCharset(message, CHARSET_CONFIDENCE_LEVEL);
}

/**
Expand Down Expand Up @@ -177,13 +174,16 @@ public static MimeType getMimeType(Message message) {

MimeType mimeType = (MimeType)message.getContext().get(MessageContext.METADATA_MIMETYPE);
if(mimeType == null) {
LOG.trace("no mimetype found in MessageContext");
return null;
}

if(message.getCharset() != null) { //and is character data?
LOG.trace("found mimetype [{}] in MessageContext with charset [{}]", ()->mimeType, message::getCharset);
return new MimeType(mimeType, Charset.forName(message.getCharset()));
}

LOG.trace("found mimetype [{}] in MessageContext without charset", mimeType);
return mimeType;
}

Expand Down Expand Up @@ -213,13 +213,15 @@ public static MimeType computeMimeType(Message message, String filename) {
}

Map<String, Object> context = message.getContext();
MimeType mimeType = getMimeType(message);
if(mimeType != null) {
return mimeType;
MimeType contextMimeType = getMimeType(message);
if(contextMimeType != null) {
LOG.debug("returning predetermined mimetype [{}]", contextMimeType);
return contextMimeType;
}

String name = (String) context.get(MessageContext.METADATA_NAME);
if(StringUtils.isNotEmpty(filename)) {
LOG.trace("using filename from MessageContext [{}]", name);
name = filename;
}

Expand All @@ -229,25 +231,26 @@ public static MimeType computeMimeType(Message message, String filename) {
metadata.set(TikaMetadataKeys.RESOURCE_NAME_KEY, name);
int tikaMimeMagicLength = tika.getMimeRepository().getMinLength();
byte[] magic = message.getMagic(tikaMimeMagicLength);
if(magic == null || magic.length == 0) {
if(magic.length == 0) {
return null;
}
org.apache.tika.mime.MediaType tikaMediaType = tika.getDetector().detect(new ByteArrayInputStream(magic), metadata);
mimeType = MimeType.valueOf(tikaMediaType.toString());
MimeType mimeType = MimeType.valueOf(tikaMediaType.toString());
context.put(MessageContext.METADATA_MIMETYPE, mimeType);
if("text".equals(mimeType.getType()) || message.getCharset() != null) { // is of type 'text' or message has charset
Charset charset = computeDecodingCharset(message);
if(charset != null) {
LOG.debug("found mimetype [{}] with charset [{}]", mimeType, charset);
return new MimeType(mimeType, charset);
}
}

LOG.debug("found mimetype [{}]", mimeType);
return mimeType;
} catch (Exception t) {
LOG.warn("error parsing message to determine mimetype", t);
return null;
}

LOG.info("unable to determine mimetype");
return null;
}

/**
Expand Down