Skip to content

Commit

Permalink
Implemented a few more Nginx fields
Browse files Browse the repository at this point in the history
  • Loading branch information
@nielsbasjes
nielsbasjes committed Jan 6, 2017
1 parent a434666 commit 4844c7a
Show file tree
Hide file tree
Showing 8 changed files with 120 additions and 62 deletions.
8 changes: 4 additions & 4 deletions ...httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/ApacheHttpdLogFormatDissector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,18 +198,18 @@ protected List<TokenParser> createAllTokenParsers() {
// %a Remote IP-address
parsers.add(new TokenParser("%a",
"connection.client.ip", "IP",
Casts.STRING_OR_LONG, TokenParser.FORMAT_CLF_IP));
Casts.STRING_ONLY, TokenParser.FORMAT_CLF_IP));

// %{c}a Underlying peer IP address of the connection (see the mod_remoteip module).
parsers.add(new TokenParser("%{c}a",
"connection.client.peerip", "IP",
Casts.STRING_OR_LONG, TokenParser.FORMAT_CLF_IP));
Casts.STRING_ONLY, TokenParser.FORMAT_CLF_IP));

// -------
// %A Local IP-address
parsers.add(new TokenParser("%A",
"connection.server.ip", "IP",
Casts.STRING_OR_LONG, TokenParser.FORMAT_CLF_IP));
Casts.STRING_ONLY, TokenParser.FORMAT_CLF_IP));

// -------
// %B Size of response in bytes, excluding HTTP headers.
Expand Down Expand Up @@ -521,7 +521,7 @@ protected List<TokenParser> createAllTokenParsers() {
// -------
// %U The URL path requested, not including any query string.
parsers.add(new TokenParser("%U",
"request.urlpath", "URI",
"request.urlpath", "URI", // FIXME: This is wrong
Casts.STRING_ONLY, TokenParser.FORMAT_NO_SPACE_STRING));

// -------
Expand Down
95 changes: 67 additions & 28 deletions .../httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/NginxHttpdLogFormatDissector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import nl.basjes.parse.core.Casts;
import nl.basjes.parse.core.Parsable;
import nl.basjes.parse.core.ParsedField;
import nl.basjes.parse.core.SimpleDissector;
import nl.basjes.parse.core.exceptions.DissectionFailure;
import nl.basjes.parse.httpdlog.dissectors.tokenformat.NamedTokenParser;
Expand Down Expand Up @@ -144,12 +143,12 @@ protected List<TokenParser> createAllTokenParsers() {
// -------
// $connection
// connection serial number (1.3.8, 1.2.5)
parsers.add(new IgnoreUnknownTokenParser("$connection", -1)); // TODO: Implement $connection token
parsers.add(new NotYetImplemented("$connection", -1)); // TODO: Implement $connection token

// -------
// $connection_requests
// current number of requests made through a connection (1.3.8, 1.2.5)
parsers.add(new IgnoreUnknownTokenParser("$connection_requests")); // TODO: Implement $connection_requests token
parsers.add(new NotYetImplemented("$connection_requests")); // TODO: Implement $connection_requests token

// -------
// $msec
Expand Down Expand Up @@ -240,7 +239,7 @@ protected List<TokenParser> createAllTokenParsers() {

// -------
// $document_root
parsers.add(new IgnoreUnknownTokenParser("$document_root")); // TODO: Implement $document_root token
parsers.add(new NotYetImplemented("$document_root")); // TODO: Implement $document_root token
// root or alias directive’s value for the current request

// -------
Expand Down Expand Up @@ -277,17 +276,17 @@ protected List<TokenParser> createAllTokenParsers() {
// -------
// $https
// “on” if connection operates in SSL mode, or an empty string otherwise
parsers.add(new IgnoreUnknownTokenParser("$https")); // TODO: Implement $https token
parsers.add(new NotYetImplemented("$https")); // TODO: Implement $https token

// -------
// $is_args
// “?” if a request line has arguments, or an empty string otherwise
parsers.add(new IgnoreUnknownTokenParser("$is_args")); // TODO: Implement $is_args token
parsers.add(new NotYetImplemented("$is_args")); // TODO: Implement $is_args token

// -------
// $limit_rate
// setting this variable enables response rate limiting; see limit_rate
parsers.add(new IgnoreUnknownTokenParser("$limit_rate")); // TODO: Implement $limit_rate token
parsers.add(new NotYetImplemented("$limit_rate")); // TODO: Implement $limit_rate token

// -------
// $nginx_version
Expand All @@ -299,24 +298,26 @@ protected List<TokenParser> createAllTokenParsers() {
// -------
// $pid
// PID of the worker process
parsers.add(new IgnoreUnknownTokenParser("$pid")); // TODO: Implement $pid token
parsers.add(new TokenParser("$pid",
"connection.server.child.processid", "NUMBER",
Casts.STRING_OR_LONG, TokenParser.FORMAT_NUMBER));

// -------
// $pipe
// “p” if request was pipelined, “.” otherwise (1.3.12, 1.2.7)
parsers.add(new IgnoreUnknownTokenParser("$pipe")); // TODO: Implement $pipe token
parsers.add(new NotYetImplemented("$pipe")); // TODO: Implement $pipe token

// -------
// $proxy_protocol_addr
// client address from the PROXY protocol header, or an empty string otherwise (1.5.12)
// The PROXY protocol must be previously enabled by setting the proxy_protocol parameter in the listen directive.
parsers.add(new IgnoreUnknownTokenParser("$proxy_protocol_addr")); // TODO: Implement $proxy_protocol_addr token
parsers.add(new NotYetImplemented("$proxy_protocol_addr")); // TODO: Implement $proxy_protocol_addr token

// -------
// $realpath_root
// an absolute pathname corresponding to the root or alias directive’s value for the current request,
// with all symbolic links resolved to real paths
parsers.add(new IgnoreUnknownTokenParser("$realpath_root")); // TODO: Implement $realpath_root token
parsers.add(new NotYetImplemented("$realpath_root")); // TODO: Implement $realpath_root token

// -------
// $remote_addr
Expand All @@ -331,7 +332,8 @@ protected List<TokenParser> createAllTokenParsers() {
String formatHexByte = "\\\\x" + FORMAT_HEXDIGIT + FORMAT_HEXDIGIT;
parsers.add(new TokenParser("$binary_remote_addr",
"connection.client.ip", "IP_BINARY",
Casts.STRING_OR_LONG, formatHexByte + formatHexByte + formatHexByte + formatHexByte));
Casts.STRING_OR_LONG, formatHexByte + formatHexByte + formatHexByte + formatHexByte,
0, new BinaryIPDissector()));

// -------
// $remote_port
Expand Down Expand Up @@ -362,7 +364,7 @@ protected List<TokenParser> createAllTokenParsers() {
// $request_body
// request body
// The variable’s value is made available in locations processed by the proxy_pass, fastcgi_pass, uwsgi_pass, and scgi_pass directives.
parsers.add(new IgnoreUnknownTokenParser("$request_body", -1)); // TODO: Implement $request_body token
parsers.add(new NotYetImplemented("$request_body", -1)); // TODO: Implement $request_body token

// -------
// $request_body_file
Expand All @@ -372,12 +374,12 @@ protected List<TokenParser> createAllTokenParsers() {
// or in a request to a FastCGI/uwsgi/SCGI server, passing the request body should be disabled by the
// proxy_pass_request_body off, fastcgi_pass_request_body off, uwsgi_pass_request_body off, or
// scgi_pass_request_body off directives, respectively.
parsers.add(new IgnoreUnknownTokenParser("$request_body_file")); // TODO: Implement $request_body_file token
parsers.add(new NotYetImplemented("$request_body_file")); // TODO: Implement $request_body_file token

// -------
// $request_completion
// “OK” if a request has completed, or an empty string otherwise
parsers.add(new IgnoreUnknownTokenParser("$request_completion")); // TODO: Implement $request_completion token
parsers.add(new NotYetImplemented("$request_completion")); // TODO: Implement $request_completion token

// -------
// $request_filename
Expand All @@ -404,7 +406,7 @@ protected List<TokenParser> createAllTokenParsers() {
// $request_time
// request processing time in seconds with a milliseconds resolution (1.3.9, 1.2.6);
// time elapsed since the first bytes were read from the client
parsers.add(new IgnoreUnknownTokenParser("$request_time")); // TODO: Implement $request_time token
parsers.add(new NotYetImplemented("$request_time")); // TODO: Implement $request_time token

// -------
// $request_uri
Expand Down Expand Up @@ -462,23 +464,23 @@ protected List<TokenParser> createAllTokenParsers() {
// $tcpinfo_rtt, $tcpinfo_rttvar, $tcpinfo_snd_cwnd, $tcpinfo_rcv_space
// information about the client TCP connection; available on systems that support the TCP_INFO socket option
// $tcpinfo_rtt
parsers.add(new IgnoreUnknownTokenParser("$tcpinfo_rtt", -1)); // TODO: Implement $tcpinfo_rtt token
parsers.add(new NotYetImplemented("$tcpinfo_rtt", -1)); // TODO: Implement $tcpinfo_rtt token
// $tcpinfo_rttvar
parsers.add(new IgnoreUnknownTokenParser("$tcpinfo_rttvar")); // TODO: Implement $tcpinfo_rttvar token
parsers.add(new NotYetImplemented("$tcpinfo_rttvar")); // TODO: Implement $tcpinfo_rttvar token
// $tcpinfo_snd_cwnd
parsers.add(new IgnoreUnknownTokenParser("$tcpinfo_snd_cwnd")); // TODO: Implement $tcpinfo_snd_cwnd token
parsers.add(new NotYetImplemented("$tcpinfo_snd_cwnd")); // TODO: Implement $tcpinfo_snd_cwnd token
// $tcpinfo_rcv_space
parsers.add(new IgnoreUnknownTokenParser("$tcpinfo_rcv_space")); // TODO: Implement $tcpinfo_rcv_space token
parsers.add(new NotYetImplemented("$tcpinfo_rcv_space")); // TODO: Implement $tcpinfo_rcv_space token


// -------
// $uri
parsers.add(new IgnoreUnknownTokenParser("$uri")); // TODO: Implement $uri token
parsers.add(new NotYetImplemented("$uri")); // TODO: Implement $uri token
// current URI in request, normalized
// The value of $uri may change during request processing, e.g. when doing internal redirects, or when using index files.
// -------
// $document_uri
parsers.add(new IgnoreUnknownTokenParser("$document_uri")); // TODO: Implement $document_uri token
parsers.add(new NotYetImplemented("$document_uri")); // TODO: Implement $document_uri token
// same as $uri

// parsers.add(new TokenParser("%r",
Expand Down Expand Up @@ -518,12 +520,7 @@ public EpochSecondsWithMillisDissector() {
}

@Override
public void dissect(Parsable<?> parsable, String inputname) throws DissectionFailure {
final ParsedField field = parsable.getParsableField(getInputType(), inputname);
String fieldValue = field.getValue().getString();
if (fieldValue == null || fieldValue.isEmpty()) {
return; // Nothing to do here
}
public void dissect(Parsable<?> parsable, String inputname, String fieldValue) throws DissectionFailure {
String[] epochStrings = fieldValue.split("\\.", 2);
Long seconds = Long.parseLong(epochStrings[0]);
Long milliseconds = Long.parseLong(epochStrings[1]);
Expand All @@ -533,4 +530,46 @@ public void dissect(Parsable<?> parsable, String inputname) throws DissectionFai
}
}

public static class BinaryIPDissector extends SimpleDissector {

private static HashMap<String, EnumSet<Casts>> epochMillisConfig = new HashMap<>();
static {
epochMillisConfig.put("IP:", Casts.STRING_OR_LONG);
}
public BinaryIPDissector() {
super("IP_BINARY", epochMillisConfig);
}

private static final String CAPTURE_HEX_BYTE = "\\\\x([0-9a-fA-F][0-9a-fA-F])";
Pattern binaryIPPattern = Pattern.compile(
CAPTURE_HEX_BYTE+CAPTURE_HEX_BYTE+CAPTURE_HEX_BYTE+CAPTURE_HEX_BYTE
);

@Override
public void dissect(Parsable<?> parsable, String inputname, String fieldValue) throws DissectionFailure {
Matcher matcher = binaryIPPattern.matcher(fieldValue);
if (matcher.matches()) {
String[] binaryIPParts = fieldValue.split("\\.");
String ip =
String.valueOf(Utils.hexCharsToByte(matcher.group(1))) + '.' +
String.valueOf(Utils.hexCharsToByte(matcher.group(2))) + '.' +
String.valueOf(Utils.hexCharsToByte(matcher.group(3))) + '.' +
String.valueOf(Utils.hexCharsToByte(matcher.group(4)));
parsable.addDissection(inputname, "IP", "", ip);
}
}
}

@Deprecated
public static class NotYetImplemented extends NotYetImplementedTokenParser {
private static final String FIELD_PREFIX = "nginx_parameter";
public NotYetImplemented(final String nLogFormatToken) {
super(nLogFormatToken, FIELD_PREFIX, 0);
}

public NotYetImplemented(final String nLogFormatToken, final int prio) {
super(nLogFormatToken, FIELD_PREFIX, prio);
}
}

}
7 changes: 7 additions & 0 deletions httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/Utils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,13 @@ public static String resilientUrlDecode(String input) {
}


public static byte hexCharsToByte(String twoHexDigits) {
if (twoHexDigits == null || twoHexDigits.length() != 2) {
throw new IllegalArgumentException("URLDecoder: Illegal hex characters : \"" + twoHexDigits + "\"");
}
return hexCharsToByte(twoHexDigits.charAt(0), twoHexDigits.charAt(1));
}

public static byte hexCharsToByte(char c1, char c2){
byte result;
switch (c1) {
Expand Down
12 changes: 6 additions & 6 deletions ...r/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/TokenFormatDissector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,15 @@ public FixedStringTokenParser(final String nLogFormatToken) {
}

// --------------------------------------------
public static class IgnoreUnknownTokenParser extends TokenParser {
public IgnoreUnknownTokenParser(final String nLogFormatToken) {
this(nLogFormatToken, 0);
public static class NotYetImplementedTokenParser extends TokenParser {
public NotYetImplementedTokenParser(final String nLogFormatToken, final String fieldPrefix) {
this(nLogFormatToken, fieldPrefix, 0);
}

public IgnoreUnknownTokenParser(final String nLogFormatToken, int nPrio) {
public NotYetImplementedTokenParser(final String nLogFormatToken, final String fieldPrefix, int nPrio) {
super(nLogFormatToken,
"ignored_" + nLogFormatToken.toLowerCase(Locale.ENGLISH).replaceAll("[^a-z0-9_]", "_"),
"IGNORED",
fieldPrefix + "_" + nLogFormatToken.toLowerCase(Locale.ENGLISH).replaceAll("[^a-z0-9_]", "_"),
"NOT_YET_IMPLEMENTED",
Casts.STRING_ONLY,
"[^\"]*",
nPrio);
Expand Down
Loading

0 comments on commit 4844c7a

Please sign in to comment.