Skip to content

Commit

Permalink
Extra Nginx tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
@nielsbasjes
nielsbasjes committed Jan 11, 2017
1 parent bf6b390 commit 32e8280
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 8 deletions.
15 changes: 11 additions & 4 deletions .../httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/NginxHttpdLogFormatDissector.java
View file
Expand Up @@ -269,7 +269,7 @@ protected List<TokenParser> createAllTokenParsers() {
// number of bytes sent to a client (1.3.8, 1.2.5)
parsers.add(new TokenParser("$bytes_sent",
"response.bytes", "BYTES",
Casts.STRING_OR_LONG, FORMAT_CLF_NUMBER));
Casts.STRING_OR_LONG, FORMAT_NUMBER));

// -------
// $connection
Expand Down Expand Up @@ -358,6 +358,11 @@ protected List<TokenParser> createAllTokenParsers() {
"response.body.bytes", "BYTES",
Casts.STRING_OR_LONG, FORMAT_NUMBER));

// // the %b variant
// parsers.add(new TokenParser("$body_bytes_sent",
// "response.body.bytesclf", "BYTES",
// Casts.STRING_OR_LONG, FORMAT_NUMBER)); // FIXME: In case of '0' there should be a null value here.

// -------
// $content_length
// “Content-Length” request header field
Expand Down Expand Up @@ -463,22 +468,24 @@ protected List<TokenParser> createAllTokenParsers() {
// The PROXY protocol must be previously enabled by setting the proxy_protocol parameter in the listen directive.
// TODO: Check if this is correct. Need actually logged example for this.
parsers.add(new TokenParser("$proxy_protocol_addr",
"connection.client.proxy.ip", "IP",
"connection.client.proxy.host", "IP",
Casts.STRING_OR_LONG, FORMAT_CLF_IP));

// TODO: Consider renaming 'host' to 'ip'

// -------
// $remote_addr
// client address
parsers.add(new TokenParser("$remote_addr",
"connection.client.ip", "IP",
"connection.client.host", "IP",
Casts.STRING_OR_LONG, FORMAT_CLF_IP));

// -------
// $binary_remote_addr
// client address in a binary form, value’s length is always 4 bytes
String formatHexByte = "\\\\x" + FORMAT_HEXDIGIT + FORMAT_HEXDIGIT;
parsers.add(new TokenParser("$binary_remote_addr",
"connection.client.ip", "IP_BINARY",
"connection.client.host", "IP_BINARY",
Casts.STRING_OR_LONG, formatHexByte + formatHexByte + formatHexByte + formatHexByte,
0, new BinaryIPDissector()));

Expand Down
93 changes: 89 additions & 4 deletions httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/NginxLogFormatTest.java
View file
Expand Up @@ -17,16 +17,28 @@

package nl.basjes.parse.httpdlog;

import nl.basjes.parse.core.Parser;
import nl.basjes.parse.core.exceptions.DissectionFailure;
import nl.basjes.parse.core.exceptions.InvalidDissectorException;
import nl.basjes.parse.core.exceptions.MissingDissectorsException;
import nl.basjes.parse.core.nl.basjes.parse.core.test.DissectorTester;
import nl.basjes.parse.core.nl.basjes.parse.core.test.TestRecord;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;

public class NginxLogFormatTest {

private static final Logger LOG = LoggerFactory.getLogger(NginxLogFormatTest.class);

@Test
public void testBasicLogFormat() {
// From: http://articles.slicehost.com/2010/8/27/customizing-nginx-web-logs
Expand All @@ -42,6 +54,79 @@ public void testBasicLogFormat() {
.printAllPossibleValues();
}

@Test
public void testBasicLogFormatDissector() {
// From: http://articles.slicehost.com/2010/8/27/customizing-nginx-web-logs
String logFormat = "combined";
String logLine = "123.65.150.10 - - [23/Aug/2010:03:50:59 +0000] \"POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1\" 200 2 \"http://www.example.com/wordpress3/wp-admin/post-new.php\" \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3\"";

HttpdLoglineParser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, "$msec");
parser.dropDissector(HttpdLogFormatDissector.class);

NginxHttpdLogFormatDissector dissector = new NginxHttpdLogFormatDissector();
dissector.setLogFormat(logFormat);
parser.addDissector(dissector);

DissectorTester.create()
.verbose()
.withParser(parser)
.withInput(logLine)
.printPossible()
.printAllPossibleValues();
}

@Ignore // FIXME: Unfinished
@Test
public void testCompareApacheAndNginxOutput() throws NoSuchMethodException, InvalidDissectorException, MissingDissectorsException, DissectionFailure {

String logFormatNginx = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\"";

// combined format except the logname was removed.
String logFormatApache = "%h - %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"";

String logLine = "1.2.3.4 - - [23/Aug/2010:03:50:59 +0000] \"POST /foo.html?aap&noot=mies HTTP/1.1\" 200 2 \"http://www.example.com/bar.html?wim&zus=jet\" \"Niels Basjes/1.0\"";

Parser<TestRecord> apacheParser = new HttpdLoglineParser<>(TestRecord.class, logFormatApache);
List<String> fields = apacheParser.getPossiblePaths();

ArrayList<String> checkFields = new ArrayList<>(fields.size() + 10);
String[] parameterNames = { "aap", "noot", "mies", "wim", "zus", "jet" };
for (String field: fields) {
if (field.endsWith(".*")){
String fieldHead = field.substring(0, field.length()-1);
for (String parameterName: parameterNames){
checkFields.add(fieldHead + parameterName);
}
} else {
checkFields.add(field);
}
}

Parser<TestRecord> nginxParser = new HttpdLoglineParser<>(TestRecord.class, logFormatNginx);
for (String field: checkFields) {
apacheParser.addParseTarget(TestRecord.class.getMethod("setStringValue", String.class, String.class), field);
nginxParser.addParseTarget(TestRecord.class.getMethod("setStringValue", String.class, String.class), field);
}

TestRecord apacheRecord = apacheParser.parse(logLine);
TestRecord nginxRecord = nginxParser.parse(logLine);

for (String field: checkFields) {
boolean apacheHasValue = apacheRecord.hasStringValue(field);
boolean nginxHasValue = nginxRecord.hasStringValue(field);
assertEquals("Apache and Nginx values for field " + field + " are different.", apacheHasValue, nginxHasValue);

if (apacheRecord.hasStringValue(field)) {
String apacheValue = apacheRecord.getStringValue(field);
String nginxValue = nginxRecord.getStringValue(field);
assertEquals("Apache and Nginx values for field " + field + " are different.", apacheValue, nginxValue);
}

}

}


@Test
public void testFullTestAllFields() {
String logFormat =
Expand Down Expand Up @@ -192,9 +277,9 @@ public void validateAllFields() {
fieldsTests.add(new SingleFieldTestcase("$time_local" , "03/Jan/2017:15:56:36 +0100" , "TIME.EPOCH:request.receive.time.epoch" , "1483455396000" ));
fieldsTests.add(new SingleFieldTestcase("$msec" , "1483455396.639" , "TIME.EPOCH:request.receive.time.epoch" , "1483455396639" ));

fieldsTests.add(new SingleFieldTestcase("$remote_addr" , "127.0.0.1" , "IP:connection.client.ip" , "127.0.0.1" ));
fieldsTests.add(new SingleFieldTestcase("$binary_remote_addr" , "\\x7F\\x00\\x00\\x01" , "IP_BINARY:connection.client.ip" , "\\x7F\\x00\\x00\\x01" ));
fieldsTests.add(new SingleFieldTestcase("$binary_remote_addr" , "\\x7F\\x00\\x00\\x01" , "IP:connection.client.ip" , "127.0.0.1" ));
fieldsTests.add(new SingleFieldTestcase("$remote_addr" , "127.0.0.1" , "IP:connection.client.host" , "127.0.0.1" ));
fieldsTests.add(new SingleFieldTestcase("$binary_remote_addr" , "\\x7F\\x00\\x00\\x01" , "IP_BINARY:connection.client.host" , "\\x7F\\x00\\x00\\x01" ));
fieldsTests.add(new SingleFieldTestcase("$binary_remote_addr" , "\\x7F\\x00\\x00\\x01" , "IP:connection.client.host" , "127.0.0.1" ));

fieldsTests.add(new SingleFieldTestcase("$remote_port" , "44448" , "PORT:connection.client.port" , "44448" ));
fieldsTests.add(new SingleFieldTestcase("$remote_user" , "-" , "STRING:connection.client.user" , null ));
Expand Down Expand Up @@ -225,7 +310,7 @@ public void validateAllFields() {
fieldsTests.add(new SingleFieldTestcase("$nginx_version" , "1.10.0" , "STRING:server.nginx.version" , "1.10.0" ));
fieldsTests.add(new SingleFieldTestcase("$pid" , "5137" , "NUMBER:connection.server.child.processid" , "5137" ));
fieldsTests.add(new SingleFieldTestcase("$pipe" , "." , "STRING:connection.nginx.pipe" , "." ));
fieldsTests.add(new SingleFieldTestcase("$proxy_protocol_addr" , "" , "IP:connection.client.proxy.ip" , "" ));
fieldsTests.add(new SingleFieldTestcase("$proxy_protocol_addr" , "" , "IP:connection.client.proxy.host" , "" ));
fieldsTests.add(new SingleFieldTestcase("$request" , "GET /?aap&noot=&mies=wim HTTP/1.1" , "HTTP.FIRSTLINE:request.firstline" , "GET /?aap&noot=&mies=wim HTTP/1.1" ));
fieldsTests.add(new SingleFieldTestcase("$request_completion" , "OK" , "STRING:request.completion" , "OK" ));
fieldsTests.add(new SingleFieldTestcase("$request_filename" , "/var/www/html/index.html" , "FILENAME:server.filename" , "/var/www/html/index.html" ));
Expand Down

0 comments on commit 32e8280

Please sign in to comment.