From 32e82802d16bfaae84f7621fab147824f11a2d86 Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Wed, 11 Jan 2017 17:09:36 +0100 Subject: [PATCH] Extra Nginx tests. --- .../NginxHttpdLogFormatDissector.java | 15 ++- .../parse/httpdlog/NginxLogFormatTest.java | 93 ++++++++++++++++++- 2 files changed, 100 insertions(+), 8 deletions(-) diff --git a/httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/NginxHttpdLogFormatDissector.java b/httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/NginxHttpdLogFormatDissector.java index 4d99f995..aa8ef097 100644 --- a/httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/NginxHttpdLogFormatDissector.java +++ b/httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/NginxHttpdLogFormatDissector.java @@ -269,7 +269,7 @@ protected List createAllTokenParsers() { // number of bytes sent to a client (1.3.8, 1.2.5) parsers.add(new TokenParser("$bytes_sent", "response.bytes", "BYTES", - Casts.STRING_OR_LONG, FORMAT_CLF_NUMBER)); + Casts.STRING_OR_LONG, FORMAT_NUMBER)); // ------- // $connection @@ -358,6 +358,11 @@ protected List createAllTokenParsers() { "response.body.bytes", "BYTES", Casts.STRING_OR_LONG, FORMAT_NUMBER)); +// // the %b variant +// parsers.add(new TokenParser("$body_bytes_sent", +// "response.body.bytesclf", "BYTES", +// Casts.STRING_OR_LONG, FORMAT_NUMBER)); // FIXME: In case of '0' there should be a null value here. + // ------- // $content_length // “Content-Length” request header field @@ -463,14 +468,16 @@ protected List createAllTokenParsers() { // The PROXY protocol must be previously enabled by setting the proxy_protocol parameter in the listen directive. // TODO: Check if this is correct. Need actually logged example for this. parsers.add(new TokenParser("$proxy_protocol_addr", - "connection.client.proxy.ip", "IP", + "connection.client.proxy.host", "IP", Casts.STRING_OR_LONG, FORMAT_CLF_IP)); + // TODO: Consider renaming 'host' to 'ip' + // ------- // $remote_addr // client address parsers.add(new TokenParser("$remote_addr", - "connection.client.ip", "IP", + "connection.client.host", "IP", Casts.STRING_OR_LONG, FORMAT_CLF_IP)); // ------- @@ -478,7 +485,7 @@ protected List createAllTokenParsers() { // client address in a binary form, value’s length is always 4 bytes String formatHexByte = "\\\\x" + FORMAT_HEXDIGIT + FORMAT_HEXDIGIT; parsers.add(new TokenParser("$binary_remote_addr", - "connection.client.ip", "IP_BINARY", + "connection.client.host", "IP_BINARY", Casts.STRING_OR_LONG, formatHexByte + formatHexByte + formatHexByte + formatHexByte, 0, new BinaryIPDissector())); diff --git a/httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/NginxLogFormatTest.java b/httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/NginxLogFormatTest.java index 2d53e353..6601b50b 100644 --- a/httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/NginxLogFormatTest.java +++ b/httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/NginxLogFormatTest.java @@ -17,16 +17,28 @@ package nl.basjes.parse.httpdlog; +import nl.basjes.parse.core.Parser; +import nl.basjes.parse.core.exceptions.DissectionFailure; +import nl.basjes.parse.core.exceptions.InvalidDissectorException; +import nl.basjes.parse.core.exceptions.MissingDissectorsException; import nl.basjes.parse.core.nl.basjes.parse.core.test.DissectorTester; import nl.basjes.parse.core.nl.basjes.parse.core.test.TestRecord; import org.junit.Ignore; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + public class NginxLogFormatTest { + private static final Logger LOG = LoggerFactory.getLogger(NginxLogFormatTest.class); + @Test public void testBasicLogFormat() { // From: http://articles.slicehost.com/2010/8/27/customizing-nginx-web-logs @@ -42,6 +54,79 @@ public void testBasicLogFormat() { .printAllPossibleValues(); } + @Test + public void testBasicLogFormatDissector() { + // From: http://articles.slicehost.com/2010/8/27/customizing-nginx-web-logs + String logFormat = "combined"; + String logLine = "123.65.150.10 - - [23/Aug/2010:03:50:59 +0000] \"POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1\" 200 2 \"http://www.example.com/wordpress3/wp-admin/post-new.php\" \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3\""; + + HttpdLoglineParser parser = new HttpdLoglineParser<>(TestRecord.class, "$msec"); + parser.dropDissector(HttpdLogFormatDissector.class); + + NginxHttpdLogFormatDissector dissector = new NginxHttpdLogFormatDissector(); + dissector.setLogFormat(logFormat); + parser.addDissector(dissector); + + DissectorTester.create() + .verbose() + .withParser(parser) + .withInput(logLine) + .printPossible() + .printAllPossibleValues(); + } + + @Ignore // FIXME: Unfinished + @Test + public void testCompareApacheAndNginxOutput() throws NoSuchMethodException, InvalidDissectorException, MissingDissectorsException, DissectionFailure { + + String logFormatNginx = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""; + + // combined format except the logname was removed. + String logFormatApache = "%h - %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\""; + + String logLine = "1.2.3.4 - - [23/Aug/2010:03:50:59 +0000] \"POST /foo.html?aap&noot=mies HTTP/1.1\" 200 2 \"http://www.example.com/bar.html?wim&zus=jet\" \"Niels Basjes/1.0\""; + + Parser apacheParser = new HttpdLoglineParser<>(TestRecord.class, logFormatApache); + List fields = apacheParser.getPossiblePaths(); + + ArrayList checkFields = new ArrayList<>(fields.size() + 10); + String[] parameterNames = { "aap", "noot", "mies", "wim", "zus", "jet" }; + for (String field: fields) { + if (field.endsWith(".*")){ + String fieldHead = field.substring(0, field.length()-1); + for (String parameterName: parameterNames){ + checkFields.add(fieldHead + parameterName); + } + } else { + checkFields.add(field); + } + } + + Parser nginxParser = new HttpdLoglineParser<>(TestRecord.class, logFormatNginx); + for (String field: checkFields) { + apacheParser.addParseTarget(TestRecord.class.getMethod("setStringValue", String.class, String.class), field); + nginxParser.addParseTarget(TestRecord.class.getMethod("setStringValue", String.class, String.class), field); + } + + TestRecord apacheRecord = apacheParser.parse(logLine); + TestRecord nginxRecord = nginxParser.parse(logLine); + + for (String field: checkFields) { + boolean apacheHasValue = apacheRecord.hasStringValue(field); + boolean nginxHasValue = nginxRecord.hasStringValue(field); + assertEquals("Apache and Nginx values for field " + field + " are different.", apacheHasValue, nginxHasValue); + + if (apacheRecord.hasStringValue(field)) { + String apacheValue = apacheRecord.getStringValue(field); + String nginxValue = nginxRecord.getStringValue(field); + assertEquals("Apache and Nginx values for field " + field + " are different.", apacheValue, nginxValue); + } + + } + + } + + @Test public void testFullTestAllFields() { String logFormat = @@ -192,9 +277,9 @@ public void validateAllFields() { fieldsTests.add(new SingleFieldTestcase("$time_local" , "03/Jan/2017:15:56:36 +0100" , "TIME.EPOCH:request.receive.time.epoch" , "1483455396000" )); fieldsTests.add(new SingleFieldTestcase("$msec" , "1483455396.639" , "TIME.EPOCH:request.receive.time.epoch" , "1483455396639" )); - fieldsTests.add(new SingleFieldTestcase("$remote_addr" , "127.0.0.1" , "IP:connection.client.ip" , "127.0.0.1" )); - fieldsTests.add(new SingleFieldTestcase("$binary_remote_addr" , "\\x7F\\x00\\x00\\x01" , "IP_BINARY:connection.client.ip" , "\\x7F\\x00\\x00\\x01" )); - fieldsTests.add(new SingleFieldTestcase("$binary_remote_addr" , "\\x7F\\x00\\x00\\x01" , "IP:connection.client.ip" , "127.0.0.1" )); + fieldsTests.add(new SingleFieldTestcase("$remote_addr" , "127.0.0.1" , "IP:connection.client.host" , "127.0.0.1" )); + fieldsTests.add(new SingleFieldTestcase("$binary_remote_addr" , "\\x7F\\x00\\x00\\x01" , "IP_BINARY:connection.client.host" , "\\x7F\\x00\\x00\\x01" )); + fieldsTests.add(new SingleFieldTestcase("$binary_remote_addr" , "\\x7F\\x00\\x00\\x01" , "IP:connection.client.host" , "127.0.0.1" )); fieldsTests.add(new SingleFieldTestcase("$remote_port" , "44448" , "PORT:connection.client.port" , "44448" )); fieldsTests.add(new SingleFieldTestcase("$remote_user" , "-" , "STRING:connection.client.user" , null )); @@ -225,7 +310,7 @@ public void validateAllFields() { fieldsTests.add(new SingleFieldTestcase("$nginx_version" , "1.10.0" , "STRING:server.nginx.version" , "1.10.0" )); fieldsTests.add(new SingleFieldTestcase("$pid" , "5137" , "NUMBER:connection.server.child.processid" , "5137" )); fieldsTests.add(new SingleFieldTestcase("$pipe" , "." , "STRING:connection.nginx.pipe" , "." )); - fieldsTests.add(new SingleFieldTestcase("$proxy_protocol_addr" , "" , "IP:connection.client.proxy.ip" , "" )); + fieldsTests.add(new SingleFieldTestcase("$proxy_protocol_addr" , "" , "IP:connection.client.proxy.host" , "" )); fieldsTests.add(new SingleFieldTestcase("$request" , "GET /?aap&noot=&mies=wim HTTP/1.1" , "HTTP.FIRSTLINE:request.firstline" , "GET /?aap&noot=&mies=wim HTTP/1.1" )); fieldsTests.add(new SingleFieldTestcase("$request_completion" , "OK" , "STRING:request.completion" , "OK" )); fieldsTests.add(new SingleFieldTestcase("$request_filename" , "/var/www/html/index.html" , "FILENAME:server.filename" , "/var/www/html/index.html" ));