Support for msec_frac timeformat

README.md

@@ -29,16 +29,23 @@ Quote from [Apache HTTPD manual](http://httpd.apache.org/docs/current/mod/mod_lo
 It cannot be extracted. A simple workaround for this limitation: replace the **%{...}t** with **%{timestamp}i** .
 You will then get this timestamp field as if it was a request header: HTTP.HEADER:request.header.timestamp
 * **Version 2.6 and newer**: You will receive it as a textual *TIME.LOCALIZEDSTRING:request.header.time* which cannot be extracted any further.
-* **Version 2.9 and newer**: Support for parsing the customized time as long as all elements can be mapped to fields supported by joda-time.
+* **Version 3.0 and newer**: Support for parsing the customized time as long as all elements can be mapped to fields supported by joda-time.
 
 **Limitation**: Only a single %{format}t entry is supported per line. 
-So examples as described in the LogFormat [examples section](http://httpd.apache.org/docs/current/mod/mod_log_config.html#examples) 
+Examples as described in the LogFormat [examples section](http://httpd.apache.org/docs/current/mod/mod_log_config.html#examples) 
 of the Apache HTTPD manual cannot be parsed.
 
     You can use the %{format}t directive multiple times to build up a time format using the extended format tokens like msec_frac:
     Timestamp including milliseconds
              "%{%d/%b/%Y %T}t.%{msec_frac}t %{%z}t"
 
+In this case where all %{format}t fields are only separated by fixed text you can rewrite this example like this
+
+    "%{%d/%b/%Y %T}t.%{msec_frac}t %{%z}t"
+    "%{%d/%b/%Y %T.msec_frac %z}t"
+
+The msec_frac has been added to this parser so the above now works as expected.
+
 
 Analyze almost anything
 ===

httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/ApacheHttpdLogFormatDissector.java

@@ -389,7 +389,7 @@ protected List<TokenParser> createAllTokenParsers() {
 
         parsers.add(new ParameterizedTokenParser("\\%\\{([^\\}]*%[^\\}]*)\\}t",
             "request.receive.time", "TIME.STRFTIME_",
-            Casts.STRING_ONLY, TokenParser.FORMAT_LOCALIZED_TIME,
+            Casts.STRING_ONLY, TokenParser.FORMAT_STRING,
             -1, new StrfTimeStampDissector())
             .setWarningMessageWhenUsed("Only some parts of localized timestamps are supported")
         );
@@ -399,14 +399,14 @@ protected List<TokenParser> createAllTokenParsers() {
 
         parsers.add(new ParameterizedTokenParser("\\%\\{begin:([^\\}]*%[^\\}]*)\\}t",
             "request.receive.time.begin", "TIME.STRFTIME_",
-            Casts.STRING_ONLY, TokenParser.FORMAT_LOCALIZED_TIME,
+            Casts.STRING_ONLY, TokenParser.FORMAT_STRING,
             0, new StrfTimeStampDissector())
             .setWarningMessageWhenUsed("Only some parts of localized timestamps are supported")
         );
 
         parsers.add(new ParameterizedTokenParser("\\%\\{end:([^\\}]*%[^\\}]*)\\}t",
             "request.receive.time.end", "TIME.STRFTIME_",
-            Casts.STRING_ONLY, TokenParser.FORMAT_LOCALIZED_TIME,
+            Casts.STRING_ONLY, TokenParser.FORMAT_STRING,
             0, new StrfTimeStampDissector())
             .setWarningMessageWhenUsed("Only some parts of localized timestamps are supported")
         );
@@ -501,7 +501,7 @@ protected List<TokenParser> createAllTokenParsers() {
         // %{UNIT}T The time taken to serve the request, in a time unit given by UNIT.
         // Valid units are ms for milliseconds, us for microseconds, and s for seconds.
         // Using s gives the same result as %T without any format;
-        // using us gives the same result as %D.
+        // Using us gives the same result as %D.
         parsers.add(new TokenParser("%{us}T",
             "response.server.processing.time", "MICROSECONDS",
             Casts.STRING_OR_LONG, TokenParser.FORMAT_NUMBER));

httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/StrfTimeStampDissector.java

@@ -128,6 +128,12 @@ public void setInputType(String newInputType) {
 
     List<String> convertStrfTimeToTimeFormat(String strftime) throws InvalidDissectorException {
         List<String> resultSet = new ArrayList<>();
+
+        // In some cases these may be still in there
+        strftime = strftime
+            .replaceAll("begin:", "")
+            .replaceAll("end:", "");
+
         resultSet.add(strftime);
 
         // Translating the strftime format into something jodatime should understand.
@@ -164,6 +170,10 @@ List<String> convertStrfTimeToTimeFormat(String strftime) throws InvalidDissecto
         List<String> quotedResultSet = new ArrayList<>();
         for (String result: resultSet) {
             result = result.replaceAll("(%.)", "'$1'");                     // Quote all
+
+            result = result.replaceAll("msec_frac", "'SSS'");    // Apache HTTPD specific: milliseconds fraction
+//            result = result.replaceAll("usec_frac", "'SSSSSSS'");  // Apache HTTPD specific: microseconds fraction
+
             result = result.replaceAll("''", "");                           // Remove the quotes between two adjacent
             result = result.replaceAll("^'(%.)", "$1");                     // Remove the quote at the front IFF first is field
             result = result.replaceAll("(%.)'$", "$1");                     // Remove the quote at the end   IFF last  is field

httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestTimeStampDissector.java

@@ -332,7 +332,6 @@ public void testSpecialTimeLeadingSpaces2() throws Exception {
             .checkExpectations();
     }
 
-    @Ignore
     @Test
     public void testMultipleSpecialTime() throws Exception {
         // As described here: http://httpd.apache.org/docs/current/mod/mod_log_config.html#examples
@@ -341,14 +340,20 @@ public void testMultipleSpecialTime() throws Exception {
         //          "%{%d/%b/%Y %T}t.%{msec_frac}t %{%z}t"
 
         String logline   = "01/Jan/2017 21:52:58.483 +0100";
-        String logformat = "%{%d/%b/%Y %T}t.%{msec_frac}t %{%z}t";
+
+        // The original logformat
+        // String logformat = "%{%d/%b/%Y %T}t.%{msec_frac}t %{%z}t";
+
+        // The transformation
+        // logformat = logformat.replaceAll("\\}t([^%{]+)%\\{","$1");
+
+        // The reformatted result.
+        String logformat = "%{%d/%b/%Y %T.msec_frac %z}t";
 
         DissectorTester.create()
             .withDissector(new HttpdLogFormatDissector(logformat))
             .withInput(logline)
-            .expect("TIME.EPOCH:request.receive.time.epoch", "1483303978000")
-            .expect("TIME.EPOCH:request.receive.time.begin.msec_frac", "483")
-            .expect("TIME.ZONE:request.receive.time.timezone", "+0100")
+            .expect("TIME.EPOCH:request.receive.time.epoch", "1483303978483")
             .checkExpectations();
     }