Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
118 lines (107 sloc) 5.97 KB
;;;; parse.lisp
;; Time-stamp: <2017-02-14 14:31:35>
;; Copyright (C) 2017 Pierre Lecocq
(defvar *log-parser-data* '((remote-addr . '((index . 0)
(eq-func . string=)
(regex . "(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})+")
(label . "Remote address")))
(time-local . '((index . 3)
(eq-func . string=)
(regex . "(\\[([^\\]]+)\\])+")
(label . "Date")))
(request . '((index . 5)
(eq-func . string=)
(regex . "(\"([^\"]+)\")+")
(label . "Request")))
(status . '((index . 6)
(eq-func . eq)
(regex . "(\\d+)")
(label . "HTTP status")))
(http-user-agent . '((index . 11)
(eq-func . string=)
(regex . "(\"([^\"]+)\")+")
(label . "User agent")))
(request-path . '((index . nil)
(eq-func . string=)
(regex . nil)
(label . "Path")))
(request-verb . '((index . 11)
(eq-func . string=)
(regex . nil)
(label . "HTTP verb")))))
(defun log-fmt-value-by-metric (metric key)
(cdr (assoc key (caddr (assoc metric *log-parser-data*)))))
(defvar *log-fmt-regex* (concatenate 'string
"(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})+" ; 0
" "
"(-\\s-)+" ; 1
" "
"(\\[([^\\]]+)\\])+" ; 2 & 3
" "
"(\"([^\"]+)\")+" ; 4 & 5
" "
"(\\d+)" ; 6
" "
"(\\d+)" ; 7
" "
"(\"([^\"]+)\")+" ; 8 & 9
" "
"(\"([^\"]+)\")+" ; 10 & 11
))
(defun short-month-to-integer (short-month)
(cdr (assoc short-month '(("Jan" . 1)
("Feb" . 2)
("Mar" . 3)
("Apr" . 4)
("May" . 5)
("Jun" . 6)
("Jul" . 7)
("Aug" . 8)
("Sept" . 9)
("Oct" . 10)
("Nov" . 11)
("Dec" . 12)) :test #'string=)))
(defun log-date-to-timestamp (raw-date)
(let ((chunks (ppcre:split "\\s+" (substitute #\Space #\/ (substitute #\Space #\: raw-date)))))
(local-time:encode-timestamp 0
(parse-integer (nth 5 chunks))
(parse-integer (nth 4 chunks))
(parse-integer (nth 3 chunks))
(parse-integer (nth 0 chunks))
(short-month-to-integer (nth 1 chunks))
(parse-integer (nth 2 chunks)))))
(defun out-of-filters-bounds (data filters)
(when filters
(let ((date (log-date-to-timestamp (aref data (log-fmt-value-by-metric 'time-local 'index))))
(start (cdr (assoc 'start filters)))
(end (cdr (assoc 'end filters))))
(or (and start (local-time:timestamp< date start))
(and end (local-time:timestamp> date end))))))
(defun entry-from-line (line filters)
(let ((scanner (ppcre:create-scanner *log-fmt-regex*)))
(multiple-value-bind (matched data) (ppcre:scan-to-strings scanner line)
(when matched
(let ((request (aref data (log-fmt-value-by-metric 'request 'index))))
(if (out-of-filters-bounds data filters)
nil
(make-instance 'entry
:remote-addr (aref data (log-fmt-value-by-metric 'remote-addr 'index))
:time-local (aref data (log-fmt-value-by-metric 'time-local 'index))
:request request
:status (parse-integer (aref data (log-fmt-value-by-metric 'status 'index)))
:http-user-agent (aref data (log-fmt-value-by-metric 'http-user-agent 'index)))))))))
(defun entries-from-file (path filters)
(with-open-file (stream path)
(loop for line = (read-line stream nil)
while line
when (entry-from-line line filters)
collect it)))
(defun group-by-property (entries property test)
(let ((groups '()))
(mapcar #'(lambda (req)
(when req
(if (assoc (slot-value req property) groups :test test)
(setf (cdr (assoc (slot-value req property) groups :test test))
(+ 1 (cdr (assoc (slot-value req property) groups :test test))))
(setq groups (acons (slot-value req property) 1 groups))))) entries)
(sort groups #'> :key #'cdr)))