diff --git a/config/go.d/web_log.conf b/config/go.d/web_log.conf index 5fedbf7a1..bfa640adb 100644 --- a/config/go.d/web_log.conf +++ b/config/go.d/web_log.conf @@ -218,3 +218,11 @@ jobs: - name: gunicorn path: /var/log/gunicorn/gunicorn-access.log + + # IIS + # This configuration assumes you are running netdata on WSL + - name: iis + path: /mnt/c/inetpub/logs/LogFiles/W3SVC1/u_ex*.log + log_type: csv + csv_config: + format: '- - $host $request_method $request_uri - $server_port - $remote_addr - - $status - - $request_time' diff --git a/modules/weblog/README.md b/modules/weblog/README.md index e6727884c..629b85e73 100644 --- a/modules/weblog/README.md +++ b/modules/weblog/README.md @@ -2,12 +2,12 @@ title: "Web server log (Apache, NGINX) monitoring with Netdata" description: "Monitor the health and performance of Apache or Nginx logs with zero configuration, per-second metric granularity, and interactive visualizations." custom_edit_url: https://github.com/netdata/go.d.plugin/edit/master/modules/weblog/README.md -sidebar_label: "Web server logs (Apache, NGINX)" +sidebar_label: "Web server logs (Apache, NGINX, Microsoft IIS)" --> -# Web log (Apache, NGINX) monitoring with Netdata +# Web log (Apache, NGINX, IIS) monitoring with Netdata -This module parses [`Apache`](https://httpd.apache.org/) and [`NGINX`](https://nginx.org/en/) web servers logs. +This module parses [`Apache`](https://httpd.apache.org/), [`NGINX`](https://nginx.org/en/) and [Microsoft IIS](https://www.iis.net/) web servers logs. ## Metrics @@ -316,6 +316,12 @@ jobs: log_type: csv csv_config: format: '- - %h - - %t \"%r\" %>s %b' + + - name: iis + path: /mnt/c/inetpub/logs/LogFiles/W3SVC1/u_ex*.log + log_type: csv + csv_config: + format: '- - $host $request_method $request_uri - $server_port - $remote_addr - - $status - - $request_time' ``` For all available options, please see the diff --git a/modules/weblog/logline.go b/modules/weblog/logline.go index 6b3197c9d..c433dcdc7 100644 --- a/modules/weblog/logline.go +++ b/modules/weblog/logline.go @@ -24,6 +24,7 @@ import ( // Variables: // - nginx: http://nginx.org/en/docs/varindex.html // - apache: http://httpd.apache.org/docs/current/mod/mod_log_config.html#logformat +// - IIS: https://learn.microsoft.com/en-us/windows/win32/http/w3c-logging /* | nginx | apache | description | @@ -607,6 +608,7 @@ func isSSLProtoValid(proto string) bool { } func timeMultiplier(time string) float64 { + // TODO: Change code to detect and modify properly IIS time (in milliseconds) // Convert to microseconds: // - nginx time is in seconds with a milliseconds' resolution. if strings.IndexByte(time, '.') > 0 { diff --git a/modules/weblog/testdata/u_ex221107.log b/modules/weblog/testdata/u_ex221107.log new file mode 100644 index 000000000..5c934fef8 --- /dev/null +++ b/modules/weblog/testdata/u_ex221107.log @@ -0,0 +1,168 @@ +#Software: Microsoft Internet Information Services 10.0 +#Version: 1.0 +#Date: 2022-11-07 14:29:06 +#Fields: date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) cs(Referer) sc-status sc-substatus sc-win32-status time-taken +2022-11-07 14:29:06 127.0.0.1 GET /us - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 130 +2022-11-07 14:29:06 127.0.0.1 GET /us - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 1 +2022-11-07 14:29:08 127.0.0.1 GET /status full&json 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:29:08 127.0.0.1 GET /status full&json 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:29:08 ::1 GET /status full&json 80 - ::1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:29:09 127.0.0.1 GET /server-status auto 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:29:09 127.0.0.1 GET /server-status auto 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +#Software: Microsoft Internet Information Services 10.0 +#Version: 1.0 +#Date: 2022-11-07 14:55:17 +#Fields: date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) cs(Referer) sc-status sc-substatus sc-win32-status time-taken +2022-11-07 14:55:17 127.0.0.1 GET /us - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 187 +2022-11-07 14:55:17 127.0.0.1 GET /us - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:17 127.0.0.1 GET /server-status format=plain 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:17 127.0.0.1 GET /server-status format=plain 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 1 +2022-11-07 14:55:18 127.0.0.1 GET /basic_status - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 1 +2022-11-07 14:55:18 127.0.0.1 GET /stub_status - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:18 127.0.0.1 GET /stub_status - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 1 +2022-11-07 14:55:18 127.0.0.1 GET /nginx_status - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:18 127.0.0.1 GET /status - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:18 127.0.0.1 GET /status/format/json - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:20 127.0.0.1 GET /admin/api.php version=true 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:20 127.0.0.1 GET /admin/api.php version=true 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:20 127.0.0.1 GET /server-status auto 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:20 127.0.0.1 GET /server-status auto 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:24 127.0.0.1 GET /status full&json 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:24 127.0.0.1 GET /status full&json 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 14:55:24 ::1 GET /status full&json 80 - ::1 Go-http-client/1.1 - 404 0 2 0 +#Software: Microsoft Internet Information Services 10.0 +#Version: 1.0 +#Date: 2022-11-07 15:42:39 +#Fields: date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) cs(Referer) sc-status sc-substatus sc-win32-status time-taken +2022-11-07 15:42:39 127.0.0.1 GET /server-status format=plain 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 149 +2022-11-07 15:42:39 127.0.0.1 GET /server-status format=plain 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:39 127.0.0.1 GET /server-status auto 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:39 127.0.0.1 GET /server-status auto 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:39 127.0.0.1 GET /status/format/json - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:41 127.0.0.1 GET /basic_status - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:41 127.0.0.1 GET /stub_status - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:41 127.0.0.1 GET /stub_status - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:41 127.0.0.1 GET /nginx_status - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:41 127.0.0.1 GET /status - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:41 127.0.0.1 GET /admin/api.php version=true 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:41 127.0.0.1 GET /admin/api.php version=true 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:41 127.0.0.1 GET /us - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:41 127.0.0.1 GET /us - 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:46 127.0.0.1 GET /status full&json 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:46 127.0.0.1 GET /status full&json 80 - 127.0.0.1 Go-http-client/1.1 - 404 0 2 0 +2022-11-07 15:42:46 ::1 GET /status full&json 80 - ::1 Go-http-client/1.1 - 404 0 2 0 +#Software: Microsoft Internet Information Services 10.0 +#Version: 1.0 +#Date: 2022-11-07 16:47:25 +#Fields: date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) cs(Referer) sc-status sc-substatus sc-win32-status time-taken +2022-11-07 16:47:25 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 - 304 0 0 256 +2022-11-07 16:47:25 ::1 GET /iisstart.png - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 http://localhost/ 304 0 0 2 +2022-11-07 16:47:25 ::1 GET /favicon.ico - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 http://localhost/ 404 0 2 16 +2022-11-07 16:48:07 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 - 304 0 0 0 +2022-11-07 16:48:08 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 - 304 0 0 1 +2022-11-07 16:48:08 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 - 304 0 0 0 +2022-11-07 16:48:08 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 - 304 0 0 0 +2022-11-07 16:48:08 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 - 304 0 0 0 +2022-11-07 16:48:08 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 - 304 0 0 0 +2022-11-07 16:48:08 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 - 304 0 0 0 +2022-11-07 16:48:09 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 - 304 0 0 0 +2022-11-07 16:48:09 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/107.0.0.0+Safari/537.36+Edg/107.0.1418.35 - 304 0 0 0 +2022-11-07 16:49:05 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:05 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:06 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:06 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:06 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 3 +2022-11-07 16:49:06 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:06 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:06 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:07 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:07 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:07 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:07 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:07 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:07 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:09 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:09 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:09 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:09 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:09 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:09 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:10 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:10 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:10 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:10 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:10 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 4 +2022-11-07 16:49:10 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:11 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:11 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:11 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:11 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:11 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:11 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:12 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:12 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:12 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:12 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:12 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:12 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:13 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:13 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:13 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 2 +2022-11-07 16:49:13 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:13 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:13 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:14 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:14 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:14 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:14 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 2 +2022-11-07 16:49:14 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:14 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:15 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:15 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:15 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:15 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:15 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:15 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:16 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:16 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:16 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:16 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:16 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:16 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:17 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:17 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:17 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 2 +2022-11-07 16:49:17 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:17 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:17 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:18 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:18 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 2 +2022-11-07 16:49:18 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:18 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 31 +2022-11-07 16:49:18 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:18 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:19 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 8 +2022-11-07 16:49:19 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:19 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:19 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:19 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:19 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:20 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:20 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:20 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:20 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:20 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:20 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:21 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:21 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:21 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:21 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:21 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:21 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:23 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:23 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:23 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:23 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:23 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:23 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 +2022-11-07 16:49:24 ::1 GET / - 80 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.20348.859 - 200 0 0 0 diff --git a/modules/weblog/weblog_test.go b/modules/weblog/weblog_test.go index 4fc8365af..1e53ffe09 100644 --- a/modules/weblog/weblog_test.go +++ b/modules/weblog/weblog_test.go @@ -24,6 +24,7 @@ var ( testFullLog, _ = os.ReadFile("testdata/full.log") testCustomLog, _ = os.ReadFile("testdata/custom.log") testCustomTimeFieldLog, _ = os.ReadFile("testdata/custom_time_fields.log") + testIISLog, _ = os.ReadFile("testdata/u_ex221107.log") ) func Test_readTestData(t *testing.T) { @@ -31,6 +32,7 @@ func Test_readTestData(t *testing.T) { assert.NotNil(t, testCommonLog) assert.NotNil(t, testCustomLog) assert.NotNil(t, testCustomTimeFieldLog) + assert.NotNil(t, testIISLog) } func TestNew(t *testing.T) { @@ -558,6 +560,78 @@ func TestWebLog_Collect_CustomTimeFieldsLogs(t *testing.T) { testCharts(t, weblog, mx) } +func TestWebLog_IISLogs(t *testing.T) { + weblog := prepareWebLogCollectIISFields(t) + + expected := map[string]int64{ + "bytes_received": 0, + "bytes_sent": 0, + "req_http_scheme": 0, + "req_https_scheme": 0, + "req_ipv4": 38, + "req_ipv6": 114, + "req_method_GET": 152, + "req_port_80": 152, + "req_proc_time_avg": 5, + "req_proc_time_count": 152, + "req_proc_time_hist_bucket_1": 133, + "req_proc_time_hist_bucket_10": 145, + "req_proc_time_hist_bucket_11": 146, + "req_proc_time_hist_bucket_2": 133, + "req_proc_time_hist_bucket_3": 133, + "req_proc_time_hist_bucket_4": 133, + "req_proc_time_hist_bucket_5": 133, + "req_proc_time_hist_bucket_6": 133, + "req_proc_time_hist_bucket_7": 133, + "req_proc_time_hist_bucket_8": 138, + "req_proc_time_hist_bucket_9": 143, + "req_proc_time_hist_count": 152, + "req_proc_time_hist_sum": 799, + "req_proc_time_max": 256, + "req_proc_time_min": 0, + "req_proc_time_sum": 799, + "req_type_bad": 42, + "req_type_error": 0, + "req_type_redirect": 0, + "req_type_success": 110, + "req_unmatched": 16, + "req_vhost_127.0.0.1": 38, + "req_vhost_::1": 114, + "requests": 168, + "resp_1xx": 0, + "resp_2xx": 99, + "resp_3xx": 11, + "resp_4xx": 42, + "resp_5xx": 0, + "resp_code_200": 99, + "resp_code_304": 11, + "resp_code_404": 42, + "uniq_ipv4": 1, + "uniq_ipv6": 1, + "upstream_resp_time_avg": 0, + "upstream_resp_time_count": 0, + "upstream_resp_time_hist_bucket_1": 0, + "upstream_resp_time_hist_bucket_10": 0, + "upstream_resp_time_hist_bucket_11": 0, + "upstream_resp_time_hist_bucket_2": 0, + "upstream_resp_time_hist_bucket_3": 0, + "upstream_resp_time_hist_bucket_4": 0, + "upstream_resp_time_hist_bucket_5": 0, + "upstream_resp_time_hist_bucket_6": 0, + "upstream_resp_time_hist_bucket_7": 0, + "upstream_resp_time_hist_bucket_8": 0, + "upstream_resp_time_hist_bucket_9": 0, + "upstream_resp_time_hist_count": 0, + "upstream_resp_time_hist_sum": 0, + "upstream_resp_time_max": 0, + "upstream_resp_time_min": 0, + "upstream_resp_time_sum": 0, + } + + mx := weblog.Collect() + assert.Equal(t, expected, mx) +} + func testCharts(t *testing.T, w *WebLog, mx map[string]int64) { testVhostChart(t, w) testPortChart(t, w) @@ -1167,6 +1241,56 @@ func prepareWebLogCollectCustomTimeFields(t *testing.T) *WebLog { return weblog } +func prepareWebLogCollectIISFields(t *testing.T) *WebLog { + t.Helper() + format := strings.Join([]string{ + "-", // date + "-", // time + "$host", // s-ip + "$request_method", // cs-method + "$request_uri", // cs-uri-stem + "-", // cs-uri-query + "$server_port", // s-port + "-", // cs-username + "$remote_addr", // c-ip + "-", // cs(User-Agent) + "-", // cs(Referer) + "$status", // sc-status + "-", // sc-substatus + "-", // sc-win32-status + "$request_time", // time-taken + }, " ") + cfg := Config{ + Parser: logs.ParserConfig{ + LogType: logs.TypeCSV, + CSV: logs.CSVConfig{ + // Users can define number of fields + FieldsPerRecord: -1, + Delimiter: " ", + TrimLeadingSpace: false, + Format: format, + CheckField: checkCSVFormatField, + }, + }, + Path: "testdata/u_ex221107.log", + ExcludePath: "", + URLPatterns: nil, + Histogram: nil, + GroupRespCodes: false, + } + + weblog := New() + weblog.Config = cfg + require.True(t, weblog.Init()) + require.True(t, weblog.Check()) + defer weblog.Cleanup() + + p, err := logs.NewCSVParser(weblog.Parser.CSV, bytes.NewReader(testIISLog)) + require.NoError(t, err) + weblog.parser = p + return weblog +} + // generateLogs is used to populate 'testdata/full.log' //func generateLogs(w io.Writer, num int) error { // var (