-
Notifications
You must be signed in to change notification settings - Fork 0
/
csvload_luc.conf
77 lines (68 loc) · 2.27 KB
/
csvload_luc.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
# read input from stdin (e.g. pipe)
input {
stdin {}
}
filter {
# filter the input by csv (i.e. comma-separated-value)
# Newline: MasterServer,JobID,ClientName,Schedule,StartTime,EndTime,Bytes,Throughput,StatusCode
#itsbus30,8436069,itsusral00341p.jnj.com,User backup,4/2/2015 11:39:48 PM,4/2/2015 11:42:28 PM,72384512,8710,0
#itsbus30,8436070,itsusraw01437,Differential Incremental,4/2/2015 11:41:49 PM,4/2/2015 11:43:26 PM,1105032192,17289,0
csv {
columns => [
"ServerName",
"JobID",
"ClientName",
"JobType",
"StartTime",
"EndTime",
"Volume-bytes",
"Throughput-KB-sec",
"StatusCode"
]
}
date {
# parse the "End Time" to create a real date
# Examples of times in this log file
# "May 29, 2015 10:00:01 PM"
# "May 9, 2015 4:47:23 AM"
# "May 23, 2015 12:23:49 PM"
# "4/2/2015 11:41:08 PM"
match => [ "EndTime",
"M/dd/YYYY hh:mm:ss aa",
"M/d/YYYY hh:mm:ss aa",
"MM/dd/YYYY hh:mm:ss aa",
"MM/d/YYYY hh:mm:ss aa",
"MMM dd, YYYY hh:mm:ss aa",
"MMM d, YYYY hh:mm:ss aa" ] }
mutate { replace => { "type" => "nbu_job" } }
mutate { gsub => ["NumberofFiles", ",", ""] }
mutate { convert => [ "NumberofFiles", "integer" ] }
mutate { gsub => ["Volume-KB", ",", ""] }
mutate { convert => [ "Volume-KB", "integer" ] }
mutate { gsub => ["Throughput-KB-sec", ",", ""] }
mutate { convert => [ "Throughput-KB-sec", "integer" ] }
# Example of Duration = "04:28:13" which is hours, minutes, and seconds
# Split up and create the respective integer fields
# grok {
# match => [ "Duration", "%{NUMBER:hours:int}:%{NUMBER:minutes:int}:%{NUMBER:seconds:int}" ]
# }
# Call ruby to perform the basic arithmetic of computing total seconds
# ruby {
# code => "event['Elapsed'] = event['hours']*3600 + event['minutes']*60 + event['seconds']"
# }
translate {
field => "ServerName"
destination => "Country"
dictionary_path => "./cmdb/ServerByCountry.yaml"
fallback => "Unknown"
}
}
# send the output to stdout, using the rubydebug codec
# rubydedug uses the Ruby Awesome Print library
output {
# stdout { codec => rubydebug }
elasticsearch {
cluster => batman
host => localhost
}
}