Permalink
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
108 lines (95 sloc) 2.61 KB
input { stdin { } }
filter {
## ----- CSV PROCESSING --------
csv {
columns => ["rawDuration","startDate","endDate","rawStartStation","rawEndStation","bikeNum","memberType"]
}
mutate {
remove_field => "message"
add_field => {
"durHours" => 0
"durMin" => 0
"durSec" => 0
"city" => "Washington, DC"
}
}
mutate {
convert => {
"durHours" => "integer"
"durMin" => "integer"
"durSec" => "integer"
}
}
## ----- CLEAN STATION NAMES --------
# remove optinonal station number from end of the station name
grok {
match => [ "rawStartStation", "^%{DATA:startStation}(\ \(%{INT}\))?$"]
remove_field => "rawStartStation"
}
# remove optinonal station number from end of the station name
grok{
match => [ "rawEndStation", "^%{DATA:endStation}(\ \(%{INT}\))?$"]
remove_field => "rawEndStation"
}
## ----- INTERRPET DURATION ------
grok {
match => [ "rawDuration", "(%{INT:durHours:int}h\ ?)?(%{INT:durMin:int}min\.?\ ?)?(%{INT:durSec:int}sec\.?)?"]
overwrite => ["durHours", "durMin", "durSec"]
}
ruby {
code => " event['duration'] = event['durHours'] * 3600 + event['durMin'] * 60 + event['durSec'] / 60.0 "
remove_field => ["rawDuration","durHours","durMin","durSec"]
}
## ----- INTERPRET DATES --------
# event date is the start Date
date {
match => ["startDate", 'MM/dd/YYYY HH:mm']
timezone => "America/New_York"
locale => en
}
# adjust date to ISO 8601
date {
match => ["startDate", 'MM/dd/YYYY HH:mm']
timezone => "America/New_York"
locale => en
target => "startDate"
}
# adjust date to ISO 8601
date {
match => ["endDate", 'MM/dd/YYYY HH:mm']
timezone => "America/New_York"
locale => en
target => "endDate"
}
## ----- Process Geospatial --------
geoEnrich {
database => "/Users/dave/dev/examples/cabi/logstash-attempt/fullDCStations.csv"
source => "startStation"
target => "startLocation"
}
geoEnrich {
database => "/Users/dave/dev/examples/cabi/logstash-attempt/fullDCStations.csv"
source => "endStation"
target => "endLocation"
}
## ------ Behavior ------
ruby {
code => " event['behavior'] = (event['startStation'] == event['endStation']) ? 'Round Trip' : 'Point to Point'"
}
#make all data sources conform to 'Subscriber' and 'Casual' for memberType
ruby {
code => " if (event['memberType'] == 'Registered') then event['memberType'] = 'Subscriber' end "
}
}
output {
#stdout { codec => rubydebug }
stdout { codec => dots }
#stdout { codec => json }
elasticsearch {
index => "bikelog-dc-%{+YYYY}"
index_type => rides
manage_template => false
host => localhost
protocol => http
}
}