Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
160 lines (136 sloc) 5.6 KB
input {
file {
path => ["/Users/mikehillwig/elastic/flights/On_Time_On_Time_Performance*.csv"]
sincedb_path => "/dev/null"
start_position => "beginning"
}
}
filter {
csv {
columns => [
"Year", "Quarter", "Month", "DayofMonth", "DayOfWeek", "FlightDate", "UniqueCarrier", "AirlineID", "Carrier", "TailNum", "FlightNum", "OriginAirportID",
"OriginAirportSeqID", "OriginCityMarketID", "Origin", "OriginCityName", "OriginState", "OriginStateFips", "OriginStateName", "OriginWac",
"DestAirportID","DestAirportSeqID","DestCityMarketID","Dest","DestCityName","DestState","DestStateFips","DestStateName","DestWac",
"CRSDepTime","DepTime","DepDelay","DepDelayMinutes","DepDel15","DepartureDelayGroups","DepTimeBlk","TaxiOut","WheelsOff","WheelsOn","TaxiIn",
"CRSArrTime","ArrTime","ArrDelay","ArrDelayMinutes","ArrDel15","ArrivalDelayGroups","ArrTimeBlk","Cancelled","CancellationCode","Diverted",
"CRSElapsedTime","ActualElapsedTime","AirTime","Flights","Distance","DistanceGroup","CarrierDelay","WeatherDelay","NASDelay","SecurityDelay",
"LateAircraftDelay","FirstDepTime","TotalAddGTime","LongestAddGTime","DivAirportLandings","DivReachedDest","DivActualElapsedTime","DivArrDelay",
"DivDistance","Div1Airport","Div1AirportID","Div1AirportSeqID","Div1WheelsOn","Div1TotalGTime","Div1LongestGTime","Div1WheelsOff","Div1TailNum",
"Div2Airport","Div2AirportID","Div2AirportSeqID","Div2WheelsOn","Div2TotalGTime","Div2LongestGTime","Div2WheelsOff","Div2TailNum","Div3Airport",
"Div3AirportID","Div3AirportSeqID","Div3WheelsOn","Div3TotalGTime","Div3LongestGTime","Div3WheelsOff","Div3TailNum","Div4Airport","Div4AirportID",
"Div4AirportSeqID","Div4WheelsOn","Div4TotalGTime","Div4LongestGTime","Div4WheelsOff","Div4TailNum","Div5Airport","Div5AirportID","Div5AirportSeqID",
"Div5WheelsOn","Div5TotalGTime","Div5LongestGTime","Div5WheelsOff","Div5TailNum", "DestStateFips", "OriginStateFips"]
separator => ","
}
# if [Carrier] != "" {
# drop {}
# }
# mutate {copy => { "TailNum" => "ShortTailNum" }}
# mutate {gsub => ["ShortTailNum", "^N?", ""]}
# mutate {
# add_field => { "ACInfo" => "" }
# }
# translate {
# field => "ShortTailNum"
# destination => "ACInfo"
# dictionary_path => '/Users/mikehillwig/elastic/flights/master.csv'
# }
mutate {
remove_field => [ "message", "path", "host",
"Div2Airport","Div2AirportID","Div2AirportSeqID","Div2WheelsOn","Div2TotalGTime","Div2LongestGTime","Div2WheelsOff","Div2TailNum","Div3Airport",
"Div3AirportID","Div3AirportSeqID","Div3WheelsOn","Div3TotalGTime","Div3LongestGTime","Div3WheelsOff","Div3TailNum","Div4Airport","Div4AirportID",
"Div4AirportSeqID","Div4WheelsOn","Div4TotalGTime","Div4LongestGTime","Div4WheelsOff","Div4TailNum","Div5Airport","Div5AirportID","Div5AirportSeqID",
"Div5WheelsOn","Div5TotalGTime","Div5LongestGTime","Div5WheelsOff","Div5TailNum", "OriginAirportID",
"OriginAirportSeqID", "DestAirportID","DestAirportSeqID"]
convert => {
"AIR_SYSTEM_DELAY" => "integer"
"SECURITY_DELAY" => "integer"
"AIRLINE_DELAY" => "integer"
"LATE_AIRCRAFT_DELAY" => "integer"
"WEATHER_DELAY" => "integer"
"QUARTER" => "integer"
"DivAirportLandings" => "integer"
"DepDelay" => "integer"
"DepDelayMinutes"=> "integer"
"ArrDelay" => "integer"
"ArrDelayMinutes" => "integer"
"CarrierDelay" => "integer"
"WeatherDelay" => "integer"
"NASDelay" => "integer"
"SecurityDelay" => "integer"
"DivDistance" => "integer"
"Year" => "integer"
"Month" => "integer"
"DayofMonth" => "integer"
"DayOfWeek" => "integer"
"ArrivalDelayGroups" => "integer"
"Flights" => "integer"
"DistanceGroup" => "integer"
"CRSElapsedTime" => "integer"
"AirTime" => "integer"
"Distance" => "integer"
"Cancelled" => "integer"
"ActualElapsedTime" => "integer"
"TaxiIn" => "integer"
"TaxiOut" => "integer" }
}
mutate{
add_field => {"ts" => "%{FlightDate}" }
}
date {
match => [ "ts" , "yyyy-MM-dd" ]
target => "@timestamp"
locale => "en-US"
#timezone => "America/New_York"
}
mutate{
add_field => {"[@metadata][indexname]" => "flights-%{+YYYY}" }
add_field => {"[CRSDepTimeDT]" => "%{FlightDate} %{CRSDepTime}"}
add_field => {"[CRSArrTimeDT]" => "%{FlightDate} %{CRSArrTime}"}
add_field => {"[DepTimeDT]" => "%{FlightDate} %{DepTime}"}
add_field => {"[ArrTimeDT]" => "%{FlightDate} %{ArrTime}"}
add_field => {"[WheelsoffDT]" => "%{FlightDate} %{WheelsOff}"}
add_field => {"[WheelsOnDT]" => "%{FlightDate} %{WheelsOn}"}
}
date {
match => [ "CRSDepTimeDT" , "yyyy-MM-dd HHmm" ]
target => "CRSDepTimeDT"
locale => "en-US"
}
date {
match => [ "CRSArrTimeDT" , "yyyy-MM-dd HHmm" ]
target => "CRSArrTimeDT"
locale => "en-US"
}
date {
match => [ "DepTimeDT" , "yyyy-MM-dd HHmm" ]
target => "DepTimeDT"
locale => "en-US"
}
date {
match => [ "ArrTimeDT" , "yyyy-MM-dd HHmm" ]
target => "ArrTimeDT"
locale => "en-US"
}
date {
match => [ "WheelsOnDT" , "yyyy-MM-dd HHmm" ]
target => "WheelsOnDT"
locale => "en-US"
}
date {
match => [ "WheelsoffDT" , "yyyy-MM-dd HHmm" ]
target => "WheelsoffDT"
locale => "en-US"
}
`
}
output {
stdout { codec => rubydebug }
# elasticsearch {
# hosts => ".eastus2.cloudapp.azure.com:9200"
# action => "index"
# index => "%{[@metadata][indexname]}"
# user => "elastic"
# password => ""
# workers => 1 }
}