modsec-rulereport.rb

#!/usr/bin/ruby
#
# A ruby script which extracts ModSec alerts out of an apache
# error log and displays them in a terse report.
#
# Multiple options exist to tailor the report. When trying to
# tune a modsecurity installation, the script can propose
# rules or directives for the apache configuration, which can 
# be used to bypass the false positives reported by the script.
#
# Call with the option --help to get an usage overview.
#
# TODO / FIXME
# - Import Error-Log
# - ignore-rule modes:
#   - non-empty username
#   - method-based
#   - reduce anomaly score as ignore-rule
#   - stats mode (number and percentages of rule hits, paths, parameters)
# - order by number of hits per rule or rule id
# - option to have anomaly scoring checks be included in the rule
#     (hidden by default)

# -----------------------------------------------------------
# INIT
# -----------------------------------------------------------

require "optparse"
require "date"
require "pp"
require 'open-uri'
require "rubygems"

$params = Hash.new

$params[:verbose] = false
$params[:debug]   = false

RULEID_DEFAULT = 10000

MODE_SUPERSIMPLE=1
MODE_SIMPLE=2
MODE_PARAMETER=3
MODE_PATH=4
MODE_COMBINED=5
MODE_RULE=7
MODE_GRAPHVIZ=8
MODE_ALL=16

$params[:mode]   = MODE_SUPERSIMPLE

$params[:filenames] = Array.new
$params[:ruleid] = RULEID_DEFAULT

Severities = {
	"NOTICE" => 2,
	"WARNING" => 3,
	"ERROR" => 4,
	"CRITICAL" => 5
}

class Event
	attr_accessor :id, :unique_id, :ip, :msg, :uri, :parameter, :hostname, :file, :tags

	def initialize(id, unique_id, ip, msg, uri, parameter, hostname, file, tags)
		@id = id
		@unique_id = unique_id
		@ip = ip
		@msg = msg
		@uri = uri
		@parameter = parameter
		@hostname = hostname
		@file = file
		@tags = tags
	end
end


# -----------------------------------------------------------
# SUB-FUNCTIONS (those that are specific to this script)
# -----------------------------------------------------------

def import_files(filenames)
  # Purpose: Import files
  # Input  : filename array
  # Output : none
  # Return : events array
  # Remarks: none

  events = Array.new()

  begin

    unless (check_stdin())

      filenames.each do |filename|

        File.open(filename, "r") do |file|

      	  vprint "Reading file #{filename} ..."
	  events.concat(read_file(file))

	end

      end

    else

      	vprint "Reading STDIN ..."
	events.concat(read_file(STDIN))

    end

  rescue Errno::ENOENT => detail
    puts_error("File could not be opened. This is fatal. Aborting.", detail)
    exit 1
  rescue => detail
    puts_error("Unknown error during file read. This is fatal. Aborting.", detail)
    exit 1
  end

  return events

end

def read_file(file)
  # Purpose: Read file
  # Input  : file handle
  # Output : none
  # Return : events array
  # Remarks: none

  events = Array.new()

  def scan_line (line, key, default)
    begin
      return line.scan(/\[#{key} \"([^"]*)\"/)[0][0]
    rescue
      return default
    end
  end

  while ! file.eof?
	line = file.readline
	if /ModSecurity: (Warning|Access denied.*). (?!Unconditional match in SecAction)/.match(line)

	  # standard parameters
          id = scan_line(line, "id", "0")
          unique_id = scan_line(line, "unique_id", "no-id-found")
          msg = scan_line(line, "msg", "none")
	  uri = scan_line(line, "uri", "/")
	  hostname = scan_line(line, "hostname", "unknown")
	  eventfile = scan_line(line, "file", "none")

	  # custom parameters
	  begin
	    ip = line.scan(/\[client ([^\]]*)\]/)[0][0]
	  rescue
	    ip = "0.0.0.0"
	  end

	  begin
	    if /ModSecurity: Warning. Pattern match/.match(line)
	    	    # example: standard operator results in:  ModSecurity: Warning. Pattern match "^[\\\\d.:]+$" at REQUEST_HEADERS:Host.
		    parameter = line.scan(/ (at|against) "?(.*?)"?( required)?\. \[file \"/)[0][1]
	    elsif /ModSecurity: Warning. detected (SQLi|XSS) using libinjection/.match(line)
	    	    # example: ModSecurity: Warning. detected SQLi using libinjection with fingerprint 'sos' [file ... ] [data "Matched Data: sos found within ARGS:install[values][GFX][processor_stripColorProfileCommand]:  profile '*'"
		    parameter = line.scan(/found within ([^ ]*): /)[0][0]
	    end
	  rescue
	    puts "XXXX"
	    exit
	    parameter = ""
	  end

	  # FIXME: read tags
	  tags = Array.new

	  events << Event.new(id, unique_id, ip, msg, uri, parameter, hostname, eventfile, tags)

        end
  end

  return events
	
end

def display_ignore_rule_mode_rule(id, event, events)
  # Purpose: display ignore rule based on rule
  # Input  : rule id, event object, events array
  # Output : report via stdout
  # Return : none
  # Remarks: none

	rules = Array.new
	puts "      # ModSec Rule Exclusion: #{event.id} : #{event.msg}"
	puts "      SecRuleRemoveById #{event.id}"
	
end

def display_ignore_rule_mode_parameter(id, event, events)
  # Purpose: display ignore rule based on parameter of event
  # Input  : rule id, event object, events array
  # Output : report via stdout
  # Return : none
  # Remarks: none

	parameters = Array.new
	events.select{|h| h.id == id }.each do |h|
		if parameters.grep(h.parameter).length == 0 
			parameters << h.parameter
		end
	end
	parameters.sort!{|x,y| x <=> y }

	if parameters.length == 0 or ( parameters.length == 1 and parameters[0] == "" )
		puts "      No parameter available to create ignore-rule proposal."
	else
		puts "      # ModSec Rule Exclusion: #{id} : #{event.msg}"


	
		parameters.each do |parameter|
			num = events.select{|h| h.id == id && h.parameter == parameter}.length
			if parameter != ""
				printf "      SecRuleUpdateTargetById %6d \"!%s\"\n", id, parameter
			end
		end
	end

end

def display_ignore_rule_mode_path(id, event, events)
  # Purpose: display ignore rule based on path of event
  # Input  : rule id, event object, events array
  # Output : report via stdout
  # Return : none
  # Remarks: none

	puts "      # ModSec Rule Exclusion: #{id} : #{event.msg}"
	puts "      SecRule REQUEST_URI \"@beginsWith /foo\" \"phase:1,nolog,pass,id:#{$params[:ruleid]},ctl:ruleRemoveById=#{id}\""
	$params[:ruleid] = $params[:ruleid] + 1

	uris = Array.new
	events.select{|h| h.id == id }.each do |h|
		if uris.grep(h.uri).length == 0 
			uris << h.uri
		end
	end
	uris.sort!{|x,y| x <=> y }

	puts
	puts "      Individual paths:"
	uris.each do |uri|
		num = events.select{|h| h.id == id && h.uri == uri}.length

		hostnames = Array.new
		events.select{|h| h.id == id and h.uri == uri}.each do |h|
			if hostnames.grep(h.hostname).length == 0
				hostnames << h.hostname
			end
		end

		if hostnames.length > 1
			printf "  %6d %s\t(multiple services: %s)\n", num.to_s, uri, hostnames.join(" ")
		else
			printf "  %6d %s\t(service %s)\n", num.to_s, uri, hostnames[0]
		end
	end

end

def display_ignore_rule_mode_path_and_parameter(id, event, events)
  # Purpose: display ignore rule based on path and parameter of event
  # Input  : rule id, event object, events array
  # Output : report via stdout
  # Return : none
  # Remarks: none

	puts "      # ModSec Rule Exclusion: #{id} : #{event.msg}"
	items = Array.new
	dprint "Building list with paths and parameters for this rule / event id:"
	events.select{|e| e.id == id }.each do |e|
		if e.parameter != ""
			num = items.select{|couple| couple[:parameter] == e.parameter && couple[:uri] == e.uri}.length
			if num == 0
				# FIXME: calling this couple is really stupid, now that's a triple
				couple = Hash.new
				couple[:parameter] = e.parameter
				couple[:uri] = e.uri
				couple[:num] = 1
				dprint "  Creating new couple with parameter #{couple[:parameter]} and uri #{couple[:uri]}"
				items << couple
			else
				couple = items.select{|couple| couple[:parameter] == e.parameter && couple[:uri] == e.uri}[0]
				dprint "  Raising number of occurrence of couple with parameter #{couple[:parameter]} and uri #{couple[:uri]} to #{couple[:num] + 1}"
				couple[:num] = couple[:num] + 1
			end
		else
			dprint "  No argument found in event. Event can thus not be handled in this mode. Passing to next event."
		end
	end
	items.sort!{|x,y| x[:parameter] <=> y[:parameter] }
	if $params[:debug]
		puts "Items/couples to be used for ignore rule with id #{id}:"
		pp items
	end

	if items.length == 0 or ( items.length == 1 and items[0] == "" )
		puts "  No parameter available to create ignore-rule proposal. Please try and use different mode."
	else
		items.each do |couple|
				prefix = ""
				if $params[:verbose]
					prefix = couple[:num].to_s + " x"
				end
				printf "     %s SecRule REQUEST_URI \"@beginsWith %s\" \"phase:2,nolog,pass,id:%d,ctl:ruleRemoveTargetById=%d;%s\"\n", prefix, couple[:uri], $params[:ruleid], id, couple[:parameter]
				$params[:ruleid] = $params[:ruleid] + 1

		end
	end
 
end



def display_report(events)
  # Purpose: display report
  # Input  : events array
  # Output : report via stdout
  # Return : none
  # Remarks: none

  ids = Array.new
  dprint "Building list of relevant ids:"
  events.each do |event|
		if ids.grep(event.id).length == 0 && 
			( event.id != "981176" && event.id != "981202" && event.id != "981203" && event.id != "981204" && event.id != "981205" ) # 981203/4/5 are the rules checking anomaly score in the end. Ignoring those
			dprint "  Adding event id #{event.id}"
			ids << event.id
		else
			# id is already part of id list
			dprint "  Ignoring event id #{event.id}"
		end
  end
  ids.sort!{|a,b| a <=> b }
  
  if $params[:mode] != MODE_SIMPLE and $params[:mode] != MODE_SUPERSIMPLE
	puts
  end

  ids.each do |id|
	event = events.find {|e| e.id == id }
	len = events.select{|e| e.id == id }.length
	case $params[:mode]
		when MODE_SIMPLE
			out = len.to_s + " x " + id.to_s + " " + event.msg + " : " 
			n = 0
			events.select{|e| e.id == id }.each do |e|
				out = out + ", " unless n == 0 
				out = out + e.parameter
				n = n + 1
			end
		when MODE_SUPERSIMPLE
			out = len.to_s + " x " + id.to_s + " " + event.msg
		else
			out = len.to_s + " x " + id.to_s + " " + event.msg
	end
	print out + "\n"
	if $params[:mode] != MODE_SIMPLE and $params[:mode] != MODE_SUPERSIMPLE
		0.upto(out.length-1) do |i| print "-"; end; print "\n" # breakline
	end
	case $params[:mode]
		when MODE_RULE
			display_ignore_rule_mode_rule(id, event, events)
		when MODE_PARAMETER
			display_ignore_rule_mode_parameter(id, event, events)
		when MODE_PATH
			display_ignore_rule_mode_path(id, event, events)
		when MODE_COMBINED
			display_ignore_rule_mode_path_and_parameter(id, event, events)
		when MODE_ALL
			display_ignore_rule_mode_parameter(id, event, events)
			puts
			display_ignore_rule_mode_path(id, event, events)
			puts
			display_ignore_rule_mode_path_and_parameter(id, event, events)
	end
	if $params[:mode] != MODE_SIMPLE and $params[:mode] != MODE_SUPERSIMPLE
		puts
	end
  end

end


			


def display_report_graphviz(events)
  # Purpose: display graphviz report
  # Input  : events array
  # Output : report via stdout
  # Return : none
  # Remarks: none

  # Prepare ids
  dprint "Building list of relevant ids:"
  ids = Array.new
  events.each do |event|
		if ids.grep(event.id).length == 0 && 
			( event.id != "981176" && event.id != "981202" && event.id != "981203" && event.id != "981204" && event.id != "981205" ) # 981203/4/5 are the rules checking anomaly score in the end. Ignoring those
			dprint "  Adding event id #{event.id}"
			ids << event.id
		else
			# id is already part of id list
			dprint "  Ignoring event id #{event.id}"
			# FIXME: Grow number
		end
  end
  ids.sort!{|a,b| a <=> b }
  
  
  # Prepare uris
  dprint "Building list of relevant uris:"
  uris = Array.new
  events.each do |event|
		if uris.grep(event.uri).length == 0
			dprint "  Adding event uri #{event.uri}"
			uris << event.uri
		else
			dprint "  Ignoring event uri #{event.uri}"
		end
  end
  uris.sort!{|a,b| a <=> b }


  
  # Prepare parameters
  dprint "Building list of relevant parameters:"
  parameters = Array.new
  events.each do |event|
		if parameters.grep(event.parameter).length == 0
			dprint "  Adding event parameter #{event.parameter}"
			parameters << event.parameter
		else
			dprint "  Ignoring event parameter #{event.parameter}"
		end
  end


  puts "// HEADER"
  puts "digraph G {"
  puts "  size = \"24,24\";"
  puts "  ranksep=\"8\";"
  puts

  puts "// DEFINING URI NODES"
  uris.each do |item|
  	puts "  \"#{item}\" [shape=house,fontname=helvetica];"
  end
  puts

  puts "// DEFINING PARAMETER NODES"
  parameters.each do |item|
  	puts "  \"#{item}\" [shape=invhouse,fontname=helvetica];"
  end
  puts

  puts "// DEFINING ID NODES"
  ids.each do |item|
  	puts "  \"#{item}\" [shape=box,fontname=helvetica];"
  end
  puts

  puts "// EDGES: URI -> PARAMETER"
  uris.each do |uri|
	event = events.find {|e| e.uri == uri }
   
  	items = Array.new

	dprint "Building list with parameters for this uri:"
	events.select{|e| e.uri == uri }.each do |e|
		if e.parameter != ""
			num = items.select{|couple| couple[:parameter] == e.parameter && couple[:uri] == e.uri}.length
			if num == 0
				couple = Hash.new
				couple[:parameter] = e.parameter
				couple[:uri] = e.uri
				couple[:num] = 1
				dprint "  Creating new couple with parameter #{couple[:parameter]} and uri #{couple[:uri]}"
				items << couple
			else
				couple = items.select{|couple| couple[:parameter] == e.parameter && couple[:uri] == e.uri}[0]
				dprint "  Raising number of occurrence of couple with parameter #{couple[:parameter]} and uri #{couple[:uri]} to #{couple[:num] + 1}"
				couple[:num] = couple[:num] + 1
			end
		else
			dprint "  No argument found in event. Event can thus not be handled in this mode. Passing to next event."
		end
	end
	items.sort!{|x,y| x[:parameter] <=> y[:parameter] }
	if $params[:debug]
		puts "Items/couples to be used in this group with uri #{uri}:"
		pp items
	end

	if items.length == 0 or ( items.length == 1 and items[0] == "" )
		puts "  No parameter available to display group."
	else
		items.each do |couple|
				penwidth = 1
				if couple[:num] >= 10
					penwidth = 4
				end
				if couple[:num] >= 100
					penwidth = 8
				end
				if couple[:num] >= 1000
					penwidth = 12
				end
				printf "  \"%s\"	-> \"%s\" [penwidth=#{penwidth},weight=#{penwidth}];\n", couple[:uri], couple[:parameter]

		end
	end
 
  end

  puts
  puts
  puts "// EDGES: PARAMETER -> ID"
  parameters.each do |parameter|
	event = events.find {|e| e.parameter == parameter }
   
  	items = Array.new

	dprint "Building list with ids for this parameter:"
	events.select{|e| e.parameter == parameter }.each do |e|
		if e.parameter != ""
			num = items.select{|couple| couple[:parameter] == e.parameter && couple[:id] == e.id}.length
			if num == 0
				couple = Hash.new
				couple[:parameter] = e.parameter
				couple[:id] = e.id
				couple[:num] = 1
				dprint "  Creating new couple with parameter #{couple[:parameter]} and id #{couple[:id]}"
				items << couple
			else
				couple = items.select{|couple| couple[:id] == e.id && couple[:parameter] == e.parameter}[0]
				dprint "  Raising number of occurrence of couple with parameter #{parameter} and id #{couple[:id]}"
				dprint "  Raising number of occurrence of couple with parameter #{parameter} and id #{couple[:id]} to #{couple[:num] + 1}"
				couple[:num] = couple[:num] + 1
			end
		else
			dprint "  No argument found in event. Event can thus not be handled in this mode. Passing to next event."
		end
	end
	items.sort!{|x,y| x[:parameter] <=> y[:parameter] }
	if $params[:debug]
		puts "Items/couples to be used in this group with uri #{uri}:"
		pp items
	end

	if items.length == 0 or ( items.length == 1 and items[0] == "" )
		puts "  No parameter available to display group."
	else
		items.each do |couple|
				penwidth = 1
				if couple[:num] >= 10
					penwidth = 4
				end
				if couple[:num] >= 100
					penwidth = 8
				end
				if couple[:num] >= 1000
					penwidth = 12
				end
				printf "  \"%s\"	-> \"%s\" [penwidth=#{penwidth},weight=#{penwidth}];\n", couple[:parameter], couple[:id]

		end
	end
 
  end


  if $params[:mode] == MODE_GRAPHVIZ
  	puts "}"
  end


end
	
# -----------------------------------------------------------
# GENERIC SUB-FUNCTIONS (those that come with every script)
# -----------------------------------------------------------
#
def dump_parameters(params)
  # Purpose: Display parameters
  # Input  : Parameter Hash
  # Output : Dump parameters to stdout
  # Return : none
  # Remarks: none
  
  puts "Paramter overview"
  puts "-----------------"
  puts "verbose    : #{params[:verbose]}"
  unless check_stdin()
  	puts "files      : #{params[:filenames].each do |x| x ; end}"
  else
  	puts "files      : [STDIN]"
  end
  puts "mode       : #{params[:mode]}" # FIXME: translate modes back to text strings

end

def vprint(text)
  # Purpose: output text if global variable $verbose is set.
  # Input  : String input
  # Output : stdout
  # Remarks: none
  
  if $params[:verbose]
    puts text + "\n"
  end

end

def dprint(text)
  # Purpose: output text if global variable $debug is set.
  # Input  : String input
  # Output : stdout
  # Remarks: none
  
  if $params[:debug]
    puts text + "\n"
  end

end

def check_stdin ()
  # Purpose: Check for access to STDIN
  # Input  : none
  # Output : bool
  # Remarks: none

  if STDIN.tty?
    # no stdin
    return false
  else
    # stdin
    return true
  end

end

def check_parameters()
  # Purpose: check parameters
  # Input  : global variable params
  # Output : stderr in case there is a problem with one of the parameters
  # Return : true if there is an error with one of the parameters; or false in absence of errors
  # Remarks: None

  err_status = false

  unless $params[:ruleid] > 0
   $stderr.puts "Error in ruleid parameter (#{$params[:ruleid]}). Has to be an integer above 0. This is fatal. Aborting."
   err_status = true
  end
  
  return err_status
  
end

def puts_error(msg, detail)
  # Purpose: Print error message
  # Input  : string msg and detail exception object
  # Output : $stderr
  # Return : None
  # Remarks: There is a ruby exception class hierarchy. 
  #          See http://makandracards.com/makandra/4851-ruby-exception-class-hierarchy

  err_status = false
  $stderr.puts msg
  $stderr.puts "Error: #{detail.message}" if detail
  $stderr.puts "Backtrace:" if detail
  $stderr.puts detail.backtrace.join("\n") if detail
  $stderr.puts "--------------------------"

end


# -----------------------------------------------------------
# COMMAND LINE PARAMETER EXTRACTION
# -----------------------------------------------------------
#

begin

parser = OptionParser.new do|opts|
  opts.banner = <<EOF
  Description of script ...
  A ruby script which extracts ModSec alerts out of an apache
  error log and displays them in a terse report.

  Multiple options exist to tailor the report. When trying to
  tune a modsecurity installation, the script can propose
  rules or directives for the apache configuration, which can 
  be used to bypass the false positives reported by the script.

  Usage: #{__FILE__} [options]
EOF

  opts.banner.gsub!(/^\t/, "")

        opts.separator ""
        opts.separator "Options:"

  opts.on('-d', '--debug', 'Display debugging infos') do |none|
    $params[:debug] = true;
  end

  opts.on('-v', '--verbose', 'Be verbose') do |none|
    $params[:verbose] = true;
  end

  opts.on('-m', '--mode STR', 'Ignore-Rule suggestion mode:
                                     One of "simple", "supersimple", "rule", "parameter", "path",
                                     "combined" or "graphviz". Default is "supersimple"') do |mode|
	  case mode
	  when "simple"
    		$params[:mode] = MODE_SIMPLE;
	  when "supersimple"
    		$params[:mode] = MODE_SUPERSIMPLE;
	  when "parameter"
    		$params[:mode] = MODE_PARAMETER;
	  when "path"
    		$params[:mode] = MODE_PATH;
	  when "combined"
    		$params[:mode] = MODE_COMBINED;
	  when "rule"
    		$params[:mode] = MODE_RULE;
	  when "graphviz"
    		$params[:mode] = MODE_GRAPHVIZ;
	  when "all"
    		$params[:mode] = MODE_ALL;
	  else
		$stderr.puts "Unknown mode \"#{mode}. This is fatal. Aborting."
		exit 1
	  end
  end
  
  opts.on('-r', '--ruleid STR', "Start of ruleid namespace to be used. Default is #{RULEID_DEFAULT}.") do |ruleid|
    $params[:ruleid] = ruleid.to_i;
  end

  opts.on('-h', '--help', 'Displays Help') do
    puts opts
    exit
  end

  # Usage notes (to be printed in help text after cli options) 
  notes = <<EOF

  Notes:

  ...

EOF

  notes.gsub!(/^\t/, "")
  
  opts.on_tail(notes)
end

parser.parse!

ARGV.each do|f|  
  $params[:filenames] << f
end

# Mandatory Argument Check
# if $params[:man].nil?
#       $stderr.puts "FIXME argument missing in call. This is fatal. Aborting."
#       exit 1
# end

rescue OptionParser::InvalidOption => detail
  puts_error("Invalid Option in command line parameter extraction. This is fatal. Aborting.", detail)
  exit 1
rescue => detail
  puts_error("Unknown error in command line parameter extraction. This is fatal. Aborting.", detail)
  exit 1

end

# -----------------------------------------------------------
# MAIN
# -----------------------------------------------------------

vprint "Starting parameter checking"

exit 1 if (check_parameters)

dump_parameters($params) if $params[:verbose]

vprint "Starting main program"

events = import_files($params[:filenames])

unless $params[:mode] == MODE_GRAPHVIZ
	display_report(events)
else
	display_report_graphviz(events)
end

vprint "Finishing main program. Bailing out."