Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 442 lines (379 sloc) 13.7 KB
#!/usr/bin/env ruby
=begin
net_discovery_reporter.rb -- @TheJoSko
Net Discovery Reporter will create a Word DOCX formatted table and populate it with parsed results from Nmap/Masscan
XML files or Hostwrangled CSV files (ref: @atucom: dotfiles/atu-hostwrangle). The script will also parse passive DNS
data from the Farsight DNSDB rdata tables. DNSDB output must be in JSON format. PassiveDNS entries from DNSDB will be
restricted the last 365 days in final output. If you wish to change this, modify the parse_dnsdb method.
The table is first sorted by target IP, then protocol, then port number.
If duplicate port listings are found when parsing the input files, the longer service (typically found with sV) will be
retained. Any subsequent services for that IP/PORT/PROTO that have the same or shorter service description length will
be dropped. This allows you to run the tool with nmap XML output from sT, sS, sU, sV, etc. scans, and it will populate a
table of unique open ports with the longest service description retained. Nmap NSE ssl-cert subjectName and SAN's will
be added to the hostnames if detected in nmap XML.
File format auto detection is included for the supported Nmap/Masscan XML, Hostwrangle CSV, and DNSDB JSON standard formats.
Table output can be further restricted with a line separated file of IPs. Useful when you perform broad "network discovery" scans initially,
but want to create a trimmed down table (without manual edits) for a final report.
for i in `< targets`; do curl -s -H 'Accept: application/json' https://<dnsdb>/lookup/rdata/ip/$i >> dnsdb_output; done
Installation:
gem install ruby-nmap
gem install caracal
gem install tty-progressbar
Usage:
Single File:
./net_discovery_reporter.rb -f nmap_oX_file.xml
Multiple Files:
./net_discovery_reporter.rb -f nmapsV.xml,nmapsT.xml,sUhostwrangled.csv
Directory:
./net_discovery_reporter.rb -d /directory/of/nmap_xml/files/
or
./net_discovery_reporter.rb -d .
Restrict output to list of IPs:
./net_discovery_reporter.rb -d ./ -t targets.txt
Example DNSDB query with JSON output:
echo "192.168.1.1,24" > targets
for i in `< targets`; do curl -s -H 'Accept: application/json' https://<dnsdb>/lookup/rdata/ip/$i >> dnsdb_output; done
=end
#Require necessary gems or gracefully error/exit if not available
['optparse','csv','nmap/xml','caracal','tty-progressbar','json'].each do |required_gem|
begin
require required_gem
rescue LoadError
puts "A required dependency is missing. The following gems are required to use this script:"
puts " ==> ruby-nmap"
puts " ==> caracal"
puts " ==> tty-progressbar\n\n"
puts "In most cases these can be installed with: gem install ruby-nmap caracal tty-progressbar"
exit
end
end
module OpenPorts
class Host
attr_accessor :ip, :hostnames, :ports
def initialize(args)
@ip = args[:ip]
self.hostnames = []
self.ports = {}
end
def self.all
ObjectSpace.each_object(self).to_a
end
def add_open_port(args)
@protocol = args[:protocol]
@port = args[:port]
@service = args[:service]
@port_hash = @port.to_s + "_" + @protocol.to_s
if is_port_unique(@port_hash, @protocol)
ports[@port_hash] = {:protocol => @protocol, :service => @service}
elsif is_service_longer(@port_hash, @service)
ports[@port_hash] = {:protocol => @protocol, :service => @service}
end
end
def is_port_unique(port_hash, protocol)
if ports.include? port_hash
return false
end
true
end
def is_service_longer(port_hash, service)
if ports[port_hash][:service].to_s.strip.length == 0
return true
end
if ports[port_hash][:service].to_s <= service.to_s
true
end
end
def add_hostname(hostname)
if is_hostname_unique(hostname)
hostnames.push(hostname)
end
end
def is_hostname_unique(hostname)
if hostnames.include? hostname
return false
end
true
end
end
end
#Print options help if no arguments supplied
ARGV << '-h' if ARGV.empty?
#Options parsing
options = {}
OptionParser.new do |opts|
opts.banner = "\nUsage: net_discovery_reporter.rb [options]\n"
opts.on('-f', '--file FILENAME,FILENAME2', Array, 'Nmap XML file (singular, or a comma-separated list)') { |file| options[:file] = file }
opts.on('-d', '--dir PATH', 'Directory of Nmap XML files') { |dir| options[:dir] = dir }
opts.on('-t', '--targets FILENAME', 'Line separated file of targets to include in table') { |targets| options[:targets] = targets }
opts.on('-v', '--verbose', 'Enable verbose messages') { |verbose| options[:verbose] = verbose }
opts.on_tail('-h', '--help', 'Display this screen') { puts opts; exit }
end.parse!
#Output filename
DOCX_FILE = 'Net_Discovery_Report.docx'
#Verbosity
if options[:verbose]
VERBOSE = true
else
VERBOSE = false
end
#Disable garbage collection or large directory parsing will result in lost Host objects
GC.disable
#Parse a CSV file. Search all objects of Host class to append, or instantiate a new object
def parse_hostwrangle(file)
csv_array = CSV.read(file)
bar = TTY::ProgressBar.new("|:bar| :percent", total: csv_array.count, width: 60)
csv_array.each do |row|
bar.advance(1)
if row[4] == "open"
existing_host = nil
ObjectSpace.each_object(OpenPorts::Host) do |obj|
if obj.ip == row[0]
existing_host = obj
break
end
end
report_host = existing_host || OpenPorts::Host.new(:ip => row[0])
report_host.add_hostname(row[2]) unless row[0] == row[2]
service = ''
service += row[5] + ": " unless row[5].nil?
service += " " + row[6] unless row[6].nil?
service += " " + row[7] unless row[7].nil?
report_host.add_open_port(
:port => row[1],
:protocol => row[3].upcase,
:service => service
)
end
end
end
#Parse a NMAP XML file. Search all objects of Host class to append, or instantiate a new object
def parse_nmap(file)
Nmap::XML.new(file) do |nmap|
bar = TTY::ProgressBar.new("|:bar| :percent", total: nmap.each_host.count, width: 60)
nmap.each_host do |host|
bar.advance(1)
host.each_port do |port|
if port.state == :open
existing_host = nil
ObjectSpace.each_object(OpenPorts::Host) do |obj|
if obj.ip == host.ip
existing_host = obj
break
end
end
report_host = existing_host || OpenPorts::Host.new(:ip => host.ip)
report_host.add_hostname(host.to_s) unless host.ip == host.to_s
#NSE ssl-cert script parsing
port.scripts.each do |id,output|
if id == 'ssl-cert'
if subjectMatch = output.match(/Subject\:\ commonName\=([^\,\n\*\/]+)/)
subjectName = subjectMatch.captures
report_host.add_hostname(subjectName[0])
end
if sanMatch = output.match(/Subject\ Alternative\ Name\:\ DNS\:([^\,\n\*\/]+)/)
subjectAltName = sanMatch.captures
subjectAltName.each do |san|
report_host.add_hostname(san)
end
end
end
end
service = ''
service += port.service.name + ": " unless port.service.nil?
service += " " + port.service.product unless port.service.nil? or port.service.product.nil?
service += " " + port.service.version unless port.service.nil? or port.service.version.nil?
report_host.add_open_port(
:port => port.number,
:protocol => port.protocol.upcase,
:service => service
)
end
end
end
end
end
def parse_dnsdb(file)
t = Time.now.to_i
json_file = open(file)
json = json_file.read
bar = TTY::ProgressBar.new("|:bar| :percent", total: json.each_line.count, width: 60)
json.each_line do |line|
bar.advance(1)
unless line =~ /Error: no results found for query./
parsed = JSON.parse(line)
#If last seen was within the last 365 days
if (!parsed["time_last"].nil? and parsed["time_last"] > t - 31536000) || (!parsed["zone_time_last"].nil? and parsed["zone_time_last"] > t - 31536000)
#Iterate through each host object
ObjectSpace.each_object(OpenPorts::Host) do |obj|
#If IP's match add hostname to host object
if obj.ip == parsed["rdata"]
obj.add_hostname(parsed["rrname"])
break
end
end
end
end
end
end
#Get format of file
def get_format(file)
if File.foreach(file).count >= 5
lines = File.foreach(file).first(5)
[0, 3, 4].each do |x|
if !lines[x].valid_encoding?
lines[x] = lines[x].encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
end
end
if lines[0] =~ /\d+.\d+.\d+.\d+.\,.*\,(tcp|udp)/
return "hostwrangle"
elsif lines[3] =~ /\<nmaprun/ || lines[4] =~ /\<nmaprun/
return "nmap"
elsif lines[0] =~ /Error\:\ no\ results\ found\ for\ query/ || lines[0] =~ /\{\"count/
return "dnsdb_json"
end
end
end
#Call parser
def run_parse(file, file_format)
if file_format == "hostwrangle"
puts "[+] Parsing Hostwrangle CSV file: #{file}"
parse_hostwrangle(file)
elsif file_format == "nmap"
puts "[+] Parsing NMAP/Masscan XML file: #{file}"
parse_nmap(file)
elsif file_format == "dnsdb_json"
puts "[+] Parsing DNSDB JSON file: #{file}"
parse_dnsdb(file)
else
puts "[!] #{file} does not match standard Nmap/Masscan or Hostwrangle... skipping" if VERBOSE
end
end
#Create the OOXML tables
def build_tables(results, targets)
#Gather all Hosts objects and sort by :ip, using a schwartzian transform
host_objs = OpenPorts::Host.all
host_objs.sort_by! { |host| host.ip.split(".").map(&:to_i) }
if host_objs.empty?
puts "[!] No hosts... because no files were parsed... exiting...."
exit
end
bar = TTY::ProgressBar.new("|:bar| :percent", total: host_objs.count, width: 60)
#Interate through each Host object adding class attributes to Caracal array
host_objs.each do |obj|
bar.advance(1)
unless targets.empty?
unless targets.include? obj.ip
next
end
end
ip_col = Caracal::Core::Models::TableCellModel.new do
p obj.ip
end
host_col = Caracal::Core::Models::TableCellModel.new do
if !obj.hostnames.empty?
obj.hostnames.sort!
obj.hostnames.each do |hostname|
p hostname.chomp('.')
end
else
p ''
end
end
ports = []
obj.ports.each do |port|
ports << [port[0][0..-5], port[1][:protocol], port[1][:service]]
end
ports.sort_by! { |port| [port[1].to_s, port[0].to_i] }
port_table = Caracal::Core::Models::TableCellModel.new do
table ports do
cell_style cols[0], width: 780
cell_style cols[1], width: 1000
cell_style cols[2], width: 3320
end
end
results << [ip_col,host_col,port_table]
end
end
targets = []
if options[:targets]
File.open(options[:targets]).each do |line|
targets << line.chomp
end
end
#Iterate through each supplied file and send to file format check
if options[:dir]
scans_hash = {}
dns_hash = {}
Dir.glob(options[:dir] + "/" + "*").each do |file|
file_format = get_format(file) if File.file?(file)
if file_format == "hostwrangle" || file_format == "nmap"
scans_hash[file] = file_format
elsif file_format == "dnsdb_json"
dns_hash[file] = file_format
else
puts "[!] #{file} does not match standard Nmap/Masscan or Hostwrangle... skipping" if VERBOSE
end
end
scans_hash.each do |scan|
run_parse(scan[0], scan[1])
end
dns_hash.each do |dns|
run_parse(dns[0], dns[1])
end
elsif options[:file]
options[:file].each do |file|
file_format = get_format(file)
run_parse(file, file_format) if File.file?(file)
end
end
#Create a DOCX file and instantiate it for editing
Caracal::Document.save DOCX_FILE do |docx|
docx.style do #Default document style
id 'Normal'
name 'Normal'
font 'Trebuchet MS'
size '18' #Measured in half-points (18 = 9pt font)
line '240' #Measured in twips
top '50' #Measured in twips
end
#Create the document header timestamp and options
docx.p do
text'Forward delete the space between the two tables and Word will merge them together.', color: 'FF0000', underline: 'true'
br
br
text 'Then, copy to the report selecting "Keep Source Formatting"'
br
br
text 'Generated: ' + DateTime.now.to_s
br
br
text 'options: ' + options.to_s
br
end
#Build the results table
puts "[+] Building table ..."
puts "[-] Limiting output to targets specified in #{options[:targets]}" if options[:targets]
results = []
build_tables(results, targets)
#Create the DOCX table from the results array
header = ['IP Address', 'Hostname (if resolved)', 'Port', 'Protocol', 'Service']
docx.table [header], border_size: 2 do
border_color '0085C3'
cell_style rows[0], background: '0085C3', color: 'ffffff', bold: true #Header row blue background, white text
cell_style cols[0], width: 1700
cell_style cols[1], width: 2560
cell_style cols[2], width: 780
cell_style cols[3], width: 1000
cell_style cols[4], width: 3320
cell_style cells, margins: { top: 100, bottom: 100, left: 30, right: 0 }
end
puts "[+] Writing to #{DOCX_FILE}"
docx.table results, border_size: 2 do
border_color '0085C3'
cell_style cols[0], bold: true, width: 1700
cell_style cols[1], width: 2560
cell_style cols[2], width: 5100
cell_style cells, margins: { top: 0, bottom: 0, left: 30, right: 0 }
end
end
puts "[+] Done"