From f9513ce1712524dd19b3f9fdc038cd5af0e50742 Mon Sep 17 00:00:00 2001 From: jcran Date: Sun, 19 Mar 2017 20:47:26 -0700 Subject: [PATCH] rework how enrichment works, and how entities can be more structured --- .gitignore | 1 + Procfile | 9 +- Rakefile | 10 +- app/helpers.rb | 14 +- app/models/capabilities/export_graph.rb | 16 +- app/models/entity.rb | 67 ++--- app/models/task_result.rb | 29 +-- app/routes/results.rb | 12 +- app/views/dossier.erb | 24 +- app/views/results/detail.erb | 2 - config/sidekiq-task-enrichment.yml.default | 6 + ..._aliases.rb => 006_drop_alias_mappings.rb} | 7 +- lib/entities/as_number.rb | 4 +- lib/entities/credential.rb | 10 +- lib/entities/dns_server.rb | 4 +- lib/entities/email_address.rb | 4 +- lib/entities/file.rb | 4 +- lib/entities/finger_server.rb | 5 +- lib/entities/ftp_server.rb | 5 +- lib/entities/github_repository.rb | 5 +- lib/entities/github_user.rb | 5 +- lib/entities/host.rb | 16 +- lib/entities/http_header.rb | 5 +- lib/entities/info.rb | 4 +- lib/entities/mongo_service.rb | 12 +- lib/entities/net_block.rb | 6 +- lib/entities/network_service.rb | 8 +- lib/entities/organization.rb | 4 +- lib/entities/person.rb | 4 +- lib/entities/phone_number.rb | 4 +- lib/entities/physical_location.rb | 8 +- lib/entities/screenshot.rb | 5 +- lib/entities/software_package.rb | 4 +- lib/entities/ssh_server.rb | 5 +- lib/entities/ssl_certificate.rb | 4 +- lib/entities/string.rb | 4 +- lib/entities/uri.rb | 8 +- lib/entities/web_account.rb | 8 +- lib/entity_manager.rb | 44 ++-- lib/strategies/discovery.rb | 21 +- .../{get_all_host_names.rb => enrich_host.rb} | 37 ++- lib/tasks/enrich/web_stack_fingerprint.rb | 1 + lib/tasks/helpers/generic.rb | 11 +- lib/tasks/helpers/web.rb | 2 + lib/tasks/nmap_scan.rb | 229 +++++++++--------- lib/tasks/uri_gather_ssl_certificate.rb | 8 +- util/control.sh.default | 2 + 47 files changed, 375 insertions(+), 332 deletions(-) create mode 100644 config/sidekiq-task-enrichment.yml.default rename db/{006_drop_aliases.rb => 006_drop_alias_mappings.rb} (74%) rename lib/tasks/enrich/{get_all_host_names.rb => enrich_host.rb} (67%) diff --git a/.gitignore b/.gitignore index 64b74bf25..388bbb996 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ lib/tasks_disabled lib/tasks/survey db/intrigue.db util/control.sh +config/sidekiq-task-enrichment.yml diff --git a/Procfile b/Procfile index cc2f5bcd9..f7bdaea85 100644 --- a/Procfile +++ b/Procfile @@ -1,4 +1,5 @@ -task-worker: bundle exec sidekiq -C config/sidekiq-task-interactive.yml -r ./core.rb -as-task-worker: bundle exec sidekiq -C config/sidekiq-task-autoscheduled.yml -r ./core.rb -app-worker: bundle exec sidekiq -C config/sidekiq-app.yml -r ./core.rb -web: bundle exec puma -C ./config/puma.rb +interactive: bundle exec sidekiq -C config/sidekiq-task-interactive.yml -r ./core.rb +autoscheduled: bundle exec sidekiq -C config/sidekiq-task-autoscheduled.yml -r ./core.rb +enrichment: bundle exec sidekiq -C config/sidekiq-task-enrichment.yml -r ./core.rb +application: bundle exec sidekiq -C config/sidekiq-app.yml -r ./core.rb +puma: bundle exec puma -C ./config/puma.rb diff --git a/Rakefile b/Rakefile index cf7118557..90b4e55e0 100644 --- a/Rakefile +++ b/Rakefile @@ -13,6 +13,7 @@ system_config_file = "#{intrigue_basedir}/config/config.json" database_config_file = "#{intrigue_basedir}/config/database.yml" sidekiq_interactive_config_file = "#{intrigue_basedir}/config/sidekiq-task-interactive.yml" sidekiq_autoscheduled_config_file = "#{intrigue_basedir}/config/sidekiq-task-autoscheduled.yml" +sidekiq_enrichment_config_file = "#{intrigue_basedir}/config/sidekiq-task-enrichment.yml" sidekiq_app_config_file = "#{intrigue_basedir}/config/sidekiq-app.yml" control_script = "#{intrigue_basedir}/util/control.sh" @@ -28,6 +29,7 @@ task :clean do FileUtils.mv database_config_file, "#{database_config_file}.backup" FileUtils.mv sidekiq_interactive_config_file, "#{sidekiq_interactive_config_file}.backup" FileUtils.mv sidekiq_autoscheduled_config_file, "#{sidekiq_autoscheduled_config_file}.backup" + FileUtils.mv sidekiq_enrichment_config_file, "#{sidekiq_enrichment_config_file}.backup" FileUtils.mv sidekiq_app_config_file, "#{sidekiq_app_config_file}.backup" FileUtils.mv geolocation_database, "#{geolocation_database}.backup" FileUtils.mv web_accounts_list, "#{web_accounts_list}.backup" @@ -83,16 +85,22 @@ task :setup do ## Copy sidekiq task worker config into place puts "[+] Setting up task worker config...." - if File.exist? sidekiq_interactive_config_file && sidekiq_autoscheduled_config_file && sidekiq_app_config_file + if File.exist?(sidekiq_interactive_config_file && + sidekiq_autoscheduled_config_file && + sidekiq_enrichment_config_file && + sidekiq_app_config_file) puts "[ ] File already exists, skipping: #{sidekiq_interactive_config_file}" puts "[ ] File already exists, skipping: #{sidekiq_autoscheduled_config_file}" + puts "[ ] File already exists, skipping: #{sidekiq_enrichment_config_file}" puts "[ ] File already exists, skipping: #{sidekiq_app_config_file}" else puts "[+] Copying: #{sidekiq_interactive_config_file}.default" puts "[+] Copying: #{sidekiq_autoscheduled_config_file}.default" + puts "[+] Copying: #{sidekiq_enrichment_config_file}.default" puts "[+] Copying: #{sidekiq_app_config_file}.default" FileUtils.cp "#{sidekiq_interactive_config_file}.default", sidekiq_interactive_config_file FileUtils.cp "#{sidekiq_autoscheduled_config_file}.default", sidekiq_autoscheduled_config_file + FileUtils.cp "#{sidekiq_enrichment_config_file}.default", sidekiq_enrichment_config_file FileUtils.cp "#{sidekiq_app_config_file}.default", sidekiq_app_config_file end diff --git a/app/helpers.rb b/app/helpers.rb index 25975be56..e415caa6a 100644 --- a/app/helpers.rb +++ b/app/helpers.rb @@ -2,6 +2,18 @@ module Intrigue module Task module Helper + def entity_exists?(project, entity_type, entity_name) + puts "Checking for existence of an entity with type: #{entity_type} and name: #{entity_name} in project: #{project.name}" + Intrigue::Model::Entity.scope_by_project_and_type(project.name,entity_type).each do |e| + if e.unique_names.include? entity_name + puts "Found! #{entity_name}" + return e + end + end + puts "Not Found! #{entity_name}" + false + end + ### ### Helper method for starting a task run ### @@ -16,7 +28,7 @@ def start_task(queue, project, existing_scan_result, task_name, entity, depth, o :options => options, :handlers => [], :base_entity => entity, - :autoscheduled => (queue == "task_autoscheduled"), + :autoscheduled => (queue == "task_autoscheduled" || queue == "task_enrichment"), :depth => depth }) diff --git a/app/models/capabilities/export_graph.rb b/app/models/capabilities/export_graph.rb index 4bd5234d0..8fc3befa5 100644 --- a/app/models/capabilities/export_graph.rb +++ b/app/models/capabilities/export_graph.rb @@ -7,26 +7,30 @@ def export_graph_json # generate the nodes nodes = [] edges = [] - edge_count = 1 + edge_count = 0 self.task_results.each do |t| + next if t.base_entity.type_string == "Uri" + # add the base entity first (provided it hasn't been deleted) x = { :id => t.base_entity.id, :label => "#{t.base_entity.name}", :type => t.base_entity.type_string} - #x[:color] = "lightgrey" if t.base_entity.secondary nodes << x unless t.base_entity.deleted? # then for each of the entities, generate the node and edges. skip if deleted. t.entities.each do |e| - #next unless e.type_string == "WebServer" + + next if e.type_string == "Uri" + x = { :id => e.id, :label => "#{e.name}", :type => e.type_string } #unless e.secondary - #x[:color] = "lightgrey" if e.secondary + #x[:color] = "lightgrey" if e.type_string == "Uri" + nodes << x unless e.deleted? unless t.base_entity.deleted? || e.deleted? - edges << {"id" => edge_count, "source" => t.base_entity.id, "target" => e.id} - edge_count += 1 + edges << {"id" => edge_count += 1, "source" => t.base_entity.id, "target" => e.id} end + end end diff --git a/app/models/entity.rb b/app/models/entity.rb index 299bacd9e..e9ab6b712 100644 --- a/app/models/entity.rb +++ b/app/models/entity.rb @@ -1,35 +1,20 @@ module Intrigue module Model -=begin - class AliasMapping < Sequel::Model - plugin :validation_helpers - #self.raise_on_save_failure = false - - many_to_one :source, :class => :'Intrigue::Model::Entity', :key => :source_id - many_to_one :target, :class => :'Intrigue::Model::Entity', :key => :target_id - - def validate - super - validates_unique([:source_id, :target_id]) # only allow a single alias - end - end -=end class Entity < Sequel::Model plugin :validation_helpers plugin :single_table_inheritance, :type - plugin :serialization, :json, :details + plugin :serialization, :json, :details #, :name self.raise_on_save_failure = false #set_allowed_columns :type, :name, :details, :project_id many_to_many :task_results many_to_one :project - #many_to_many :aliases, :left_key=>:source_id,:right_key=>:target_id, :join_table=>:alias_mappings, :class=>self def validate super - validates_unique([:name, :project_id]) + validates_unique([:project_id, :name]) end def self.scope_by_project(project_name) @@ -42,6 +27,11 @@ def self.scope_by_project_and_type(project, type) where(Sequel.&(:project_id => named_project_id, :type => type.to_s)) end + # easy way to refer to all names (overridden in some entities) + def unique_names + [name] + end + def deleted? return true if deleted false @@ -58,9 +48,6 @@ def created_by?(task_name) false end - def names - end - def allowed_tasks ### XXX - this needs to be limited to tasks that accept this type TaskFactory.allowed_tasks_for_entity_type(type_string) @@ -100,6 +87,10 @@ def form } end + def get_detail(key) + details[key] + end + def self.descendants x = ObjectSpace.each_object(Class).select{ |klass| klass < self } end @@ -114,10 +105,6 @@ def export_hash :name => name, :deleted => deleted, :details => details, - #:aliases => self.aliases.map{|x| { - # "id" => x.id, - # "type" => x.type, - # "name" => x.name }}, :task_results => task_results.map{ |t| {:id => t.id, :name => t.name } } } end @@ -126,30 +113,26 @@ def export_json export_hash.to_json end - # export id, type, name, and details on a single line, removing spaces and commas - def export_csv - export_string = "#{id},#{type_string},#{name.gsub(/[\,,\s]/,"")}," - details.each{|k,v| export_string << "#{k}=#{v};".gsub(/[\,,\s]/,"") } - export_string - end - - def export_tsv - export_string = "#{id}\t#{type_string}\t#{name}\t" - details.each{|k,v| export_string << "#{k}##{v};" } - export_string - end - private def _escape_html(text) Rack::Utils.escape_html(text) text end - # have an easy way to sort and hash all the aliases (which should include - # all names) - def _unique_name - string = aliases.split(",").sort_by{|x| x.downcase}.join(", ") - Digest::SHA1.hexdigest string + + ### VALIDATIONS! + + # https://tools.ietf.org/html/rfc1123 + def _v4_regex + /(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/ + end + + def _v6_regex + /^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$/ + end + + def _dns_regex + /^(\w|-|\.).*\.(\w|-|\.).*$/ end end diff --git a/app/models/task_result.rb b/app/models/task_result.rb index 7e7010e7d..db0aaabaf 100644 --- a/app/models/task_result.rb +++ b/app/models/task_result.rb @@ -23,8 +23,6 @@ def validate def start(queue) if queue == "task_autoscheduled" - autoscheduled = true - self.job_id = Sidekiq::Client.push({ "class" => Intrigue::TaskFactory.create_by_name(task_name).class.to_s, "queue" => "task_autoscheduled", @@ -32,15 +30,23 @@ def start(queue) "args" => [id] }) - save + elsif queue == "task_enrichment" + self.job_id = Sidekiq::Client.push({ + "class" => Intrigue::TaskFactory.create_by_name(task_name).class.to_s, + "queue" => "task_enrichment", + "retry" => true, + "args" => [id] + }) else # start it in the task queues task = Intrigue::TaskFactory.create_by_name(task_name) self.job_id = task.class.perform_async id - save + end - job_id + save + + self.job_id end def log @@ -64,19 +70,6 @@ def task Intrigue::TaskFactory.create_by_name(task_name) end - ### Export! - def export_csv - output_string = "" - entities.each{ |x| output_string << x.export_csv << "\n" } - output_string - end - - def export_tsv - export_string = "" - entities.map{ |x| export_string << x.export_tsv + "\n" } - export_string - end - def export_hash { "id" => id, diff --git a/app/routes/results.rb b/app/routes/results.rb index 12b6957c2..dd01c1968 100644 --- a/app/routes/results.rb +++ b/app/routes/results.rb @@ -20,6 +20,7 @@ class IntrigueApp < Sinatra::Base entity_id = @params["entity_id"] depth = @params["depth"].to_i current_project = Intrigue::Model::Project.first(:name => @project_name) + entity_name = "#{@params["attrib_name"]}" # Construct the attributes hash from the parameters. Loop through each of the # parameters looking for things that look like attributes, and add them to our @@ -32,7 +33,7 @@ class IntrigueApp < Sinatra::Base end end - # hack! remove the name, no longer needed in the details + # HACK! add the name to aliases, name detail no longer needed entity_details.delete("name") # Construct an entity from the data we have @@ -43,17 +44,14 @@ class IntrigueApp < Sinatra::Base return unless entity_type # TODO - SECURITY - validate that it's a valid entity type before we eval - klass = eval("Intrigue::Entity::#{entity_type}") - entity_name = "#{@params["attrib_name"]}" - # TODO - we'll need to check all aliases of all entities within the project here - entity = Intrigue::Model::Entity.scope_by_project_and_type(@project_name, klass.to_s).first(:name => entity_name) + entity = entity_exists?(current_project,entity_type,entity_name) unless entity entity = Intrigue::Model::Entity.create( - { :type => klass, - :name => entity_name, + { :name => entity_name, + :type => klass, :details => entity_details, :project => current_project }) diff --git a/app/views/dossier.erb b/app/views/dossier.erb index 074486576..98b4dec51 100644 --- a/app/views/dossier.erb +++ b/app/views/dossier.erb @@ -3,28 +3,28 @@

People:

Email:

Phone Numbers:

Software:

@@ -32,7 +32,7 @@ @@ -41,48 +41,48 @@ <% @uris.sort{|x,y| "#{x.details["stack"]}" <=> "#{y.details["stack"]}"}.each do |e| next unless e.details["stack"] next unless e.details["stack"].count > 0 %> -
  • <%= h "#{e.details["stack"]}: ... #{e.name}" %>
    • +
  • <%= h "#{e.details["stack"]}: ... #{e.unique_names}" %>
    • <% end %>

    SSL Certificates:

    Uris:

    Network Services:

    Hosts:

    Networks:

    Other:

    diff --git a/app/views/results/detail.erb b/app/views/results/detail.erb index 706a43671..4fd1c1459 100644 --- a/app/views/results/detail.erb +++ b/app/views/results/detail.erb @@ -44,9 +44,7 @@ getEntities(<%=@result.id%>);

    Complete: <%= @result.complete %>

    Export: Entities:
    diff --git a/config/sidekiq-task-enrichment.yml.default b/config/sidekiq-task-enrichment.yml.default new file mode 100644 index 000000000..84e62b5e9 --- /dev/null +++ b/config/sidekiq-task-enrichment.yml.default @@ -0,0 +1,6 @@ +:verbose: false +#:logfile: ./log/task-enrichment.log # this is commented bc we supply it on the cli +#:pidfile: ./tmp/pids/task-enrichment.pid # this is commented bc we supply it on the cli +:concurrency: 10 +:queues: + - task_enrichment diff --git a/db/006_drop_aliases.rb b/db/006_drop_alias_mappings.rb similarity index 74% rename from db/006_drop_aliases.rb rename to db/006_drop_alias_mappings.rb index 6fd706417..fa3738598 100644 --- a/db/006_drop_aliases.rb +++ b/db/006_drop_alias_mappings.rb @@ -1,11 +1,12 @@ Sequel.migration do change do - #alter_table :entities do - # String :names, :text => true - #end drop_table :alias_mappings + #alter_table :entities do + # String :aliases, :text => true + #end + end end diff --git a/lib/entities/as_number.rb b/lib/entities/as_number.rb index 4a364c53e..778c15d0c 100644 --- a/lib/entities/as_number.rb +++ b/lib/entities/as_number.rb @@ -9,8 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ + def validate_entity + name =~ /^\w.*$/ end end diff --git a/lib/entities/credential.rb b/lib/entities/credential.rb index c5711f4b7..c01ea9b77 100644 --- a/lib/entities/credential.rb +++ b/lib/entities/credential.rb @@ -9,11 +9,11 @@ def self.metadata } end - def validate_content - @name =~ /^.*/ && - @details["username"].to_s =~ /^.*$/ && - @details["password"].to_s =~ /^.*$/ && - @details["uri"].to_s =~ /^.*$/ + def validate_entity + name =~ /^\w.*/ && + details["username"].to_s =~ /^\w.*$/ && + details["password"].to_s =~ /^\w.*$/ && + details["uri"].to_s =~ /^\w.*$/ end end diff --git a/lib/entities/dns_server.rb b/lib/entities/dns_server.rb index 14c07be17..53c32d0e5 100644 --- a/lib/entities/dns_server.rb +++ b/lib/entities/dns_server.rb @@ -9,8 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^[a-zA-Z0-9\.].*/ + def validate_entity + (name =~ _v4_regex || name =~ _v6_regex || name == _dns_regex) && details["port"].to_s =~ /^\d{1,5}$/ end end diff --git a/lib/entities/email_address.rb b/lib/entities/email_address.rb index d01bfbcc1..d2ad605a2 100644 --- a/lib/entities/email_address.rb +++ b/lib/entities/email_address.rb @@ -10,8 +10,8 @@ def self.metadata end - def validate_content - @name =~ /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,8}/ + def validate_entity + name =~ /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,8}/ end end diff --git a/lib/entities/file.rb b/lib/entities/file.rb index 009a53193..ef594ff5c 100644 --- a/lib/entities/file.rb +++ b/lib/entities/file.rb @@ -9,8 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ + def validate_entity + name =~ /^\w.*$/ end end diff --git a/lib/entities/finger_server.rb b/lib/entities/finger_server.rb index 3b2c2de69..57eba4ec7 100644 --- a/lib/entities/finger_server.rb +++ b/lib/entities/finger_server.rb @@ -9,9 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^[a-zA-Z0-9\.\:\/\ ].*/ && - @details["port"].to_s =~ /^\d{1,5}$/ + def validate_entity + (name =~ _v4_regex || name =~ _v6_regex || name == _dns_regex) && details["port"].to_s =~ /^\d{1,5}$/ end end diff --git a/lib/entities/ftp_server.rb b/lib/entities/ftp_server.rb index 8eba56b7e..87cae49d7 100644 --- a/lib/entities/ftp_server.rb +++ b/lib/entities/ftp_server.rb @@ -9,9 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^[a-zA-Z0-9\.\:\/\ ].*/ && - @details["port"].to_s =~ /^\d{1,5}$/ + def validate_entity + (name =~ _v4_regex || name =~ _v6_regex || name == _dns_regex) && details["port"].to_s =~ /^\d{1,5}$/ end end diff --git a/lib/entities/github_repository.rb b/lib/entities/github_repository.rb index 87202ff31..dca8cc8a4 100644 --- a/lib/entities/github_repository.rb +++ b/lib/entities/github_repository.rb @@ -9,9 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*/ && - @uri =~ /^.*/ + def validate_entity + name =~ /^\w.*/ && details["uri"] =~ /^\w.*/ end end diff --git a/lib/entities/github_user.rb b/lib/entities/github_user.rb index 89b71de86..4661177d7 100644 --- a/lib/entities/github_user.rb +++ b/lib/entities/github_user.rb @@ -9,9 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*/ && - @uri =~ /^.*/ + def validate_entity + name =~ /^\w.*/ && details["uri"] =~ /^.*/ end end diff --git a/lib/entities/host.rb b/lib/entities/host.rb index 9d99bb59c..abf322c73 100644 --- a/lib/entities/host.rb +++ b/lib/entities/host.rb @@ -9,17 +9,19 @@ def self.metadata } end - def validate_content - - v4_regex = /(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/ - v6_regex = /^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$/ + def validate_entity + return (name =~ _v4_regex || name =~ _v6_regex || name =~ _dns_regex) + end - # https://tools.ietf.org/html/rfc1123 - dns_regex = /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/ + def unique_names + x = [name] + x.concat(details["dns_names"]) if details["dns_names"] + x.concat(details["ip_addresses"]) if details["ip_addresses"] - return (@name =~ v4_regex || @name =~ v6_regex || @name == dns_regex) + x.sort.uniq end + end end end diff --git a/lib/entities/http_header.rb b/lib/entities/http_header.rb index ed4fafea9..615dd5559 100644 --- a/lib/entities/http_header.rb +++ b/lib/entities/http_header.rb @@ -9,9 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ && - @details["content"] =~ /^.*$/ + def validate_entity + name =~ /^\w.*$/ && details["content"] =~ /^.*$/ end end diff --git a/lib/entities/info.rb b/lib/entities/info.rb index 95d8804c9..3e7768fe4 100644 --- a/lib/entities/info.rb +++ b/lib/entities/info.rb @@ -9,8 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ + def validate_entity + name =~ /^\w.*$/ end end diff --git a/lib/entities/mongo_service.rb b/lib/entities/mongo_service.rb index db56e1e4b..5239b03fb 100644 --- a/lib/entities/mongo_service.rb +++ b/lib/entities/mongo_service.rb @@ -10,17 +10,15 @@ def self.metadata end - def validate_content - @details["ip_address"].to_s =~ /^.*$/ && - @details["port"].to_s =~ /^\d{1,5}$/ && - @details["proto"].to_s =~ /^(tcp|udp)$/ + def validate_entity + (name =~ _v4_regex || name =~ _v6_regex || name == _dns_regex) && details["port"].to_s =~ /^\d{1,5}$/ end def form output = super - output << "
    " - output << "
    " - output << "
    " + output << "
    " + output << "
    " + output << "
    " output end diff --git a/lib/entities/net_block.rb b/lib/entities/net_block.rb index c9ca84b73..539437b80 100644 --- a/lib/entities/net_block.rb +++ b/lib/entities/net_block.rb @@ -10,13 +10,13 @@ def self.metadata end - def validate_content + def validate_entity # required: - @name =~ /^.*$/ + name =~ /^\w.*$/ # suggested: - # @details["organization_reference"] + # details["organization_reference"] end diff --git a/lib/entities/network_service.rb b/lib/entities/network_service.rb index 367daf124..83865bd4f 100644 --- a/lib/entities/network_service.rb +++ b/lib/entities/network_service.rb @@ -10,10 +10,10 @@ def self.metadata end - def validate_content - @details["ip_address"].to_s =~ /^.*$/ && - @details["port"].to_s =~ /^\d{1,5}$/ && - @details["proto"].to_s =~ /^(tcp|udp)$/ + def validate_entity + (details["ip_address"].to_s =~ _v4_regex || details["ip_address"].to_s =~ _v6_regex) && + details["port"].to_s =~ /^\d{1,5}$/ && + details["proto"].to_s =~ /^(tcp|udp)$/ end def form diff --git a/lib/entities/organization.rb b/lib/entities/organization.rb index 3f3074d14..8acdfd20e 100644 --- a/lib/entities/organization.rb +++ b/lib/entities/organization.rb @@ -10,8 +10,8 @@ def self.metadata end - def validate_content - @name =~ /^.*$/ + def validate_entity + name =~ /^\w.*$/ end end diff --git a/lib/entities/person.rb b/lib/entities/person.rb index dd9e6f99c..edf0e7a79 100644 --- a/lib/entities/person.rb +++ b/lib/entities/person.rb @@ -10,8 +10,8 @@ def self.metadata end - def validate_content - @name =~ /^.*$/ + def validate_entity + name =~ /^\w.*$/ end end diff --git a/lib/entities/phone_number.rb b/lib/entities/phone_number.rb index c8da5105e..2ebc072ca 100644 --- a/lib/entities/phone_number.rb +++ b/lib/entities/phone_number.rb @@ -9,8 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ + def validate_entity + name =~ /^\w.*$/ end end diff --git a/lib/entities/physical_location.rb b/lib/entities/physical_location.rb index 6fa7aca94..12b03e0f0 100644 --- a/lib/entities/physical_location.rb +++ b/lib/entities/physical_location.rb @@ -9,10 +9,10 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ #&& - #@details["latitude"] =~ /^([-+]?\d{1,2}[.]\d+)$/ && - #@details["longitude"] =~ /^([-+]?\d{1,3}[.]\d+)$/ + def validate_entity + name =~ /^\w.*$/ #&& + #details["latitude"] =~ /^([-+]?\d{1,2}[.]\d+)$/ && + #details["longitude"] =~ /^([-+]?\d{1,3}[.]\d+)$/ end end diff --git a/lib/entities/screenshot.rb b/lib/entities/screenshot.rb index 5ed418bc5..bafe7d980 100644 --- a/lib/entities/screenshot.rb +++ b/lib/entities/screenshot.rb @@ -9,9 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ # XXX - too loose - #@details[:file] =~ /^.*$/ # XXX - too loose + def validate_entity + name =~ /^\w.*$/ # TODO - tighten this up end end diff --git a/lib/entities/software_package.rb b/lib/entities/software_package.rb index 35850f58a..81278c4af 100644 --- a/lib/entities/software_package.rb +++ b/lib/entities/software_package.rb @@ -9,8 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ + def validate_entity + name =~ /^\w.*$/ end end diff --git a/lib/entities/ssh_server.rb b/lib/entities/ssh_server.rb index a284a9f49..ff3e3c908 100644 --- a/lib/entities/ssh_server.rb +++ b/lib/entities/ssh_server.rb @@ -9,9 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^[a-zA-Z0-9\.\:\/\ ].*/ && - @details["port"].to_s =~ /^\d{1,5}$/ + def validate_entity + (name =~ _v4_regex || name =~ _v6_regex || name == _dns_regex) && details["port"].to_s =~ /^\d{1,5}$/ end end diff --git a/lib/entities/ssl_certificate.rb b/lib/entities/ssl_certificate.rb index 326c9481f..a48b21e1b 100644 --- a/lib/entities/ssl_certificate.rb +++ b/lib/entities/ssl_certificate.rb @@ -9,8 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ + def validate_entity + name =~ /^.*$/ end end diff --git a/lib/entities/string.rb b/lib/entities/string.rb index f8958e597..307316b58 100644 --- a/lib/entities/string.rb +++ b/lib/entities/string.rb @@ -9,8 +9,8 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ + def validate_entity + name =~ /^\w.*$/ end end diff --git a/lib/entities/uri.rb b/lib/entities/uri.rb index fc484d8f0..bbad92c29 100644 --- a/lib/entities/uri.rb +++ b/lib/entities/uri.rb @@ -5,14 +5,12 @@ class Uri < Intrigue::Model::Entity def self.metadata { :name => "Uri", - :description => "TODO" + :description => "A Uniform Resource Identifier (URI) is a string of characters used to identify a resource." } end - def validate_content - @name =~ /^.*$/ #&& - #@details["web_application"] && - #@details["web_server"] + def validate_entity + name =~ /^\w.*$/ end end diff --git a/lib/entities/web_account.rb b/lib/entities/web_account.rb index 69bb2f4bd..988613deb 100644 --- a/lib/entities/web_account.rb +++ b/lib/entities/web_account.rb @@ -9,10 +9,10 @@ def self.metadata } end - def validate_content - @name =~ /^.*$/ && - @details["domain"] =~ /^.*$/ && - @details["uri"] =~ /^http.*$/ + def validate_entity + name =~ /^\w.*$/ && + details["domain"] =~ /^.*$/ && + details["uri"] =~ /^http.*$/ end end diff --git a/lib/entity_manager.rb b/lib/entity_manager.rb index 2b59fa7c9..6d3b64383 100644 --- a/lib/entity_manager.rb +++ b/lib/entity_manager.rb @@ -6,15 +6,7 @@ class EntityManager # NOTE: We don't auto-register entities like the other factories (handled by # single table inheritance) - # NOTE: The user's desired depth of recursion is stored on the task_result. - - def self.entity_exists?(type,name) - return true if Intrigue::Model::Entity.first(:name=>name,:type=>type) - false - end - def self.resolve_type(type_string) - # TODO - SECURITY - don't eval unless it's one of our valid entity types x = eval("Intrigue::Entity::#{type_string}") false unless x.kind_of? Intrigue::Model::Entity @@ -22,28 +14,26 @@ def self.resolve_type(type_string) end # This method creates a new entity, and kicks off a strategy - def self.create_or_merge_entity_recursive(task_result,type_string,name,details) + def self.create_or_merge_entity(task_result,type_string,name,details) project = task_result.project # convenience + downcased_name = name.downcase # Clean up in case there are encoding issues - name = _encode_string(name) - details = _encode_hash(details.merge(:aliases => ["#{name}"])) + #name = _encode_string(name) + #details = _encode_hash(details.merge(:aliases => "#{name}"])) type = resolve_type(type_string) # Merge the details if it already exists - entity = nil - entity = Intrigue::Model::Entity.scope_by_project_and_type(project.name,type).first(:name => name) - - if entity.kind_of? Intrigue::Model::Entity - # We're going to have to look for each of the aliases as well. - # deep_merge_aliases(?) - entity.details = details.merge(entity.details) + entity = entity_exists?(project,type,downcased_name) + if entity + # TODO - DEEP MERGE + entity.details = details.deep_merge(entity.details) entity.save else # Create a new entity, validating the attributes entity = Intrigue::Model::Entity.create({ - :name => name, + :name => downcased_name, :project => project, :type => type, :details => details @@ -51,10 +41,16 @@ def self.create_or_merge_entity_recursive(task_result,type_string,name,details) end unless entity - puts "ERROR! Unable to create entity: #{type}##{name}" + puts "ERROR! Unable to create or find entity: #{type}##{downcased_name}" return nil end + unless Intrigue::Model::Entity.find(:id => entity.id).validate_entity + puts "ERROR! validation of entity failed: #{entity}" + return nil + end + + # Add to our result set for this task task_result.add_entity entity task_result.save @@ -69,16 +65,18 @@ def self.create_or_merge_entity_recursive(task_result,type_string,name,details) # START PROCESSING OF ENRICHMENT (to depth of 1) if task_result.depth > 0 - unless prohibited_entity? entity + #unless prohibited_entity? entity if entity.type_string == "Host" - start_task("task_autoscheduled", project, task_result.scan_result, "get_all_host_names", entity, 1, [],[]) + start_task("task_enrichment", project, task_result.scan_result, "enrich_host", entity, 1, [],[]) elsif entity.type_string == "Uri" start_task("task_autoscheduled", project, task_result.scan_result, "check_api_endpoint", entity, 1, [],[]) start_task("task_autoscheduled", project, task_result.scan_result, "web_stack_fingerprint", entity, 1, [],[]) end - end + #end end# END PROCESSING OF ENRICHMENT + #sleep 3 # give time for the enrichment to complete in case we don't have a backlog + # START PROCESSING OF RECURSION BY STRATEGY TYPE scan_result = task_result.scan_result if scan_result && task_result.depth > 0 # if this is a scan and we're within depth diff --git a/lib/strategies/discovery.rb b/lib/strategies/discovery.rb index 1edabe7ef..95186f123 100644 --- a/lib/strategies/discovery.rb +++ b/lib/strategies/discovery.rb @@ -6,10 +6,23 @@ def self.recurse(entity, task_result) if entity.type_string == "Host" + # Wait until enrichment has happened before going further + entity_id = entity.id + countdown = 10 # HACK + while !entity.get_detail("enriched") && countdown > 0 do + puts "waiting for #{entity} enrichment: #{entity.get_detail("enriched")} (#{countdown})" + #puts "#{entity.id}" + #puts "#{entity.details.inspect}" + sleep 3 + countdown -= 1 + entity = Intrigue::Model::Entity.find(:id => entity_id) + end + start_recursive_task(task_result,"nmap_scan",entity) ### DNS Subdomain Bruteforce # Do a big bruteforce if the size is small enough + # Do a big bruteforce if the size is small enough if (entity.name.split(".").length < 3) start_recursive_task(task_result,"dns_brute_sub",entity,[ {"name" => "use_file", "value" => true }]) @@ -29,22 +42,22 @@ def self.recurse(entity, task_result) elsif entity.type_string == "NetBlock" # Make sure it's small enough not to be disruptive, and if it is, scan it - cidr = entity.name.split("/").last.to_i - if cidr >= 16 + #cidr = entity.name.split("/").last.to_i + #if cidr >= 16 start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 21}]) start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 80}]) start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 443}]) start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 8080}]) start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 8081}]) start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 8443}]) - end + #end elsif entity.type_string == "Uri" ## Grab the Web Server ## Grab the SSL Certificate - start_recursive_task(task_result,"uri_gather_ssl_certificate",entity) if entity.name =~ /^https/ + start_recursive_task(task_result,"uri_gather_ssl_certificate",entity) #if entity.name =~ /^https/ ## Spider, looking for metadata start_recursive_task(task_result,"uri_spider",entity,[ diff --git a/lib/tasks/enrich/get_all_host_names.rb b/lib/tasks/enrich/enrich_host.rb similarity index 67% rename from lib/tasks/enrich/get_all_host_names.rb rename to lib/tasks/enrich/enrich_host.rb index 0a410d779..769fe3535 100644 --- a/lib/tasks/enrich/get_all_host_names.rb +++ b/lib/tasks/enrich/enrich_host.rb @@ -1,12 +1,12 @@ require 'dnsruby' module Intrigue -class GetAllHostNames < BaseTask +class EnrichHost < BaseTask def self.metadata { - :name => "get_all_host_names", - :pretty_name => "Get All Host Names", + :name => "enrich_host", + :pretty_name => "Enrich Host", :authors => ["jcran"], :description => "Look up all names of a given entity.", :references => [], @@ -27,6 +27,14 @@ def run opt_resolver = _get_option "resolver" lookup_name = _get_entity_name + ip_addresses = [] + dns_names = [] + if lookup_name.is_ip_address? + ip_addresses << lookup_name + else + dns_names << lookup_name + end + begin resolver = Dnsruby::Resolver.new( :recurse => "true", @@ -41,25 +49,34 @@ def run _log_error "Nothing?" if result.answer.empty? # For each of the found addresses - names = [] result.answer.map do |resource| next if resource.type == Dnsruby::Types::RRSIG #TODO parsing this out is a pain, not sure if it's valuable _log "Adding name from: #{resource}" - names << resource.address.to_s if resource.respond_to? :address - names << resource.name.to_s + ip_addresses << resource.address.to_s if resource.respond_to? :address + dns_names << resource.name.to_s.downcase end #end result.answer - @entity.update(:details => @entity.details.merge("aliases" => names.sort.uniq)) - @entity.save - + rescue Dnsruby::ServFail => e + _log_error "Unable to resolve: #{@entity}, error: #{e}" rescue Dnsruby::NXDomain => e - _log_error "Unable to resolve: #{e}" + _log_error "Unable to resolve: #{@entity}, error: #{e}" rescue Dnsruby::ResolvTimeout => e _log_error "Unable to resolve, timed out: #{e}" rescue Errno::ENETUNREACH => e _log_error "Hit exception: #{e}. Are you sure you're connected?" + #rescue Exception => e # _log_error "Hit exception: #{e}" + ensure + + temp_details = @entity.details + temp_details["ip_addresses"] = ip_addresses.sort.uniq + temp_details["dns_names"] = dns_names.sort.uniq + temp_details["enriched"] = true + + @entity.update(:details => temp_details) + @entity.save + end _log "Ran enrichment task!" diff --git a/lib/tasks/enrich/web_stack_fingerprint.rb b/lib/tasks/enrich/web_stack_fingerprint.rb index 2d221bb99..c52b09226 100644 --- a/lib/tasks/enrich/web_stack_fingerprint.rb +++ b/lib/tasks/enrich/web_stack_fingerprint.rb @@ -214,6 +214,7 @@ def _check_cookies(response) temp << "ASP.NET" if header =~ /^.*ASPSESSIONID.*$/ temp << "ASP.NET" if header =~ /^.*ASP.NET_SessionId.*$/ temp << "BEA WebLogic" if header =~ /^.*WebLogicSession*$/ + temp << "BigIP" if header =~ /^.*BIGipServer*$/ temp << "Coldfusion" if header =~ /^.*CFID.*$/ temp << "Coldfusion" if header =~ /^.*CFTOKEN.*$/ temp << "Coldfusion" if header =~ /^.*CFGLOBALS.*$/ diff --git a/lib/tasks/helpers/generic.rb b/lib/tasks/helpers/generic.rb index d1e72fdb6..2d5c97dc0 100644 --- a/lib/tasks/helpers/generic.rb +++ b/lib/tasks/helpers/generic.rb @@ -4,15 +4,16 @@ module Generic private + ### + ### Helper method to reach out to the entity manager + ### def _create_entity(type, hash) - # NOTE: this is a hack - the _create_entity call should be updated in each task - # to create the entity in a form that's ready for create_or_merge_entity_recursive - name = hash["name"] # Pull out the name from the hash - hash.delete("name") # No need for a name in the hash now, remove it + # No need for a name in the hash now, remove it & pull out the name from the hash + name = hash.delete("name") # Create or merge the entity - entity = EntityManager.create_or_merge_entity_recursive(@task_result, type, name, hash) + EntityManager.create_or_merge_entity(@task_result, type, name, hash) end ### diff --git a/lib/tasks/helpers/web.rb b/lib/tasks/helpers/web.rb index bdfc36239..6ff9eb0da 100644 --- a/lib/tasks/helpers/web.rb +++ b/lib/tasks/helpers/web.rb @@ -178,6 +178,8 @@ def http_get(uri_string, headers={}, limit = 10, open_timeout=15, read_timeout=1 @task_result.logger.log_error "Timeout : #{e}" if @task_result rescue Net::ReadTimeout => e @task_result.logger.log_error "Timeout : #{e}" if @task_result + rescue Errno::ETIMEDOUT => e + @task_result.logger.log_error "Timeout : #{e}" if @task_result rescue Errno::ENETUNREACH => e @task_result.logger.log_error "Unable to connect: #{e}" if @task_result rescue URI::InvalidURIError => e diff --git a/lib/tasks/nmap_scan.rb b/lib/tasks/nmap_scan.rb index 025495a4a..2b4717403 100644 --- a/lib/tasks/nmap_scan.rb +++ b/lib/tasks/nmap_scan.rb @@ -33,119 +33,126 @@ def run #ports = _get_option "ports" # Get range, or host - to_scan = _get_entity_name - - # Create a tempfile to store results - temp_file = "#{Dir::tmpdir}/nmap_scan_#{rand(100000000)}.xml" - - # Check for IPv6 - nmap_options = "" - nmap_options << "-6 " if to_scan =~ /:/ - - # shell out to nmap and run the scan - _log "Scanning #{to_scan} and storing in #{temp_file}" - _log "NMap options: #{nmap_options}" - nmap_string = "nmap #{to_scan} #{nmap_options} -P0 --top-ports 100 --min-parallelism 10 -oX #{temp_file}" - _log "Running... #{nmap_string}" - _unsafe_system(nmap_string) - - # Gather the XML and parse - #_log "Raw Result:\n #{File.open(temp_file).read}" - _log "Parsing #{temp_file}" - - parser = Nmap::XML.new(temp_file) - - # Create entities for each discovered service - parser.each_host do |host| - _log "Handling nmap data for #{host.ip}" + if @entity.type_string == "NetBlock" + ip_addresses = [_get_entity_name] + else + ip_addresses = @entity.details["ip_addresses"] + return unless ip_addresses + end - # Handle the case of a netblock or domain - where we will need to create host entity(s) - if @entity.type_string == "NetBlock" or @entity.type_string == "Host" - # Only create if we've got ports to report. - _create_entity("Host", { "name" => host.ip } ) if host.ports.count > 0 + ip_addresses.each do |to_scan| + # Create a tempfile to store results + temp_file = "#{Dir::tmpdir}/nmap_scan_#{rand(100000000)}.xml" + + # Check for IPv6 + nmap_options = "" + nmap_options << "-6 " if to_scan =~ /:/ + + # shell out to nmap and run the scan + _log "Scanning #{to_scan} and storing in #{temp_file}" + _log "NMap options: #{nmap_options}" + nmap_string = "nmap #{to_scan} #{nmap_options} -P0 --top-ports 100 --min-parallelism 10 -oX #{temp_file}" + _log "Running... #{nmap_string}" + _unsafe_system(nmap_string) + + # Gather the XML and parse + #_log "Raw Result:\n #{File.open(temp_file).read}" + _log "Parsing #{temp_file}" + + parser = Nmap::XML.new(temp_file) + + # Create entities for each discovered service + parser.each_host do |host| + _log "Handling nmap data for #{host.ip}" + + # Handle the case of a netblock or domain - where we will need to create host entity(s) + if @entity.type_string == "NetBlock" #or @entity.type_string == "Host" + # Only create if we've got ports to report. + _create_entity("Host", { "name" => host.ip } ) if host.ports.count > 0 + end + + host.each_port do |port| + if port.state == :open + + # Handle WebApps first + if port.protocol == :tcp && + [80,443,8080,8081,8443].include?(port.number) + + # determine if this is an SSL application + ssl = true if [443,8443].include?(port.number) + protocol = ssl ? "https://" : "http://" # construct uri + + # Create URI + uri = "#{protocol}#{host.ip}:#{port.number}" + _create_entity("Uri", "name" => uri, "uri" => uri ) # create an entity + + # and create the entities if we have dns resolution + @entity.details["dns_names"].each do |hostname| + uri = "#{protocol}#{hostname}:#{port.number}" + _create_entity("Uri", "name" => uri, "uri" => uri ) + end + + # then FtpServer + elsif [21].include?(port.number) + uri = "ftp://#{host.ip}:#{port.number}" + _create_entity("FtpServer", { + "name" => uri, + "ip_address" => "#{host.ip}", + "port" => port.number, + "proto" => port.protocol, + "uri" => uri }) + + # Then SshServer + elsif [22].include?(port.number) + uri = "ssh://#{host.ip}:#{port.number}" + _create_entity("SshServer", { + "name" => uri, + "ip_address" => "#{host.ip}", + "port" => port.number, + "proto" => port.protocol, + "uri" => uri }) + + # then DnsServer + elsif [53].include?(port.number) + uri = "#{host.ip}:#{port.number}" + _create_entity("DnsServer", { + "name" => uri, + "ip_address" => "#{host.ip}", + "port" => port.number, + "proto" => port.protocol, + "uri" => uri }) + + # then FingerServer + elsif [79].include?(port.number) + uri = "finger://#{host.ip}:#{port.number}" + _create_entity("FingerServer", { + "name" => uri, + "ip_address" => "#{host.ip}", + "port" => port.number, + "proto" => port.protocol, + "uri" => uri }) + + # Otherwise default to an unknown network service + else + + _create_entity("NetworkService", { + "name" => "#{host.ip}:#{port.number}/#{port.protocol}", + "ip_address" => "#{host.ip}", + "port" => port.number, + "proto" => port.protocol, + "fingerprint" => "#{port.service}"}) + + end # end if + end # end if port.state == :open + end # end host.each_port + end # end parser + + # Clean up! + begin + File.delete(temp_file) + rescue Errno::EPERM + _log_error "Unable to delete file" end - - host.each_port do |port| - if port.state == :open - - # Handle WebApps first - if port.protocol == :tcp && - [80,443,8080,8081,8443].include?(port.number) - - # determine if this is an SSL application - ssl = true if [443,8443].include?(port.number) - protocol = ssl ? "https://" : "http://" # construct uri - - # Create URI - uri = "#{protocol}#{host.ip}:#{port.number}" - _create_entity("Uri", "name" => uri, "uri" => uri ) # create an entity - - # and create the entities if we have dns resolution - host.hostnames.each do |hostname| - uri = "#{protocol}#{hostname}:#{port.number}" - _create_entity("Uri", "name" => uri, "uri" => uri ) - end - - # then FtpServer - elsif [21].include?(port.number) - uri = "ftp://#{host.ip}:#{port.number}" - _create_entity("FtpServer", { - "name" => uri, - "ip_address" => "#{host.ip}", - "port" => port.number, - "proto" => port.protocol, - "uri" => uri }) - - # Then SshServer - elsif [22].include?(port.number) - uri = "ssh://#{host.ip}:#{port.number}" - _create_entity("SshServer", { - "name" => uri, - "ip_address" => "#{host.ip}", - "port" => port.number, - "proto" => port.protocol, - "uri" => uri }) - - # then DnsServer - elsif [53].include?(port.number) - uri = "#{host.ip}:#{port.number}" - _create_entity("DnsServer", { - "name" => uri, - "ip_address" => "#{host.ip}", - "port" => port.number, - "proto" => port.protocol, - "uri" => uri }) - - # then FingerServer - elsif [79].include?(port.number) - uri = "finger://#{host.ip}:#{port.number}" - _create_entity("FingerServer", { - "name" => uri, - "ip_address" => "#{host.ip}", - "port" => port.number, - "proto" => port.protocol, - "uri" => uri }) - - # Otherwise default to an unknown network service - else - - _create_entity("NetworkService", { - "name" => "#{host.ip}:#{port.number}/#{port.protocol}", - "ip_address" => "#{host.ip}", - "port" => port.number, - "proto" => port.protocol, - "fingerprint" => "#{port.service}"}) - - end # end if - end # end if port.state == :open - end # end host.each_port - end # end parser - - # Clean up! - begin - File.delete(temp_file) - rescue Errno::EPERM - _log_error "Unable to delete file" end end diff --git a/lib/tasks/uri_gather_ssl_certificate.rb b/lib/tasks/uri_gather_ssl_certificate.rb index c0d506388..1bc898bdb 100644 --- a/lib/tasks/uri_gather_ssl_certificate.rb +++ b/lib/tasks/uri_gather_ssl_certificate.rb @@ -63,8 +63,10 @@ def run x.gsub(/DNS:/,"").strip end - alt_names.each do |alt_name| + _log "Got alt_names: #{alt_names.inspect}" + # Iterate through, looking for trouble + alt_names.each do |alt_name| if alt_name =~ /cloudflare.com$/ && opt_skip_cloudflare _log "This is a cloudflare certificate, skipping further entity creation" return @@ -94,6 +96,10 @@ def run _log "This is a wpengine certificate, skipping further entity creation" return end + end + + #assuming we made it this far, let's proceed + alt_names.each do |alt_name| # Remove any leading wildcards so we get a sensible domain name if alt_name[0..1] == "*." diff --git a/util/control.sh.default b/util/control.sh.default index 1f327a8ea..d40cca2d9 100755 --- a/util/control.sh.default +++ b/util/control.sh.default @@ -22,6 +22,8 @@ function start_server { bundle exec sidekiq -C $IDIR/config/sidekiq-task-interactive.yml -r $IDIR/core.rb -d -L $IDIR/log/task-interactive.log echo "Starting autoscheduled task processing..." bundle exec sidekiq -C $IDIR/config/sidekiq-task-autoscheduled.yml -r $IDIR/core.rb -d -L $IDIR/log/task-autoscheduled.log -P $IDIR/tmp/pids/sidekiq-task-autoscheduled.pid + echo "Starting enrichment task processing..." + bundle exec sidekiq -C $IDIR/config/sidekiq-task-enrichment.yml -r $IDIR/core.rb -d -L $IDIR/log/task-enrichment.log -P $IDIR/tmp/pids/sidekiq-task-enrichment.pid echo "Starting background worker processing..." bundle exec sidekiq -C $IDIR/config/sidekiq-app.yml -r $IDIR/core.rb -d -L $IDIR/log/app.log echo "Starting puma..."