diff --git a/.gitignore b/.gitignore
index 64b74bf25..388bbb996 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,3 +21,4 @@ lib/tasks_disabled
lib/tasks/survey
db/intrigue.db
util/control.sh
+config/sidekiq-task-enrichment.yml
diff --git a/Procfile b/Procfile
index cc2f5bcd9..f7bdaea85 100644
--- a/Procfile
+++ b/Procfile
@@ -1,4 +1,5 @@
-task-worker: bundle exec sidekiq -C config/sidekiq-task-interactive.yml -r ./core.rb
-as-task-worker: bundle exec sidekiq -C config/sidekiq-task-autoscheduled.yml -r ./core.rb
-app-worker: bundle exec sidekiq -C config/sidekiq-app.yml -r ./core.rb
-web: bundle exec puma -C ./config/puma.rb
+interactive: bundle exec sidekiq -C config/sidekiq-task-interactive.yml -r ./core.rb
+autoscheduled: bundle exec sidekiq -C config/sidekiq-task-autoscheduled.yml -r ./core.rb
+enrichment: bundle exec sidekiq -C config/sidekiq-task-enrichment.yml -r ./core.rb
+application: bundle exec sidekiq -C config/sidekiq-app.yml -r ./core.rb
+puma: bundle exec puma -C ./config/puma.rb
diff --git a/Rakefile b/Rakefile
index cf7118557..90b4e55e0 100644
--- a/Rakefile
+++ b/Rakefile
@@ -13,6 +13,7 @@ system_config_file = "#{intrigue_basedir}/config/config.json"
database_config_file = "#{intrigue_basedir}/config/database.yml"
sidekiq_interactive_config_file = "#{intrigue_basedir}/config/sidekiq-task-interactive.yml"
sidekiq_autoscheduled_config_file = "#{intrigue_basedir}/config/sidekiq-task-autoscheduled.yml"
+sidekiq_enrichment_config_file = "#{intrigue_basedir}/config/sidekiq-task-enrichment.yml"
sidekiq_app_config_file = "#{intrigue_basedir}/config/sidekiq-app.yml"
control_script = "#{intrigue_basedir}/util/control.sh"
@@ -28,6 +29,7 @@ task :clean do
FileUtils.mv database_config_file, "#{database_config_file}.backup"
FileUtils.mv sidekiq_interactive_config_file, "#{sidekiq_interactive_config_file}.backup"
FileUtils.mv sidekiq_autoscheduled_config_file, "#{sidekiq_autoscheduled_config_file}.backup"
+ FileUtils.mv sidekiq_enrichment_config_file, "#{sidekiq_enrichment_config_file}.backup"
FileUtils.mv sidekiq_app_config_file, "#{sidekiq_app_config_file}.backup"
FileUtils.mv geolocation_database, "#{geolocation_database}.backup"
FileUtils.mv web_accounts_list, "#{web_accounts_list}.backup"
@@ -83,16 +85,22 @@ task :setup do
## Copy sidekiq task worker config into place
puts "[+] Setting up task worker config...."
- if File.exist? sidekiq_interactive_config_file && sidekiq_autoscheduled_config_file && sidekiq_app_config_file
+ if File.exist?(sidekiq_interactive_config_file &&
+ sidekiq_autoscheduled_config_file &&
+ sidekiq_enrichment_config_file &&
+ sidekiq_app_config_file)
puts "[ ] File already exists, skipping: #{sidekiq_interactive_config_file}"
puts "[ ] File already exists, skipping: #{sidekiq_autoscheduled_config_file}"
+ puts "[ ] File already exists, skipping: #{sidekiq_enrichment_config_file}"
puts "[ ] File already exists, skipping: #{sidekiq_app_config_file}"
else
puts "[+] Copying: #{sidekiq_interactive_config_file}.default"
puts "[+] Copying: #{sidekiq_autoscheduled_config_file}.default"
+ puts "[+] Copying: #{sidekiq_enrichment_config_file}.default"
puts "[+] Copying: #{sidekiq_app_config_file}.default"
FileUtils.cp "#{sidekiq_interactive_config_file}.default", sidekiq_interactive_config_file
FileUtils.cp "#{sidekiq_autoscheduled_config_file}.default", sidekiq_autoscheduled_config_file
+ FileUtils.cp "#{sidekiq_enrichment_config_file}.default", sidekiq_enrichment_config_file
FileUtils.cp "#{sidekiq_app_config_file}.default", sidekiq_app_config_file
end
diff --git a/app/helpers.rb b/app/helpers.rb
index 25975be56..e415caa6a 100644
--- a/app/helpers.rb
+++ b/app/helpers.rb
@@ -2,6 +2,18 @@ module Intrigue
module Task
module Helper
+ def entity_exists?(project, entity_type, entity_name)
+ puts "Checking for existence of an entity with type: #{entity_type} and name: #{entity_name} in project: #{project.name}"
+ Intrigue::Model::Entity.scope_by_project_and_type(project.name,entity_type).each do |e|
+ if e.unique_names.include? entity_name
+ puts "Found! #{entity_name}"
+ return e
+ end
+ end
+ puts "Not Found! #{entity_name}"
+ false
+ end
+
###
### Helper method for starting a task run
###
@@ -16,7 +28,7 @@ def start_task(queue, project, existing_scan_result, task_name, entity, depth, o
:options => options,
:handlers => [],
:base_entity => entity,
- :autoscheduled => (queue == "task_autoscheduled"),
+ :autoscheduled => (queue == "task_autoscheduled" || queue == "task_enrichment"),
:depth => depth
})
diff --git a/app/models/capabilities/export_graph.rb b/app/models/capabilities/export_graph.rb
index 4bd5234d0..8fc3befa5 100644
--- a/app/models/capabilities/export_graph.rb
+++ b/app/models/capabilities/export_graph.rb
@@ -7,26 +7,30 @@ def export_graph_json
# generate the nodes
nodes = []
edges = []
- edge_count = 1
+ edge_count = 0
self.task_results.each do |t|
+ next if t.base_entity.type_string == "Uri"
+
# add the base entity first (provided it hasn't been deleted)
x = { :id => t.base_entity.id, :label => "#{t.base_entity.name}", :type => t.base_entity.type_string}
- #x[:color] = "lightgrey" if t.base_entity.secondary
nodes << x unless t.base_entity.deleted?
# then for each of the entities, generate the node and edges. skip if deleted.
t.entities.each do |e|
- #next unless e.type_string == "WebServer"
+
+ next if e.type_string == "Uri"
+
x = { :id => e.id, :label => "#{e.name}", :type => e.type_string } #unless e.secondary
- #x[:color] = "lightgrey" if e.secondary
+ #x[:color] = "lightgrey" if e.type_string == "Uri"
+
nodes << x unless e.deleted?
unless t.base_entity.deleted? || e.deleted?
- edges << {"id" => edge_count, "source" => t.base_entity.id, "target" => e.id}
- edge_count += 1
+ edges << {"id" => edge_count += 1, "source" => t.base_entity.id, "target" => e.id}
end
+
end
end
diff --git a/app/models/entity.rb b/app/models/entity.rb
index 299bacd9e..e9ab6b712 100644
--- a/app/models/entity.rb
+++ b/app/models/entity.rb
@@ -1,35 +1,20 @@
module Intrigue
module Model
-=begin
- class AliasMapping < Sequel::Model
- plugin :validation_helpers
- #self.raise_on_save_failure = false
-
- many_to_one :source, :class => :'Intrigue::Model::Entity', :key => :source_id
- many_to_one :target, :class => :'Intrigue::Model::Entity', :key => :target_id
-
- def validate
- super
- validates_unique([:source_id, :target_id]) # only allow a single alias
- end
- end
-=end
class Entity < Sequel::Model
plugin :validation_helpers
plugin :single_table_inheritance, :type
- plugin :serialization, :json, :details
+ plugin :serialization, :json, :details #, :name
self.raise_on_save_failure = false
#set_allowed_columns :type, :name, :details, :project_id
many_to_many :task_results
many_to_one :project
- #many_to_many :aliases, :left_key=>:source_id,:right_key=>:target_id, :join_table=>:alias_mappings, :class=>self
def validate
super
- validates_unique([:name, :project_id])
+ validates_unique([:project_id, :name])
end
def self.scope_by_project(project_name)
@@ -42,6 +27,11 @@ def self.scope_by_project_and_type(project, type)
where(Sequel.&(:project_id => named_project_id, :type => type.to_s))
end
+ # easy way to refer to all names (overridden in some entities)
+ def unique_names
+ [name]
+ end
+
def deleted?
return true if deleted
false
@@ -58,9 +48,6 @@ def created_by?(task_name)
false
end
- def names
- end
-
def allowed_tasks
### XXX - this needs to be limited to tasks that accept this type
TaskFactory.allowed_tasks_for_entity_type(type_string)
@@ -100,6 +87,10 @@ def form
}
end
+ def get_detail(key)
+ details[key]
+ end
+
def self.descendants
x = ObjectSpace.each_object(Class).select{ |klass| klass < self }
end
@@ -114,10 +105,6 @@ def export_hash
:name => name,
:deleted => deleted,
:details => details,
- #:aliases => self.aliases.map{|x| {
- # "id" => x.id,
- # "type" => x.type,
- # "name" => x.name }},
:task_results => task_results.map{ |t| {:id => t.id, :name => t.name } }
}
end
@@ -126,30 +113,26 @@ def export_json
export_hash.to_json
end
- # export id, type, name, and details on a single line, removing spaces and commas
- def export_csv
- export_string = "#{id},#{type_string},#{name.gsub(/[\,,\s]/,"")},"
- details.each{|k,v| export_string << "#{k}=#{v};".gsub(/[\,,\s]/,"") }
- export_string
- end
-
- def export_tsv
- export_string = "#{id}\t#{type_string}\t#{name}\t"
- details.each{|k,v| export_string << "#{k}##{v};" }
- export_string
- end
-
private
def _escape_html(text)
Rack::Utils.escape_html(text)
text
end
- # have an easy way to sort and hash all the aliases (which should include
- # all names)
- def _unique_name
- string = aliases.split(",").sort_by{|x| x.downcase}.join(", ")
- Digest::SHA1.hexdigest string
+
+ ### VALIDATIONS!
+
+ # https://tools.ietf.org/html/rfc1123
+ def _v4_regex
+ /(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/
+ end
+
+ def _v6_regex
+ /^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$/
+ end
+
+ def _dns_regex
+ /^(\w|-|\.).*\.(\w|-|\.).*$/
end
end
diff --git a/app/models/task_result.rb b/app/models/task_result.rb
index 7e7010e7d..db0aaabaf 100644
--- a/app/models/task_result.rb
+++ b/app/models/task_result.rb
@@ -23,8 +23,6 @@ def validate
def start(queue)
if queue == "task_autoscheduled"
- autoscheduled = true
-
self.job_id = Sidekiq::Client.push({
"class" => Intrigue::TaskFactory.create_by_name(task_name).class.to_s,
"queue" => "task_autoscheduled",
@@ -32,15 +30,23 @@ def start(queue)
"args" => [id]
})
- save
+ elsif queue == "task_enrichment"
+ self.job_id = Sidekiq::Client.push({
+ "class" => Intrigue::TaskFactory.create_by_name(task_name).class.to_s,
+ "queue" => "task_enrichment",
+ "retry" => true,
+ "args" => [id]
+ })
else # start it in the task queues
task = Intrigue::TaskFactory.create_by_name(task_name)
self.job_id = task.class.perform_async id
- save
+
end
- job_id
+ save
+
+ self.job_id
end
def log
@@ -64,19 +70,6 @@ def task
Intrigue::TaskFactory.create_by_name(task_name)
end
- ### Export!
- def export_csv
- output_string = ""
- entities.each{ |x| output_string << x.export_csv << "\n" }
- output_string
- end
-
- def export_tsv
- export_string = ""
- entities.map{ |x| export_string << x.export_tsv + "\n" }
- export_string
- end
-
def export_hash
{
"id" => id,
diff --git a/app/routes/results.rb b/app/routes/results.rb
index 12b6957c2..dd01c1968 100644
--- a/app/routes/results.rb
+++ b/app/routes/results.rb
@@ -20,6 +20,7 @@ class IntrigueApp < Sinatra::Base
entity_id = @params["entity_id"]
depth = @params["depth"].to_i
current_project = Intrigue::Model::Project.first(:name => @project_name)
+ entity_name = "#{@params["attrib_name"]}"
# Construct the attributes hash from the parameters. Loop through each of the
# parameters looking for things that look like attributes, and add them to our
@@ -32,7 +33,7 @@ class IntrigueApp < Sinatra::Base
end
end
- # hack! remove the name, no longer needed in the details
+ # HACK! add the name to aliases, name detail no longer needed
entity_details.delete("name")
# Construct an entity from the data we have
@@ -43,17 +44,14 @@ class IntrigueApp < Sinatra::Base
return unless entity_type
# TODO - SECURITY - validate that it's a valid entity type before we eval
-
klass = eval("Intrigue::Entity::#{entity_type}")
- entity_name = "#{@params["attrib_name"]}"
- # TODO - we'll need to check all aliases of all entities within the project here
- entity = Intrigue::Model::Entity.scope_by_project_and_type(@project_name, klass.to_s).first(:name => entity_name)
+ entity = entity_exists?(current_project,entity_type,entity_name)
unless entity
entity = Intrigue::Model::Entity.create(
- { :type => klass,
- :name => entity_name,
+ { :name => entity_name,
+ :type => klass,
:details => entity_details,
:project => current_project
})
diff --git a/app/views/dossier.erb b/app/views/dossier.erb
index 074486576..98b4dec51 100644
--- a/app/views/dossier.erb
+++ b/app/views/dossier.erb
@@ -3,28 +3,28 @@
People:
Email:
Phone Numbers:
Software:
@@ -32,7 +32,7 @@
@@ -41,48 +41,48 @@
<% @uris.sort{|x,y| "#{x.details["stack"]}" <=> "#{y.details["stack"]}"}.each do |e|
next unless e.details["stack"]
next unless e.details["stack"].count > 0 %>
- <%= h "#{e.details["stack"]}: ... #{e.name}" %>
+ <%= h "#{e.details["stack"]}: ... #{e.unique_names}" %>
<% end %>
SSL Certificates:
Uris:
Network Services:
Hosts:
Networks:
Other:
diff --git a/app/views/results/detail.erb b/app/views/results/detail.erb
index 706a43671..4fd1c1459 100644
--- a/app/views/results/detail.erb
+++ b/app/views/results/detail.erb
@@ -44,9 +44,7 @@ getEntities(<%=@result.id%>);
Complete: <%= @result.complete %>
Export:
Entities:
diff --git a/config/sidekiq-task-enrichment.yml.default b/config/sidekiq-task-enrichment.yml.default
new file mode 100644
index 000000000..84e62b5e9
--- /dev/null
+++ b/config/sidekiq-task-enrichment.yml.default
@@ -0,0 +1,6 @@
+:verbose: false
+#:logfile: ./log/task-enrichment.log # this is commented bc we supply it on the cli
+#:pidfile: ./tmp/pids/task-enrichment.pid # this is commented bc we supply it on the cli
+:concurrency: 10
+:queues:
+ - task_enrichment
diff --git a/db/006_drop_aliases.rb b/db/006_drop_alias_mappings.rb
similarity index 74%
rename from db/006_drop_aliases.rb
rename to db/006_drop_alias_mappings.rb
index 6fd706417..fa3738598 100644
--- a/db/006_drop_aliases.rb
+++ b/db/006_drop_alias_mappings.rb
@@ -1,11 +1,12 @@
Sequel.migration do
change do
- #alter_table :entities do
- # String :names, :text => true
- #end
drop_table :alias_mappings
+ #alter_table :entities do
+ # String :aliases, :text => true
+ #end
+
end
end
diff --git a/lib/entities/as_number.rb b/lib/entities/as_number.rb
index 4a364c53e..778c15d0c 100644
--- a/lib/entities/as_number.rb
+++ b/lib/entities/as_number.rb
@@ -9,8 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/
+ def validate_entity
+ name =~ /^\w.*$/
end
end
diff --git a/lib/entities/credential.rb b/lib/entities/credential.rb
index c5711f4b7..c01ea9b77 100644
--- a/lib/entities/credential.rb
+++ b/lib/entities/credential.rb
@@ -9,11 +9,11 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*/ &&
- @details["username"].to_s =~ /^.*$/ &&
- @details["password"].to_s =~ /^.*$/ &&
- @details["uri"].to_s =~ /^.*$/
+ def validate_entity
+ name =~ /^\w.*/ &&
+ details["username"].to_s =~ /^\w.*$/ &&
+ details["password"].to_s =~ /^\w.*$/ &&
+ details["uri"].to_s =~ /^\w.*$/
end
end
diff --git a/lib/entities/dns_server.rb b/lib/entities/dns_server.rb
index 14c07be17..53c32d0e5 100644
--- a/lib/entities/dns_server.rb
+++ b/lib/entities/dns_server.rb
@@ -9,8 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^[a-zA-Z0-9\.].*/
+ def validate_entity
+ (name =~ _v4_regex || name =~ _v6_regex || name == _dns_regex) && details["port"].to_s =~ /^\d{1,5}$/
end
end
diff --git a/lib/entities/email_address.rb b/lib/entities/email_address.rb
index d01bfbcc1..d2ad605a2 100644
--- a/lib/entities/email_address.rb
+++ b/lib/entities/email_address.rb
@@ -10,8 +10,8 @@ def self.metadata
end
- def validate_content
- @name =~ /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,8}/
+ def validate_entity
+ name =~ /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,8}/
end
end
diff --git a/lib/entities/file.rb b/lib/entities/file.rb
index 009a53193..ef594ff5c 100644
--- a/lib/entities/file.rb
+++ b/lib/entities/file.rb
@@ -9,8 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/
+ def validate_entity
+ name =~ /^\w.*$/
end
end
diff --git a/lib/entities/finger_server.rb b/lib/entities/finger_server.rb
index 3b2c2de69..57eba4ec7 100644
--- a/lib/entities/finger_server.rb
+++ b/lib/entities/finger_server.rb
@@ -9,9 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^[a-zA-Z0-9\.\:\/\ ].*/ &&
- @details["port"].to_s =~ /^\d{1,5}$/
+ def validate_entity
+ (name =~ _v4_regex || name =~ _v6_regex || name == _dns_regex) && details["port"].to_s =~ /^\d{1,5}$/
end
end
diff --git a/lib/entities/ftp_server.rb b/lib/entities/ftp_server.rb
index 8eba56b7e..87cae49d7 100644
--- a/lib/entities/ftp_server.rb
+++ b/lib/entities/ftp_server.rb
@@ -9,9 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^[a-zA-Z0-9\.\:\/\ ].*/ &&
- @details["port"].to_s =~ /^\d{1,5}$/
+ def validate_entity
+ (name =~ _v4_regex || name =~ _v6_regex || name == _dns_regex) && details["port"].to_s =~ /^\d{1,5}$/
end
end
diff --git a/lib/entities/github_repository.rb b/lib/entities/github_repository.rb
index 87202ff31..dca8cc8a4 100644
--- a/lib/entities/github_repository.rb
+++ b/lib/entities/github_repository.rb
@@ -9,9 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*/ &&
- @uri =~ /^.*/
+ def validate_entity
+ name =~ /^\w.*/ && details["uri"] =~ /^\w.*/
end
end
diff --git a/lib/entities/github_user.rb b/lib/entities/github_user.rb
index 89b71de86..4661177d7 100644
--- a/lib/entities/github_user.rb
+++ b/lib/entities/github_user.rb
@@ -9,9 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*/ &&
- @uri =~ /^.*/
+ def validate_entity
+ name =~ /^\w.*/ && details["uri"] =~ /^.*/
end
end
diff --git a/lib/entities/host.rb b/lib/entities/host.rb
index 9d99bb59c..abf322c73 100644
--- a/lib/entities/host.rb
+++ b/lib/entities/host.rb
@@ -9,17 +9,19 @@ def self.metadata
}
end
- def validate_content
-
- v4_regex = /(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/
- v6_regex = /^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$/
+ def validate_entity
+ return (name =~ _v4_regex || name =~ _v6_regex || name =~ _dns_regex)
+ end
- # https://tools.ietf.org/html/rfc1123
- dns_regex = /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/
+ def unique_names
+ x = [name]
+ x.concat(details["dns_names"]) if details["dns_names"]
+ x.concat(details["ip_addresses"]) if details["ip_addresses"]
- return (@name =~ v4_regex || @name =~ v6_regex || @name == dns_regex)
+ x.sort.uniq
end
+
end
end
end
diff --git a/lib/entities/http_header.rb b/lib/entities/http_header.rb
index ed4fafea9..615dd5559 100644
--- a/lib/entities/http_header.rb
+++ b/lib/entities/http_header.rb
@@ -9,9 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/ &&
- @details["content"] =~ /^.*$/
+ def validate_entity
+ name =~ /^\w.*$/ && details["content"] =~ /^.*$/
end
end
diff --git a/lib/entities/info.rb b/lib/entities/info.rb
index 95d8804c9..3e7768fe4 100644
--- a/lib/entities/info.rb
+++ b/lib/entities/info.rb
@@ -9,8 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/
+ def validate_entity
+ name =~ /^\w.*$/
end
end
diff --git a/lib/entities/mongo_service.rb b/lib/entities/mongo_service.rb
index db56e1e4b..5239b03fb 100644
--- a/lib/entities/mongo_service.rb
+++ b/lib/entities/mongo_service.rb
@@ -10,17 +10,15 @@ def self.metadata
end
- def validate_content
- @details["ip_address"].to_s =~ /^.*$/ &&
- @details["port"].to_s =~ /^\d{1,5}$/ &&
- @details["proto"].to_s =~ /^(tcp|udp)$/
+ def validate_entity
+ (name =~ _v4_regex || name =~ _v6_regex || name == _dns_regex) && details["port"].to_s =~ /^\d{1,5}$/
end
def form
output = super
- output << "
"
- output << "
"
- output << "
"
+ output << "
"
+ output << "
"
+ output << "
"
output
end
diff --git a/lib/entities/net_block.rb b/lib/entities/net_block.rb
index c9ca84b73..539437b80 100644
--- a/lib/entities/net_block.rb
+++ b/lib/entities/net_block.rb
@@ -10,13 +10,13 @@ def self.metadata
end
- def validate_content
+ def validate_entity
# required:
- @name =~ /^.*$/
+ name =~ /^\w.*$/
# suggested:
- # @details["organization_reference"]
+ # details["organization_reference"]
end
diff --git a/lib/entities/network_service.rb b/lib/entities/network_service.rb
index 367daf124..83865bd4f 100644
--- a/lib/entities/network_service.rb
+++ b/lib/entities/network_service.rb
@@ -10,10 +10,10 @@ def self.metadata
end
- def validate_content
- @details["ip_address"].to_s =~ /^.*$/ &&
- @details["port"].to_s =~ /^\d{1,5}$/ &&
- @details["proto"].to_s =~ /^(tcp|udp)$/
+ def validate_entity
+ (details["ip_address"].to_s =~ _v4_regex || details["ip_address"].to_s =~ _v6_regex) &&
+ details["port"].to_s =~ /^\d{1,5}$/ &&
+ details["proto"].to_s =~ /^(tcp|udp)$/
end
def form
diff --git a/lib/entities/organization.rb b/lib/entities/organization.rb
index 3f3074d14..8acdfd20e 100644
--- a/lib/entities/organization.rb
+++ b/lib/entities/organization.rb
@@ -10,8 +10,8 @@ def self.metadata
end
- def validate_content
- @name =~ /^.*$/
+ def validate_entity
+ name =~ /^\w.*$/
end
end
diff --git a/lib/entities/person.rb b/lib/entities/person.rb
index dd9e6f99c..edf0e7a79 100644
--- a/lib/entities/person.rb
+++ b/lib/entities/person.rb
@@ -10,8 +10,8 @@ def self.metadata
end
- def validate_content
- @name =~ /^.*$/
+ def validate_entity
+ name =~ /^\w.*$/
end
end
diff --git a/lib/entities/phone_number.rb b/lib/entities/phone_number.rb
index c8da5105e..2ebc072ca 100644
--- a/lib/entities/phone_number.rb
+++ b/lib/entities/phone_number.rb
@@ -9,8 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/
+ def validate_entity
+ name =~ /^\w.*$/
end
end
diff --git a/lib/entities/physical_location.rb b/lib/entities/physical_location.rb
index 6fa7aca94..12b03e0f0 100644
--- a/lib/entities/physical_location.rb
+++ b/lib/entities/physical_location.rb
@@ -9,10 +9,10 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/ #&&
- #@details["latitude"] =~ /^([-+]?\d{1,2}[.]\d+)$/ &&
- #@details["longitude"] =~ /^([-+]?\d{1,3}[.]\d+)$/
+ def validate_entity
+ name =~ /^\w.*$/ #&&
+ #details["latitude"] =~ /^([-+]?\d{1,2}[.]\d+)$/ &&
+ #details["longitude"] =~ /^([-+]?\d{1,3}[.]\d+)$/
end
end
diff --git a/lib/entities/screenshot.rb b/lib/entities/screenshot.rb
index 5ed418bc5..bafe7d980 100644
--- a/lib/entities/screenshot.rb
+++ b/lib/entities/screenshot.rb
@@ -9,9 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/ # XXX - too loose
- #@details[:file] =~ /^.*$/ # XXX - too loose
+ def validate_entity
+ name =~ /^\w.*$/ # TODO - tighten this up
end
end
diff --git a/lib/entities/software_package.rb b/lib/entities/software_package.rb
index 35850f58a..81278c4af 100644
--- a/lib/entities/software_package.rb
+++ b/lib/entities/software_package.rb
@@ -9,8 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/
+ def validate_entity
+ name =~ /^\w.*$/
end
end
diff --git a/lib/entities/ssh_server.rb b/lib/entities/ssh_server.rb
index a284a9f49..ff3e3c908 100644
--- a/lib/entities/ssh_server.rb
+++ b/lib/entities/ssh_server.rb
@@ -9,9 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^[a-zA-Z0-9\.\:\/\ ].*/ &&
- @details["port"].to_s =~ /^\d{1,5}$/
+ def validate_entity
+ (name =~ _v4_regex || name =~ _v6_regex || name == _dns_regex) && details["port"].to_s =~ /^\d{1,5}$/
end
end
diff --git a/lib/entities/ssl_certificate.rb b/lib/entities/ssl_certificate.rb
index 326c9481f..a48b21e1b 100644
--- a/lib/entities/ssl_certificate.rb
+++ b/lib/entities/ssl_certificate.rb
@@ -9,8 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/
+ def validate_entity
+ name =~ /^.*$/
end
end
diff --git a/lib/entities/string.rb b/lib/entities/string.rb
index f8958e597..307316b58 100644
--- a/lib/entities/string.rb
+++ b/lib/entities/string.rb
@@ -9,8 +9,8 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/
+ def validate_entity
+ name =~ /^\w.*$/
end
end
diff --git a/lib/entities/uri.rb b/lib/entities/uri.rb
index fc484d8f0..bbad92c29 100644
--- a/lib/entities/uri.rb
+++ b/lib/entities/uri.rb
@@ -5,14 +5,12 @@ class Uri < Intrigue::Model::Entity
def self.metadata
{
:name => "Uri",
- :description => "TODO"
+ :description => "A Uniform Resource Identifier (URI) is a string of characters used to identify a resource."
}
end
- def validate_content
- @name =~ /^.*$/ #&&
- #@details["web_application"] &&
- #@details["web_server"]
+ def validate_entity
+ name =~ /^\w.*$/
end
end
diff --git a/lib/entities/web_account.rb b/lib/entities/web_account.rb
index 69bb2f4bd..988613deb 100644
--- a/lib/entities/web_account.rb
+++ b/lib/entities/web_account.rb
@@ -9,10 +9,10 @@ def self.metadata
}
end
- def validate_content
- @name =~ /^.*$/ &&
- @details["domain"] =~ /^.*$/ &&
- @details["uri"] =~ /^http.*$/
+ def validate_entity
+ name =~ /^\w.*$/ &&
+ details["domain"] =~ /^.*$/ &&
+ details["uri"] =~ /^http.*$/
end
end
diff --git a/lib/entity_manager.rb b/lib/entity_manager.rb
index 2b59fa7c9..6d3b64383 100644
--- a/lib/entity_manager.rb
+++ b/lib/entity_manager.rb
@@ -6,15 +6,7 @@ class EntityManager
# NOTE: We don't auto-register entities like the other factories (handled by
# single table inheritance)
- # NOTE: The user's desired depth of recursion is stored on the task_result.
-
- def self.entity_exists?(type,name)
- return true if Intrigue::Model::Entity.first(:name=>name,:type=>type)
- false
- end
-
def self.resolve_type(type_string)
-
# TODO - SECURITY - don't eval unless it's one of our valid entity types
x = eval("Intrigue::Entity::#{type_string}")
false unless x.kind_of? Intrigue::Model::Entity
@@ -22,28 +14,26 @@ def self.resolve_type(type_string)
end
# This method creates a new entity, and kicks off a strategy
- def self.create_or_merge_entity_recursive(task_result,type_string,name,details)
+ def self.create_or_merge_entity(task_result,type_string,name,details)
project = task_result.project # convenience
+ downcased_name = name.downcase
# Clean up in case there are encoding issues
- name = _encode_string(name)
- details = _encode_hash(details.merge(:aliases => ["#{name}"]))
+ #name = _encode_string(name)
+ #details = _encode_hash(details.merge(:aliases => "#{name}"]))
type = resolve_type(type_string)
# Merge the details if it already exists
- entity = nil
- entity = Intrigue::Model::Entity.scope_by_project_and_type(project.name,type).first(:name => name)
-
- if entity.kind_of? Intrigue::Model::Entity
- # We're going to have to look for each of the aliases as well.
- # deep_merge_aliases(?)
- entity.details = details.merge(entity.details)
+ entity = entity_exists?(project,type,downcased_name)
+ if entity
+ # TODO - DEEP MERGE
+ entity.details = details.deep_merge(entity.details)
entity.save
else
# Create a new entity, validating the attributes
entity = Intrigue::Model::Entity.create({
- :name => name,
+ :name => downcased_name,
:project => project,
:type => type,
:details => details
@@ -51,10 +41,16 @@ def self.create_or_merge_entity_recursive(task_result,type_string,name,details)
end
unless entity
- puts "ERROR! Unable to create entity: #{type}##{name}"
+ puts "ERROR! Unable to create or find entity: #{type}##{downcased_name}"
return nil
end
+ unless Intrigue::Model::Entity.find(:id => entity.id).validate_entity
+ puts "ERROR! validation of entity failed: #{entity}"
+ return nil
+ end
+
+
# Add to our result set for this task
task_result.add_entity entity
task_result.save
@@ -69,16 +65,18 @@ def self.create_or_merge_entity_recursive(task_result,type_string,name,details)
# START PROCESSING OF ENRICHMENT (to depth of 1)
if task_result.depth > 0
- unless prohibited_entity? entity
+ #unless prohibited_entity? entity
if entity.type_string == "Host"
- start_task("task_autoscheduled", project, task_result.scan_result, "get_all_host_names", entity, 1, [],[])
+ start_task("task_enrichment", project, task_result.scan_result, "enrich_host", entity, 1, [],[])
elsif entity.type_string == "Uri"
start_task("task_autoscheduled", project, task_result.scan_result, "check_api_endpoint", entity, 1, [],[])
start_task("task_autoscheduled", project, task_result.scan_result, "web_stack_fingerprint", entity, 1, [],[])
end
- end
+ #end
end# END PROCESSING OF ENRICHMENT
+ #sleep 3 # give time for the enrichment to complete in case we don't have a backlog
+
# START PROCESSING OF RECURSION BY STRATEGY TYPE
scan_result = task_result.scan_result
if scan_result && task_result.depth > 0 # if this is a scan and we're within depth
diff --git a/lib/strategies/discovery.rb b/lib/strategies/discovery.rb
index 1edabe7ef..95186f123 100644
--- a/lib/strategies/discovery.rb
+++ b/lib/strategies/discovery.rb
@@ -6,10 +6,23 @@ def self.recurse(entity, task_result)
if entity.type_string == "Host"
+ # Wait until enrichment has happened before going further
+ entity_id = entity.id
+ countdown = 10 # HACK
+ while !entity.get_detail("enriched") && countdown > 0 do
+ puts "waiting for #{entity} enrichment: #{entity.get_detail("enriched")} (#{countdown})"
+ #puts "#{entity.id}"
+ #puts "#{entity.details.inspect}"
+ sleep 3
+ countdown -= 1
+ entity = Intrigue::Model::Entity.find(:id => entity_id)
+ end
+
start_recursive_task(task_result,"nmap_scan",entity)
### DNS Subdomain Bruteforce
# Do a big bruteforce if the size is small enough
+ # Do a big bruteforce if the size is small enough
if (entity.name.split(".").length < 3)
start_recursive_task(task_result,"dns_brute_sub",entity,[
{"name" => "use_file", "value" => true }])
@@ -29,22 +42,22 @@ def self.recurse(entity, task_result)
elsif entity.type_string == "NetBlock"
# Make sure it's small enough not to be disruptive, and if it is, scan it
- cidr = entity.name.split("/").last.to_i
- if cidr >= 16
+ #cidr = entity.name.split("/").last.to_i
+ #if cidr >= 16
start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 21}])
start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 80}])
start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 443}])
start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 8080}])
start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 8081}])
start_recursive_task(task_result,"masscan_scan",entity, [{"port" => 8443}])
- end
+ #end
elsif entity.type_string == "Uri"
## Grab the Web Server
## Grab the SSL Certificate
- start_recursive_task(task_result,"uri_gather_ssl_certificate",entity) if entity.name =~ /^https/
+ start_recursive_task(task_result,"uri_gather_ssl_certificate",entity) #if entity.name =~ /^https/
## Spider, looking for metadata
start_recursive_task(task_result,"uri_spider",entity,[
diff --git a/lib/tasks/enrich/get_all_host_names.rb b/lib/tasks/enrich/enrich_host.rb
similarity index 67%
rename from lib/tasks/enrich/get_all_host_names.rb
rename to lib/tasks/enrich/enrich_host.rb
index 0a410d779..769fe3535 100644
--- a/lib/tasks/enrich/get_all_host_names.rb
+++ b/lib/tasks/enrich/enrich_host.rb
@@ -1,12 +1,12 @@
require 'dnsruby'
module Intrigue
-class GetAllHostNames < BaseTask
+class EnrichHost < BaseTask
def self.metadata
{
- :name => "get_all_host_names",
- :pretty_name => "Get All Host Names",
+ :name => "enrich_host",
+ :pretty_name => "Enrich Host",
:authors => ["jcran"],
:description => "Look up all names of a given entity.",
:references => [],
@@ -27,6 +27,14 @@ def run
opt_resolver = _get_option "resolver"
lookup_name = _get_entity_name
+ ip_addresses = []
+ dns_names = []
+ if lookup_name.is_ip_address?
+ ip_addresses << lookup_name
+ else
+ dns_names << lookup_name
+ end
+
begin
resolver = Dnsruby::Resolver.new(
:recurse => "true",
@@ -41,25 +49,34 @@ def run
_log_error "Nothing?" if result.answer.empty?
# For each of the found addresses
- names = []
result.answer.map do |resource|
next if resource.type == Dnsruby::Types::RRSIG #TODO parsing this out is a pain, not sure if it's valuable
_log "Adding name from: #{resource}"
- names << resource.address.to_s if resource.respond_to? :address
- names << resource.name.to_s
+ ip_addresses << resource.address.to_s if resource.respond_to? :address
+ dns_names << resource.name.to_s.downcase
end #end result.answer
- @entity.update(:details => @entity.details.merge("aliases" => names.sort.uniq))
- @entity.save
-
+ rescue Dnsruby::ServFail => e
+ _log_error "Unable to resolve: #{@entity}, error: #{e}"
rescue Dnsruby::NXDomain => e
- _log_error "Unable to resolve: #{e}"
+ _log_error "Unable to resolve: #{@entity}, error: #{e}"
rescue Dnsruby::ResolvTimeout => e
_log_error "Unable to resolve, timed out: #{e}"
rescue Errno::ENETUNREACH => e
_log_error "Hit exception: #{e}. Are you sure you're connected?"
+
#rescue Exception => e
# _log_error "Hit exception: #{e}"
+ ensure
+
+ temp_details = @entity.details
+ temp_details["ip_addresses"] = ip_addresses.sort.uniq
+ temp_details["dns_names"] = dns_names.sort.uniq
+ temp_details["enriched"] = true
+
+ @entity.update(:details => temp_details)
+ @entity.save
+
end
_log "Ran enrichment task!"
diff --git a/lib/tasks/enrich/web_stack_fingerprint.rb b/lib/tasks/enrich/web_stack_fingerprint.rb
index 2d221bb99..c52b09226 100644
--- a/lib/tasks/enrich/web_stack_fingerprint.rb
+++ b/lib/tasks/enrich/web_stack_fingerprint.rb
@@ -214,6 +214,7 @@ def _check_cookies(response)
temp << "ASP.NET" if header =~ /^.*ASPSESSIONID.*$/
temp << "ASP.NET" if header =~ /^.*ASP.NET_SessionId.*$/
temp << "BEA WebLogic" if header =~ /^.*WebLogicSession*$/
+ temp << "BigIP" if header =~ /^.*BIGipServer*$/
temp << "Coldfusion" if header =~ /^.*CFID.*$/
temp << "Coldfusion" if header =~ /^.*CFTOKEN.*$/
temp << "Coldfusion" if header =~ /^.*CFGLOBALS.*$/
diff --git a/lib/tasks/helpers/generic.rb b/lib/tasks/helpers/generic.rb
index d1e72fdb6..2d5c97dc0 100644
--- a/lib/tasks/helpers/generic.rb
+++ b/lib/tasks/helpers/generic.rb
@@ -4,15 +4,16 @@ module Generic
private
+ ###
+ ### Helper method to reach out to the entity manager
+ ###
def _create_entity(type, hash)
- # NOTE: this is a hack - the _create_entity call should be updated in each task
- # to create the entity in a form that's ready for create_or_merge_entity_recursive
- name = hash["name"] # Pull out the name from the hash
- hash.delete("name") # No need for a name in the hash now, remove it
+ # No need for a name in the hash now, remove it & pull out the name from the hash
+ name = hash.delete("name")
# Create or merge the entity
- entity = EntityManager.create_or_merge_entity_recursive(@task_result, type, name, hash)
+ EntityManager.create_or_merge_entity(@task_result, type, name, hash)
end
###
diff --git a/lib/tasks/helpers/web.rb b/lib/tasks/helpers/web.rb
index bdfc36239..6ff9eb0da 100644
--- a/lib/tasks/helpers/web.rb
+++ b/lib/tasks/helpers/web.rb
@@ -178,6 +178,8 @@ def http_get(uri_string, headers={}, limit = 10, open_timeout=15, read_timeout=1
@task_result.logger.log_error "Timeout : #{e}" if @task_result
rescue Net::ReadTimeout => e
@task_result.logger.log_error "Timeout : #{e}" if @task_result
+ rescue Errno::ETIMEDOUT => e
+ @task_result.logger.log_error "Timeout : #{e}" if @task_result
rescue Errno::ENETUNREACH => e
@task_result.logger.log_error "Unable to connect: #{e}" if @task_result
rescue URI::InvalidURIError => e
diff --git a/lib/tasks/nmap_scan.rb b/lib/tasks/nmap_scan.rb
index 025495a4a..2b4717403 100644
--- a/lib/tasks/nmap_scan.rb
+++ b/lib/tasks/nmap_scan.rb
@@ -33,119 +33,126 @@ def run
#ports = _get_option "ports"
# Get range, or host
- to_scan = _get_entity_name
-
- # Create a tempfile to store results
- temp_file = "#{Dir::tmpdir}/nmap_scan_#{rand(100000000)}.xml"
-
- # Check for IPv6
- nmap_options = ""
- nmap_options << "-6 " if to_scan =~ /:/
-
- # shell out to nmap and run the scan
- _log "Scanning #{to_scan} and storing in #{temp_file}"
- _log "NMap options: #{nmap_options}"
- nmap_string = "nmap #{to_scan} #{nmap_options} -P0 --top-ports 100 --min-parallelism 10 -oX #{temp_file}"
- _log "Running... #{nmap_string}"
- _unsafe_system(nmap_string)
-
- # Gather the XML and parse
- #_log "Raw Result:\n #{File.open(temp_file).read}"
- _log "Parsing #{temp_file}"
-
- parser = Nmap::XML.new(temp_file)
-
- # Create entities for each discovered service
- parser.each_host do |host|
- _log "Handling nmap data for #{host.ip}"
+ if @entity.type_string == "NetBlock"
+ ip_addresses = [_get_entity_name]
+ else
+ ip_addresses = @entity.details["ip_addresses"]
+ return unless ip_addresses
+ end
- # Handle the case of a netblock or domain - where we will need to create host entity(s)
- if @entity.type_string == "NetBlock" or @entity.type_string == "Host"
- # Only create if we've got ports to report.
- _create_entity("Host", { "name" => host.ip } ) if host.ports.count > 0
+ ip_addresses.each do |to_scan|
+ # Create a tempfile to store results
+ temp_file = "#{Dir::tmpdir}/nmap_scan_#{rand(100000000)}.xml"
+
+ # Check for IPv6
+ nmap_options = ""
+ nmap_options << "-6 " if to_scan =~ /:/
+
+ # shell out to nmap and run the scan
+ _log "Scanning #{to_scan} and storing in #{temp_file}"
+ _log "NMap options: #{nmap_options}"
+ nmap_string = "nmap #{to_scan} #{nmap_options} -P0 --top-ports 100 --min-parallelism 10 -oX #{temp_file}"
+ _log "Running... #{nmap_string}"
+ _unsafe_system(nmap_string)
+
+ # Gather the XML and parse
+ #_log "Raw Result:\n #{File.open(temp_file).read}"
+ _log "Parsing #{temp_file}"
+
+ parser = Nmap::XML.new(temp_file)
+
+ # Create entities for each discovered service
+ parser.each_host do |host|
+ _log "Handling nmap data for #{host.ip}"
+
+ # Handle the case of a netblock or domain - where we will need to create host entity(s)
+ if @entity.type_string == "NetBlock" #or @entity.type_string == "Host"
+ # Only create if we've got ports to report.
+ _create_entity("Host", { "name" => host.ip } ) if host.ports.count > 0
+ end
+
+ host.each_port do |port|
+ if port.state == :open
+
+ # Handle WebApps first
+ if port.protocol == :tcp &&
+ [80,443,8080,8081,8443].include?(port.number)
+
+ # determine if this is an SSL application
+ ssl = true if [443,8443].include?(port.number)
+ protocol = ssl ? "https://" : "http://" # construct uri
+
+ # Create URI
+ uri = "#{protocol}#{host.ip}:#{port.number}"
+ _create_entity("Uri", "name" => uri, "uri" => uri ) # create an entity
+
+ # and create the entities if we have dns resolution
+ @entity.details["dns_names"].each do |hostname|
+ uri = "#{protocol}#{hostname}:#{port.number}"
+ _create_entity("Uri", "name" => uri, "uri" => uri )
+ end
+
+ # then FtpServer
+ elsif [21].include?(port.number)
+ uri = "ftp://#{host.ip}:#{port.number}"
+ _create_entity("FtpServer", {
+ "name" => uri,
+ "ip_address" => "#{host.ip}",
+ "port" => port.number,
+ "proto" => port.protocol,
+ "uri" => uri })
+
+ # Then SshServer
+ elsif [22].include?(port.number)
+ uri = "ssh://#{host.ip}:#{port.number}"
+ _create_entity("SshServer", {
+ "name" => uri,
+ "ip_address" => "#{host.ip}",
+ "port" => port.number,
+ "proto" => port.protocol,
+ "uri" => uri })
+
+ # then DnsServer
+ elsif [53].include?(port.number)
+ uri = "#{host.ip}:#{port.number}"
+ _create_entity("DnsServer", {
+ "name" => uri,
+ "ip_address" => "#{host.ip}",
+ "port" => port.number,
+ "proto" => port.protocol,
+ "uri" => uri })
+
+ # then FingerServer
+ elsif [79].include?(port.number)
+ uri = "finger://#{host.ip}:#{port.number}"
+ _create_entity("FingerServer", {
+ "name" => uri,
+ "ip_address" => "#{host.ip}",
+ "port" => port.number,
+ "proto" => port.protocol,
+ "uri" => uri })
+
+ # Otherwise default to an unknown network service
+ else
+
+ _create_entity("NetworkService", {
+ "name" => "#{host.ip}:#{port.number}/#{port.protocol}",
+ "ip_address" => "#{host.ip}",
+ "port" => port.number,
+ "proto" => port.protocol,
+ "fingerprint" => "#{port.service}"})
+
+ end # end if
+ end # end if port.state == :open
+ end # end host.each_port
+ end # end parser
+
+ # Clean up!
+ begin
+ File.delete(temp_file)
+ rescue Errno::EPERM
+ _log_error "Unable to delete file"
end
-
- host.each_port do |port|
- if port.state == :open
-
- # Handle WebApps first
- if port.protocol == :tcp &&
- [80,443,8080,8081,8443].include?(port.number)
-
- # determine if this is an SSL application
- ssl = true if [443,8443].include?(port.number)
- protocol = ssl ? "https://" : "http://" # construct uri
-
- # Create URI
- uri = "#{protocol}#{host.ip}:#{port.number}"
- _create_entity("Uri", "name" => uri, "uri" => uri ) # create an entity
-
- # and create the entities if we have dns resolution
- host.hostnames.each do |hostname|
- uri = "#{protocol}#{hostname}:#{port.number}"
- _create_entity("Uri", "name" => uri, "uri" => uri )
- end
-
- # then FtpServer
- elsif [21].include?(port.number)
- uri = "ftp://#{host.ip}:#{port.number}"
- _create_entity("FtpServer", {
- "name" => uri,
- "ip_address" => "#{host.ip}",
- "port" => port.number,
- "proto" => port.protocol,
- "uri" => uri })
-
- # Then SshServer
- elsif [22].include?(port.number)
- uri = "ssh://#{host.ip}:#{port.number}"
- _create_entity("SshServer", {
- "name" => uri,
- "ip_address" => "#{host.ip}",
- "port" => port.number,
- "proto" => port.protocol,
- "uri" => uri })
-
- # then DnsServer
- elsif [53].include?(port.number)
- uri = "#{host.ip}:#{port.number}"
- _create_entity("DnsServer", {
- "name" => uri,
- "ip_address" => "#{host.ip}",
- "port" => port.number,
- "proto" => port.protocol,
- "uri" => uri })
-
- # then FingerServer
- elsif [79].include?(port.number)
- uri = "finger://#{host.ip}:#{port.number}"
- _create_entity("FingerServer", {
- "name" => uri,
- "ip_address" => "#{host.ip}",
- "port" => port.number,
- "proto" => port.protocol,
- "uri" => uri })
-
- # Otherwise default to an unknown network service
- else
-
- _create_entity("NetworkService", {
- "name" => "#{host.ip}:#{port.number}/#{port.protocol}",
- "ip_address" => "#{host.ip}",
- "port" => port.number,
- "proto" => port.protocol,
- "fingerprint" => "#{port.service}"})
-
- end # end if
- end # end if port.state == :open
- end # end host.each_port
- end # end parser
-
- # Clean up!
- begin
- File.delete(temp_file)
- rescue Errno::EPERM
- _log_error "Unable to delete file"
end
end
diff --git a/lib/tasks/uri_gather_ssl_certificate.rb b/lib/tasks/uri_gather_ssl_certificate.rb
index c0d506388..1bc898bdb 100644
--- a/lib/tasks/uri_gather_ssl_certificate.rb
+++ b/lib/tasks/uri_gather_ssl_certificate.rb
@@ -63,8 +63,10 @@ def run
x.gsub(/DNS:/,"").strip
end
- alt_names.each do |alt_name|
+ _log "Got alt_names: #{alt_names.inspect}"
+ # Iterate through, looking for trouble
+ alt_names.each do |alt_name|
if alt_name =~ /cloudflare.com$/ && opt_skip_cloudflare
_log "This is a cloudflare certificate, skipping further entity creation"
return
@@ -94,6 +96,10 @@ def run
_log "This is a wpengine certificate, skipping further entity creation"
return
end
+ end
+
+ #assuming we made it this far, let's proceed
+ alt_names.each do |alt_name|
# Remove any leading wildcards so we get a sensible domain name
if alt_name[0..1] == "*."
diff --git a/util/control.sh.default b/util/control.sh.default
index 1f327a8ea..d40cca2d9 100755
--- a/util/control.sh.default
+++ b/util/control.sh.default
@@ -22,6 +22,8 @@ function start_server {
bundle exec sidekiq -C $IDIR/config/sidekiq-task-interactive.yml -r $IDIR/core.rb -d -L $IDIR/log/task-interactive.log
echo "Starting autoscheduled task processing..."
bundle exec sidekiq -C $IDIR/config/sidekiq-task-autoscheduled.yml -r $IDIR/core.rb -d -L $IDIR/log/task-autoscheduled.log -P $IDIR/tmp/pids/sidekiq-task-autoscheduled.pid
+ echo "Starting enrichment task processing..."
+ bundle exec sidekiq -C $IDIR/config/sidekiq-task-enrichment.yml -r $IDIR/core.rb -d -L $IDIR/log/task-enrichment.log -P $IDIR/tmp/pids/sidekiq-task-enrichment.pid
echo "Starting background worker processing..."
bundle exec sidekiq -C $IDIR/config/sidekiq-app.yml -r $IDIR/core.rb -d -L $IDIR/log/app.log
echo "Starting puma..."