rework how enrichment works, and how entities can be more structured

.gitignore

@@ -21,3 +21,4 @@ lib/tasks_disabled
 lib/tasks/survey
 db/intrigue.db
 util/control.sh
+config/sidekiq-task-enrichment.yml

Procfile

@@ -1,4 +1,5 @@
-task-worker: bundle exec sidekiq -C config/sidekiq-task-interactive.yml -r ./core.rb
-as-task-worker: bundle exec sidekiq -C config/sidekiq-task-autoscheduled.yml -r ./core.rb
-app-worker: bundle exec sidekiq -C config/sidekiq-app.yml -r ./core.rb
-web: bundle exec  puma -C ./config/puma.rb
+interactive: bundle exec sidekiq -C config/sidekiq-task-interactive.yml -r ./core.rb
+autoscheduled: bundle exec sidekiq -C config/sidekiq-task-autoscheduled.yml -r ./core.rb
+enrichment: bundle exec sidekiq -C config/sidekiq-task-enrichment.yml -r ./core.rb
+application: bundle exec sidekiq -C config/sidekiq-app.yml -r ./core.rb
+puma: bundle exec  puma -C ./config/puma.rb

Rakefile

@@ -13,6 +13,7 @@ system_config_file = "#{intrigue_basedir}/config/config.json"
 database_config_file = "#{intrigue_basedir}/config/database.yml"
 sidekiq_interactive_config_file = "#{intrigue_basedir}/config/sidekiq-task-interactive.yml"
 sidekiq_autoscheduled_config_file = "#{intrigue_basedir}/config/sidekiq-task-autoscheduled.yml"
+sidekiq_enrichment_config_file = "#{intrigue_basedir}/config/sidekiq-task-enrichment.yml"
 sidekiq_app_config_file = "#{intrigue_basedir}/config/sidekiq-app.yml"
 control_script = "#{intrigue_basedir}/util/control.sh"
 
@@ -28,6 +29,7 @@ task :clean do
   FileUtils.mv  database_config_file, "#{database_config_file}.backup"
   FileUtils.mv  sidekiq_interactive_config_file, "#{sidekiq_interactive_config_file}.backup"
   FileUtils.mv  sidekiq_autoscheduled_config_file, "#{sidekiq_autoscheduled_config_file}.backup"
+  FileUtils.mv  sidekiq_enrichment_config_file, "#{sidekiq_enrichment_config_file}.backup"
   FileUtils.mv  sidekiq_app_config_file, "#{sidekiq_app_config_file}.backup"
   FileUtils.mv  geolocation_database, "#{geolocation_database}.backup"
   FileUtils.mv  web_accounts_list, "#{web_accounts_list}.backup"
@@ -83,16 +85,22 @@ task :setup do
 
   ## Copy sidekiq task worker config into place
   puts "[+] Setting up task worker config...."
-  if File.exist? sidekiq_interactive_config_file && sidekiq_autoscheduled_config_file && sidekiq_app_config_file
+  if File.exist?(sidekiq_interactive_config_file &&
+      sidekiq_autoscheduled_config_file &&
+      sidekiq_enrichment_config_file &&
+      sidekiq_app_config_file)
     puts "[ ] File already exists, skipping: #{sidekiq_interactive_config_file}"
     puts "[ ] File already exists, skipping: #{sidekiq_autoscheduled_config_file}"
+    puts "[ ] File already exists, skipping: #{sidekiq_enrichment_config_file}"
     puts "[ ] File already exists, skipping: #{sidekiq_app_config_file}"
   else
     puts "[+] Copying: #{sidekiq_interactive_config_file}.default"
     puts "[+] Copying: #{sidekiq_autoscheduled_config_file}.default"
+    puts "[+] Copying: #{sidekiq_enrichment_config_file}.default"
     puts "[+] Copying: #{sidekiq_app_config_file}.default"
     FileUtils.cp "#{sidekiq_interactive_config_file}.default", sidekiq_interactive_config_file
     FileUtils.cp "#{sidekiq_autoscheduled_config_file}.default", sidekiq_autoscheduled_config_file
+    FileUtils.cp "#{sidekiq_enrichment_config_file}.default", sidekiq_enrichment_config_file
     FileUtils.cp "#{sidekiq_app_config_file}.default", sidekiq_app_config_file
   end
 

app/helpers.rb

@@ -2,6 +2,18 @@ module Intrigue
 module Task
 module Helper
 
+  def entity_exists?(project, entity_type, entity_name)
+    puts "Checking for existence of an entity with type: #{entity_type} and name: #{entity_name} in project: #{project.name}"
+    Intrigue::Model::Entity.scope_by_project_and_type(project.name,entity_type).each do |e|
+      if e.unique_names.include? entity_name
+        puts "Found! #{entity_name}"
+        return e
+      end
+    end
+  puts "Not Found! #{entity_name}"
+  false
+  end
+
   ###
   ### Helper method for starting a task run
   ###
@@ -16,7 +28,7 @@ def start_task(queue, project, existing_scan_result, task_name, entity, depth, o
       :options => options,
       :handlers => [],
       :base_entity => entity,
-      :autoscheduled => (queue == "task_autoscheduled"),
+      :autoscheduled => (queue == "task_autoscheduled" || queue == "task_enrichment"),
       :depth => depth
     })
 

app/models/capabilities/export_graph.rb

@@ -7,26 +7,30 @@ def export_graph_json
         # generate the nodes
         nodes = []
         edges = []
-        edge_count = 1
+        edge_count = 0
 
         self.task_results.each do |t|
 
+          next if t.base_entity.type_string == "Uri"
+
           # add the base entity first (provided it hasn't been deleted)
           x = { :id => t.base_entity.id, :label => "#{t.base_entity.name}", :type => t.base_entity.type_string}
-          #x[:color] = "lightgrey" if t.base_entity.secondary
           nodes << x unless t.base_entity.deleted?
 
           # then for each of the entities, generate the node and edges. skip if deleted.
           t.entities.each do |e|
-            #next unless e.type_string == "WebServer"
+
+            next if e.type_string == "Uri"
+
             x = { :id => e.id, :label => "#{e.name}", :type => e.type_string } #unless e.secondary
-            #x[:color] = "lightgrey" if e.secondary
+            #x[:color] = "lightgrey" if e.type_string == "Uri"
+
             nodes << x unless e.deleted?
 
             unless t.base_entity.deleted? || e.deleted?
-              edges << {"id" => edge_count, "source" => t.base_entity.id, "target" => e.id}
-              edge_count += 1
+              edges << {"id" => edge_count += 1, "source" => t.base_entity.id, "target" => e.id}
             end
+
           end
         end
 

app/models/entity.rb

@@ -1,35 +1,20 @@
 module Intrigue
   module Model
-=begin
-    class AliasMapping < Sequel::Model
-      plugin :validation_helpers
-      #self.raise_on_save_failure = false
-
-      many_to_one :source, :class => :'Intrigue::Model::Entity', :key => :source_id
-      many_to_one :target, :class => :'Intrigue::Model::Entity', :key => :target_id
-
-      def validate
-        super
-        validates_unique([:source_id, :target_id]) # only allow a single alias
-      end
-    end
-=end
 
     class Entity < Sequel::Model
       plugin :validation_helpers
       plugin :single_table_inheritance, :type
-      plugin :serialization, :json, :details
+      plugin :serialization, :json, :details #, :name
       self.raise_on_save_failure = false
 
       #set_allowed_columns :type, :name, :details, :project_id
 
       many_to_many :task_results
       many_to_one  :project
-      #many_to_many :aliases, :left_key=>:source_id,:right_key=>:target_id, :join_table=>:alias_mappings, :class=>self
 
       def validate
         super
-        validates_unique([:name, :project_id])
+        validates_unique([:project_id, :name])
       end
 
       def self.scope_by_project(project_name)
@@ -42,6 +27,11 @@ def self.scope_by_project_and_type(project, type)
         where(Sequel.&(:project_id => named_project_id, :type => type.to_s))
       end
 
+      # easy way to refer to all names (overridden in some entities)
+      def unique_names
+        [name]
+      end
+
       def deleted?
         return true if deleted
       false
@@ -58,9 +48,6 @@ def created_by?(task_name)
       false
       end
 
-      def names
-      end
-
       def allowed_tasks
         ### XXX - this needs to be limited to tasks that accept this type
         TaskFactory.allowed_tasks_for_entity_type(type_string)
@@ -100,6 +87,10 @@ def form
         </div>}
       end
 
+      def get_detail(key)
+        details[key]
+      end
+
       def self.descendants
         x = ObjectSpace.each_object(Class).select{ |klass| klass < self }
       end
@@ -114,10 +105,6 @@ def export_hash
           :name =>  name,
           :deleted => deleted,
           :details => details,
-          #:aliases => self.aliases.map{|x| {
-          #  "id" => x.id,
-          #  "type" => x.type,
-          #  "name" => x.name }},
           :task_results => task_results.map{ |t| {:id => t.id, :name => t.name } }
         }
       end
@@ -126,30 +113,26 @@ def export_json
         export_hash.to_json
       end
 
-      # export id, type, name, and details on a single line, removing spaces and commas
-      def export_csv
-        export_string = "#{id},#{type_string},#{name.gsub(/[\,,\s]/,"")},"
-        details.each{|k,v| export_string << "#{k}=#{v};".gsub(/[\,,\s]/,"") }
-      export_string
-      end
-
-      def export_tsv
-        export_string = "#{id}\t#{type_string}\t#{name}\t"
-        details.each{|k,v| export_string << "#{k}##{v};" }
-      export_string
-      end
-
       private
       def _escape_html(text)
         Rack::Utils.escape_html(text)
         text
       end
 
-      # have an easy way to sort and hash all the aliases (which should include
-      # all names)
-      def _unique_name
-        string = aliases.split(",").sort_by{|x| x.downcase}.join(", ")
-        Digest::SHA1.hexdigest string
+
+      ### VALIDATIONS!
+
+      # https://tools.ietf.org/html/rfc1123
+      def _v4_regex
+        /(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/
+      end
+
+      def _v6_regex
+        /^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$/
+      end
+
+      def _dns_regex
+        /^(\w|-|\.).*\.(\w|-|\.).*$/
       end
 
     end

app/models/task_result.rb

@@ -23,24 +23,30 @@ def validate
       def start(queue)
 
         if queue == "task_autoscheduled"
-          autoscheduled = true
-
           self.job_id = Sidekiq::Client.push({
             "class" => Intrigue::TaskFactory.create_by_name(task_name).class.to_s,
             "queue" => "task_autoscheduled",
             "retry" => true,
             "args" => [id]
           })
 
-          save
+        elsif queue == "task_enrichment"
+          self.job_id = Sidekiq::Client.push({
+            "class" => Intrigue::TaskFactory.create_by_name(task_name).class.to_s,
+            "queue" => "task_enrichment",
+            "retry" => true,
+            "args" => [id]
+          })
 
         else # start it in the task queues
           task = Intrigue::TaskFactory.create_by_name(task_name)
           self.job_id = task.class.perform_async id
-          save
+
         end
 
-      job_id
+        save
+
+      self.job_id
       end
 
       def log
@@ -64,19 +70,6 @@ def task
         Intrigue::TaskFactory.create_by_name(task_name)
       end
 
-      ### Export!
-      def export_csv
-        output_string = ""
-        entities.each{ |x| output_string << x.export_csv << "\n" }
-      output_string
-      end
-
-      def export_tsv
-        export_string = ""
-        entities.map{ |x| export_string << x.export_tsv + "\n" }
-      export_string
-      end
-
       def export_hash
         {
           "id" => id,

app/routes/results.rb

@@ -20,6 +20,7 @@ class IntrigueApp < Sinatra::Base
       entity_id = @params["entity_id"]
       depth = @params["depth"].to_i
       current_project = Intrigue::Model::Project.first(:name => @project_name)
+      entity_name = "#{@params["attrib_name"]}"
 
       # Construct the attributes hash from the parameters. Loop through each of the
       # parameters looking for things that look like attributes, and add them to our
@@ -32,7 +33,7 @@ class IntrigueApp < Sinatra::Base
         end
       end
 
-      # hack! remove the name, no longer needed in the details
+      # HACK! add the name to aliases, name detail no longer needed
       entity_details.delete("name")
 
       # Construct an entity from the data we have
@@ -43,17 +44,14 @@ class IntrigueApp < Sinatra::Base
         return unless entity_type
 
         # TODO - SECURITY - validate that it's a valid entity type before we eval
-
         klass = eval("Intrigue::Entity::#{entity_type}")
-        entity_name = "#{@params["attrib_name"]}"
 
-        # TODO - we'll need to check all aliases of all entities within the project here
-        entity = Intrigue::Model::Entity.scope_by_project_and_type(@project_name, klass.to_s).first(:name => entity_name)
+        entity = entity_exists?(current_project,entity_type,entity_name)
 
         unless entity
           entity = Intrigue::Model::Entity.create(
-            { :type => klass,
-              :name => entity_name,
+            { :name => entity_name,
+              :type => klass,
               :details => entity_details,
               :project => current_project
             })