discovery meta-analysis plugin:

  * Updated to add remarks to logged issues
  * Added spec

  [Issue #209]

CHANGELOG.md

@@ -51,6 +51,8 @@
   - Meta-analysis
       - Timing-attacks: Updated to add a remark to affected issues about the
             suboptimal state of the server while the issue was identified.
+      - Discovery: Updated to add a remark to affected issues about the
+            extreme similarities between issues of similar type.
 - Modules
   - General
       - Updated module names along with some descriptions and issue names.

plugins/defaults/meta/remedies/discovery.rb

@@ -38,6 +38,11 @@ class Arachni::Plugins::Discovery < Arachni::Plugin::Base
     # in common which makes it possible to spot them without much bother
     SIMILARITY_TOLERANCE = 0.25
 
+    REMARK = "This issue was logged by a directory-busting/discovery module but " +
+        "the response for the resource it identified is very similar to responses " +
+        "of other identified resources. This probably means that the server responses " +
+        "were too erratic to be successfully identified as custom 404s and thus ignored."
+
     def prepare
         wait_while_framework_running
     end
@@ -52,22 +57,22 @@ def run
         # URL path => size of responses
         response_size_per_path  = {}
 
-        framework.auditstore.issues.each_with_index do |issue, idx|
+        framework.modules.issues.each_with_index do |issue, idx|
             next if !issue.tags.includes_tags?( :discovery )
 
             # discovery issues only have 1 variation
-            variation = issue.variations.first
+            #variation = issue.variations.first
 
             # grab the URL path of the issue which will actually be the
             # parent of the logged page because whatever is under the parent path
             # will control the behavior under that path
             #
             # did that make any sense?
-            path = File.dirname( uri_parse( variation.url ).path )
+            path = File.dirname( uri_parse( issue.url ).path )
 
             # gathering total response sizes for issues per path
             response_size_per_path[path] ||= 0
-            response_size_per_path[path] += variation.response.size
+            response_size_per_path[path] += issue.response.size
 
             # categorize issues per path as well
             issues_per_path[path] ||= []
@@ -88,10 +93,10 @@ def run
             # on the other hand, valid responses will be dissimilar since the
             # discovery modules look for different things.
             diffs_per_path[path] = if !diffs_per_path[path]
-                 variation['response']
-            else
-                diffs_per_path[path].rdiff( variation['response'] )
-            end
+                                       issue['response']
+                                    else
+                                        diffs_per_path[path].rdiff( issue['response'] )
+                                    end
         end
 
         issues = []
@@ -103,6 +108,16 @@ def run
             issues |= issues_per_path[path] if similarity >= SIMILARITY_TOLERANCE
         end
 
+        issue_digests = issues.map { |i| i['hash'] }
+        framework.modules.issues.each do |issue|
+            next if !issue_digests.include?( issue.digest )
+
+            issue.add_remark :meta_analysis, REMARK
+
+            # Requires manual verification.
+            issue.verification = true
+        end
+
         register_results( issues ) if !issues.empty?
     end
 

spec/plugins/meta/remedies/discovery_spec.rb

@@ -3,13 +3,19 @@
 describe name_from_filename do
     include_examples 'plugin'
 
-    #before( :all ) do
-    #    framework.modules.load_by_tags :discovery
-    #end
+    before( :all ) do
+        options.url = url
+        options.audit :forms
 
-    #def results
-    #    :nil
-    #end
+        framework.modules.load :common_files
+    end
+
+    it 'should mark issues with too similar response bodies as needing manual verification and add remarks' do
+        run
+        framework.auditstore.issues.each do |issue|
+            issue.variations.map( &:verification ).uniq == [true]
+            issue.variations.first.remarks[:meta_analysis].should be_true
+        end
+    end
 
-    #easy_test
 end

spec/servers/plugins/meta/remedies/discovery.rb

@@ -1,7 +1,8 @@
 require 'sinatra'
+require File.dirname( __FILE__ ) + '/../../../modules/module_server'
 
-get '/*' do
-    # we add the request path and random number to avoid
-    # being seen as a custom 404 handler
-    env['REQUEST_PATH'] + 'same crap' + rand( 9 ).to_s
+framework.modules[:common_files].filenames.each do |name|
+    get( "/#{name}" ) { 'stuff' }
 end
+
+get( '/' ) {}