improvements in sets of children (name servers with multiple IP addre…

…sses)

bin/dnstraverse

@@ -71,6 +71,10 @@ def progress_main(args)
       print " -- resolving"
     end
     print "\n"
+  when :new_referral_set then
+    refid = r.refid[0,r.refid.rindex('.')]
+    print "#{refid} #{r.parent_ip}"
+    print "\n"
   when :answer_fast then
     print o[:verbose] ? referral_txt_verbose(r) : referral_txt_normal(r)
     puts " -- completed earlier (#{r.replaced_by.refid})"

lib/dnstraverse/decoded_query.rb

@@ -120,7 +120,6 @@ def inside_bailiwick?(name)
     end
 
     NOERROR = Dnsruby::RCode.NOERROR
-    NXDOMAIN = Dnsruby::RCode.NXDOMAIN
 
     def process
       return process_exception if @message.is_a? Exception
@@ -155,9 +154,9 @@ def process_error
         @error_message = "Server failure (SERVFAIL)"
         when Dnsruby::RCode::NXDOMAIN
         @error_message = "No such domain (NXDOMAIN)"
-        when Dnsruby::RCode.NOTIMP
+        when Dnsruby::RCode::NOTIMP
         @error_message = "Not implemented (NOTIMP)"
-        when Dnsruby::RCode.REFUSED
+        when Dnsruby::RCode::REFUSED
         @error_message = "Refused"
       else
         @error_message = @message.rcode.to_s

lib/dnstraverse/referral.rb

@@ -364,21 +364,24 @@ def is_rootroot?
 
     # process this Referral object:
     #   query each IP in @serverips and create a Response object
-    #   return an array of children
+    #   return an array of sets of children
     def process(args)
       raise "This Referral object has already been processed" if processed?
       raise "You need to resolve this Referral object" unless resolved?
       @processed = true
       unless (server) then
         # special case - no server means start from the top with the roots
         process_add_roots(args)
-        return @children.values.flatten
+        #return @children.values.flatten
+        return [ @children.values.flatten ] # one set
       end
       process_normal(args)
       # return a set of Referral objects that need to be processed
       # this is just using @serverips for ordering the children properly
       # because we numbered them all already
-      return @serverips.map {|ip| @children[ip] }.flatten.select {|x| x.is_a? Referral }
+      #return @serverips.map {|ip| @children[ip] }.flatten.select {|x| x.is_a? Referral }
+      # use serverips to keep ordering, skip key: entries
+      return @serverips.select {|ip| @children[ip] }.map {|ip| @children[ip] } # array of sets of children
     end
 
     def process_add_roots(args)
@@ -398,10 +401,9 @@ def process_add_roots(args)
 
     def process_normal(args)
       Log.debug { "process " + self.to_s }
-      childsets = @serverips.reject {|ip| ip =~ /^key:/ }.count
-      childset = 0
+      # two phases in order to calculate number of childsets
+      childsets = 0
       for ip in @serverips do
-        childset+= 1
         Log.debug { "Process normal #{ip}" }
         next if ip =~ /^key:/ # resolve failed on something
         m = nil
@@ -416,12 +418,19 @@ def process_normal(args)
                                       :bailiwick => @bailiwick,
                                       :infocache => @infocache, :ip => ip,
                                       :server => @server,
+                                      :parent_ip => @parent_ip,
                                       :decoded_query_cache => @decoded_query_cache)
         Log.debug { "Process normal #{ip} - done making response" }
         @responses[ip] = r
-        case r.status
-          when :restart, :referral then
-          Log.debug { "Process normal #{ip} - making referrals" }
+        if r.status == :restart or r.status == :referral then
+          childsets+= 1
+        end
+      end
+      childset = 0
+      @responses.each_pair do |ip, r|
+        if r.status == :restart or r.status == :referral then
+          childset+= 1
+          Log.debug { "Process normal #{ip} - making referrals (childset #{childset})" }
           refid =  childsets == 1 ? @refid :  "#{@refid}.#{childset}"
           refkey = childsets == 1 ? @refkey : "#{@refkey}.#{childsets}"
           @children[ip] = make_referrals(:qname => r.endname,

lib/dnstraverse/response.rb

@@ -38,6 +38,7 @@ def initialize(args)
       @infocache = InfoCache.new(args[:infocache]) # our infocache
       @starters = nil # initial servers for :referral/:restart
       @starters_bailiwick = nil # for initial servers for :referral/:restart
+      @parent_ip = args[:parent_ip] # passed in in case we get referral_lame, for the key
       evaluate
       update_stats_key
       return self
@@ -56,8 +57,10 @@ def method_missing(key, *args, &block)
     def update_stats_key
       r = @decoded_query
       @stats_key = "key:#{@status}:#{r.ip}:#{@server}:#{r.qname}:#{r.qclass}:#{r.qtype}"
-      if @stats == :exception and r.message.is_a? Exception then
+      if @status == :exception and r.message.is_a? Exception then
         @stats_key+= ":#{r.message}"
+      elsif @status == :referral_lame then
+        @stats_key+= ":#{@parent_ip}"
       end
     end
 

lib/dnstraverse/traverser.rb

@@ -156,9 +156,10 @@ def run(r, args = {})
           r = stack.pop
           r.answer_calculate
           report_progress r, :stage => :answer
-          r.cleanup(cleanup)
-          if @fast then
+          special = r.responses.values.map {|x| x.status }.include?(:referral_lame)
+          if @fast and r.status == :normal and (not special) then
             # store away in @answered hash so we can lookup later
+            # normal status only, i.e. not :loop or :noglue
             key = "#{r.qname}:#{r.qclass}:#{r.qtype}:#{r.server}:#{r.txt_ips_verbose}".downcase!
             Log.debug { "Fast mode cache store: #{key}" }
             @answered[key] = r
@@ -168,6 +169,7 @@ def run(r, args = {})
             @seen[r.server.downcase].concat(r.ips_as_array)
             @seen[r.server.downcase].uniq!
           end
+          r.cleanup(cleanup)
           next
         end
         # ok time to process a new item
@@ -176,11 +178,11 @@ def run(r, args = {})
           key = "#{r.qname}:#{r.qclass}:#{r.qtype}:#{r.server}:#{r.txt_ips_verbose}".downcase!
           Log.debug { "Fast mode cache lookup: #{key}" }
           # check for previously stored answer
-          # special case noglue situation, don't use previous answer
+          # special case noglue and loop situations
           # because attributes are complicated for stats collection and
           # we don't want to merge them together - creating the noglue
           # response object is fast anyway
-          if @answered.key?(key) and (not r.noglue?) then
+          if @answered.key?(key) and (not r.noglue?) and (not r.loop?) then
             Log.debug { "Fast method - completed #{r}" }
             r.parent.replace_child(r, @answered[key])
             report_progress r, :stage => :answer_fast
@@ -199,30 +201,31 @@ def run(r, args = {})
         # put a placeholder on the stack
         stack << r << :calc_answer
         # get children, one set per IP address of this name server in array
-        children = r.process({})
+        children_sets = r.process({})
         # now report progress.  we already can tell whether this will be
         # completed in fast mode or not, so we report this information
-        total_sets = children.map { |c| c.parent_ip }.uniq.count
         # if there is more than one set (i.e. a DNS server has more than one
         # IP address and we had to do multiple queries), the children will
         # been numbered with an extra set digit, and we want to report this to
         # the user interface
         seen_parent_ip = Hash.new
-        children.each do |c|
-          if total_sets > 1 and not seen_parent_ip.include?(c.parent_ip) then
-            report_progress c, :stage => :new_referral_set
-            seen_parent_ip[c.parent_ip] = true
-          end
-          if @fast
-            key = "#{c.qname}:#{c.qclass}:#{c.qtype}:#{c.server}:#{c.txt_ips_verbose}".downcase!
-            stage = @answered.key?(key) ? :new_fast : :new
-          else
-            stage = :new
+        for children in children_sets do
+          children.each do |c|
+            if children_sets.length > 1 and not seen_parent_ip.include?(c.parent_ip) then
+              report_progress c, :stage => :new_referral_set
+              seen_parent_ip[c.parent_ip] = true
+            end
+            if @fast
+              key = "#{c.qname}:#{c.qclass}:#{c.qtype}:#{c.server}:#{c.txt_ips_verbose}".downcase!
+              stage = @answered.key?(key) ? :new_fast : :new
+            else
+              stage = :new
+            end
+            report_progress c, :stage => stage
           end
-          report_progress c, :stage => stage
         end
         # push the children on the stack
-        stack.push(*children.reverse)
+        stack.push(*children_sets.flatten.reverse)
       end
     end
 

lib/dnstraverse/version.rb

@@ -2,7 +2,7 @@ module DNSTraverse
   module Version
     MAJOR = 0
     MINOR = 1
-    PATCH = 9
+    PATCH = 10
     STRING = "#{MAJOR}.#{MINOR}.#{PATCH}"
   end
 end