Merge branch 'master' of git@github.com:jpf/domain-profiler

README.md

@@ -81,13 +81,14 @@ Thanks
 * [Brian Lopez](http://github.com/brianmario) for help with Ruby and RSpec.
 * Team Cymru for the awesome [IP to ASN API](http://www.team-cymru.org/Services/ip-to-asn.html).
 * Hacker News user <a href="http://news.ycombinator.com/user?id=brett">brett</a> for the inspiration.
+* [Jed Smith](http://jedsmith.org/) for the advice on using ARIN's WHOIS to find more accurate OrgName data on IP addresses.
 
 LICENSE
 =======
 
     (The MIT License)
 
-    Copyright (c) 2009 Joel Franusic <joel@franusic.com>
+    Copyright (c) 2009-2010 Joel Franusic <joel@franusic.com>
 
     Permission is hereby granted, free of charge, to any person obtaining
     a copy of this software and associated documentation files (the

Rakefile

@@ -1,4 +1,5 @@
 require 'rake'
+gem "rspec", "~>1.3.0"
 require 'spec/rake/spectask'
 
 desc "Run all examples with RCov"
@@ -20,4 +21,4 @@ Spec::Rake::SpecTask.new('tests_with_rcov') do |t|
   t.rcov = true
   t.rcov_opts = ['--exclude', 'spec']
 
-end
+end

config/name-lookups.yaml

@@ -12,6 +12,8 @@
   - softlayer technologies inc.
   - softlayer technologies
   - softlayer.com
+  - theplanet.com internet services
+  - theplanet.com internet services, inc.
 :netsol: 
   :name: Network Solutions
   :dba: 
@@ -46,6 +48,7 @@
   - mydyndns.org
   - dynect.net
   - dynamic network services, inc
+  - dynamic network services, inc.
 :amazon: 
   :name: Amazon.com
   :dba: 
@@ -95,11 +98,6 @@
   - aol transit data network
   - aol, llc
   - aol.com
-:theplanet: 
-  :name: The Planet
-  :dba: 
-  - theplanet.com internet services
-  - theplanet.com internet services, inc.
 :pair: 
   :name: pair Networks
   :dba: 
@@ -122,6 +120,7 @@
   :dba: 
   - yahoo.com
   - inktomi corporation
+  - altavista company
 :savis: 
   :name: Savvis
   :dba: 
@@ -213,11 +212,6 @@
   :name: DomainPeople
   :dba: 
   - domainpeople, inc.
-:slicehost: 
-  :name: Slicehost
-  :dba: 
-  - slicehost llc
-  - slicehost.net
 :netnames: 
   :name: NetNames
   :dba: 
@@ -241,6 +235,13 @@
   :dba: 
   - rackspace.com
   - rackspace.com, ltd.
+  - rackspace hosting
+  - stabletransit.com
+:slicehost:
+  :name: Slicehost
+  :dba:
+  - slicehost llc
+  - slicehost.net
 :internap: 
   :name: Internap
   :dba: 
@@ -263,3 +264,27 @@
   :name: Nettica
   :dba:
   - nettica.com
+  - nettica domains, inc.
+:he: 
+  :name: Hurricane Electric
+  :dba: 
+  - hurricane electric, inc.
+:dreamhost: 
+  :name: DreamHost
+  :dba: 
+  - new dream network, llc
+  - dreamhost.com
+:zerigo: 
+  :name: Zerigo
+  :dba: 
+  - zerigo.net
+:linode: 
+  :name: Linode
+  :dba: 
+  - linode
+  - linode.com
+  - global net access, llc
+:issue:
+  :name: DNS issue
+  :dba:
+  - opendns, llc

domain-profiler-mind-map.mm

@@ -0,0 +1,58 @@
+<map version="0.8.1">
+<!-- To view this file, download free mind mapping software FreeMind from http://freemind.sourceforge.net -->
+<node CREATED="1249535541029" ID="Freemind_Link_404127227" MODIFIED="1249535554161" TEXT="Domain Profiler">
+<node CREATED="1249535591163" ID="_" MODIFIED="1249535596580" POSITION="right" TEXT="Motavations">
+<node CREATED="1249535637238" ID="Freemind_Link_1370486696" MODIFIED="1249535640864" TEXT="i do this all the time">
+<node CREATED="1249535786527" ID="Freemind_Link_401559732" MODIFIED="1249535793526" TEXT="I wanted a way to AUZTOMATE"/>
+</node>
+<node CREATED="1249535911638" ID="Freemind_Link_283580413" MODIFIED="1249535934175" TEXT="wanted an easy way to see what decisions other comanies have made"/>
+<node CREATED="1249535934547" FOLDED="true" ID="Freemind_Link_1584666302" MODIFIED="1249535939141" TEXT="Decisions">
+<node CREATED="1249535939617" ID="Freemind_Link_1819729497" MODIFIED="1249535947617" TEXT="ones that cost money">
+<node CREATED="1249535954938" ID="Freemind_Link_315756999" MODIFIED="1249535957500" TEXT="Web Host"/>
+<node CREATED="1249535958215" ID="Freemind_Link_1910640874" MODIFIED="1249535962521" TEXT="DNS Host"/>
+<node CREATED="1249535962798" ID="Freemind_Link_1818244965" MODIFIED="1249535965224" TEXT="Mail Host"/>
+<node CREATED="1249535965828" ID="Freemind_Link_708358152" MODIFIED="1249535970645" TEXT="Registrar"/>
+<node CREATED="1249535970938" ID="Freemind_Link_525657358" MODIFIED="1249535986917" TEXT="SSL Issuer"/>
+<node CREATED="1249535978589" ID="Freemind_Link_871738618" MODIFIED="1249535982303" TEXT="SSL cert type"/>
+</node>
+<node CREATED="1249535947941" ID="Freemind_Link_257002870" MODIFIED="1249535953949" TEXT="ones that don&apos;t cost money">
+<node CREATED="1249536016651" ID="Freemind_Link_898068561" MODIFIED="1249536026153" TEXT="DNS TTL&apos;s"/>
+<node CREATED="1249536032043" FOLDED="true" ID="Freemind_Link_24671833" MODIFIED="1249536157310" TEXT="SPF records">
+<node CREATED="1249536157803" ID="Freemind_Link_1158443831" MODIFIED="1249536159877" TEXT="do they exist?">
+<node CREATED="1249536160889" ID="Freemind_Link_1377275554" MODIFIED="1249536167521" TEXT="what are they set to?"/>
+</node>
+</node>
+<node CREATED="1249536169148" ID="Freemind_Link_1147510388" MODIFIED="1249536176261" TEXT="Expiration for domain"/>
+<node CREATED="1249536176793" ID="Freemind_Link_1140307037" MODIFIED="1249536179851" TEXT="Expiration for SSL"/>
+<node CREATED="1249536180855" FOLDED="true" ID="Freemind_Link_1240505849" MODIFIED="1249536198458" TEXT="Is there more than one result for:">
+<node CREATED="1249536199454" ID="Freemind_Link_1009947120" MODIFIED="1249536200625" TEXT="A"/>
+<node CREATED="1249536202444" ID="Freemind_Link_698695469" MODIFIED="1249536203183" TEXT="MX"/>
+<node CREATED="1249536203508" ID="Freemind_Link_1384144196" MODIFIED="1249536204863" TEXT="NS"/>
+</node>
+<node CREATED="1249536208009" ID="Freemind_Link_8690959" MODIFIED="1249536216177" TEXT="Are hosts in different ASes"/>
+<node CREATED="1249536220387" ID="Freemind_Link_1238754757" MODIFIED="1249536229699" TEXT="Main webpage valid XHTML"/>
+<node CREATED="1249536230087" FOLDED="true" ID="Freemind_Link_734525337" MODIFIED="1249536348436" TEXT="Detect type of">
+<node CREATED="1249536348729" ID="Freemind_Link_1871644528" MODIFIED="1249536351921" TEXT="Frontend"/>
+<node CREATED="1249536352151" ID="Freemind_Link_1276249868" MODIFIED="1249536354833" TEXT="Mail Server"/>
+<node CREATED="1249536355060" ID="Freemind_Link_1801591655" MODIFIED="1249536364084" TEXT="DNS Server"/>
+</node>
+<node CREATED="1249536384968" ID="Freemind_Link_1171770420" MODIFIED="1249536393144" TEXT="DNS has a * record?"/>
+</node>
+</node>
+</node>
+<node CREATED="1249535598121" ID="Freemind_Link_374603701" MODIFIED="1249535615963" POSITION="right" TEXT="Expected findings">
+<node CREATED="1249535795379" ID="Freemind_Link_1331996271" MODIFIED="1249535825414" TEXT="homogenous results for YC startups">
+<node CREATED="1249535833624" ID="Freemind_Link_911959890" MODIFIED="1249535837313" TEXT="esp for DNS"/>
+</node>
+</node>
+<node CREATED="1249535616248" ID="Freemind_Link_1613849065" MODIFIED="1249535620474" POSITION="right" TEXT="Actual findings">
+<node CREATED="1249535828818" ID="Freemind_Link_1382512003" MODIFIED="1249535845805" TEXT="YC startups not homogenous">
+<node CREATED="1249535846697" ID="Freemind_Link_1653588069" MODIFIED="1249535849139" TEXT="esp for DNS"/>
+</node>
+<node CREATED="1249536449090" ID="Freemind_Link_1544229921" MODIFIED="1249536461134" TEXT="discovered previously unknown companies/services">
+<node CREATED="1249536461748" ID="Freemind_Link_1844525773" MODIFIED="1249536465038" TEXT="Mark Monitor"/>
+<node CREATED="1249536465602" ID="Freemind_Link_1893076224" MODIFIED="1249536470938" TEXT="Corporation Services Company"/>
+</node>
+</node>
+</node>
+</map>

lib/domain-profiler.rb

@@ -1,6 +1,7 @@
 require 'ipaddr'
 require 'domain-profiler/dns'
 require 'domain-profiler/whois'
+require 'domain-profiler/whois-orgname'
 require 'domain-profiler/ssl'
 require 'domain-profiler/name'
 require 'domain-profiler/information'
@@ -26,7 +27,12 @@ def initialize(host)
 
 def orgname(input)
   begin
-    IPtoASN.new(input).asn.orgname
+    name = WhoisOrgName.new(input).orgname
+    if name == 'Unknown'
+      IPtoASN.new(input).asn.orgname
+    else
+      name
+    end
   rescue
     input
   end

lib/domain-profiler/whois-orgname.rb

@@ -0,0 +1,33 @@
+# Thanks to HN user 'jedsmith' for telling me about this! (http://news.ycombinator.com/item?id=2081312)
+class WhoisOrgName
+  def initialize(input, use_cache = true)
+    begin
+      ip_address = IPAddr.new(input)
+    rescue
+      raise ArgumentError, "Input '#{input}' is invalid"
+    end
+
+
+    version = 'v1'
+    filename = "cache/whois-orgname-#{ip_address}.#{version}.cache"
+    @data = {}
+    if File.exists?(filename) and use_cache
+      @data = open(filename) { |f| Marshal.load(f) }
+    else
+      @data = `sleep 1; whois 'n #{ip_address}'`
+      open(filename, 'w') { |f| Marshal.dump(@data, f) } if use_cache
+    end
+  end
+
+  def grep(pattern)
+    begin
+      @data.grep(pattern).to_s.split(':', 2)[1].strip ||= 'Unknown'
+    rescue
+      'Unknown'
+    end
+  end
+
+  def orgname
+    grep(/OrgName:/)
+  end
+end

profile-list

@@ -19,11 +19,16 @@ class Array
 end
 
 
+if ARGV.length == 0
+   print "Usage: ./profile-list [file containing one domain per line] 'header to put in HTML output'\n"
+   exit
+end
+
 filename = ARGV[0]
 file = File.new(filename)
 
 hosts = {}
-list_name = 'REPLACE ME'
+list_name = ARGV[1] ? ARGV[1] : 'REPLACE ME'
 
 file.map {|host|
   if match_data = host.match(/^#List:(.*)/)