feat: add urlscan.io support (#19)

ninoseki · Oct 12, 2018 · c1ddc7b · c1ddc7b
1 parent 5fb9bb9
commit c1ddc7b
Show file tree

Hide file tree

Showing 9 changed files with 3,382 additions and 126 deletions.
diff --git a/README.md b/README.md
@@ -27,6 +27,8 @@ Ryo is a yet another website recon tool powered by Ruby.
   - By using [SimpleWhatWeb](https://github.com/ninoseki/SimpleWhatWeb)
 - [x] Whois
   - By using [DomainBigData](https://domainbigdata.com/)
+- [x] Website history
+  - By using [urlscan.io](https://urlscan.io)
 - [x] Threading support
 - [ ] Port scanning
 
@@ -50,6 +52,8 @@ Commands:
   ryo shodan URL      # Discover Shodan information of a given URL
   ryo subdomain URL   # Discover subdomains of a given URL
   ryo tech URL        # Discover used technolgies of a given URL
+  ryo urlscan URL     # Discover urlscan.io scan results of a given URL
+  ryo whois URL       # Discover whois information of a given URL
 ```
 
 In order to use Shodan search, please set your Shodan API key as `SHODAN_API_KEY` environment variable.
@@ -69,43 +73,7 @@ $ ryo all http://localhost:8000 | jq .
   "cert": {
     "error": "N/A"
   },
-  "dir": [
-    "http://localhost:8000/.DS_Store",
-    "http://localhost:8000/.env",
-    "http://localhost:8000/.git/FETCH_HEAD",
-    "http://localhost:8000/.git/COMMIT_EDITMSG",
-    "http://localhost:8000/.git/config",
-    "http://localhost:8000/.git/HEAD",
-    "http://localhost:8000/.git/branches/",
-    "http://localhost:8000/.git/index",
-    "http://localhost:8000/.git/description",
-    "http://localhost:8000/.git/info/exclude",
-    "http://localhost:8000/.git/",
-    "http://localhost:8000/.git/info/",
-    "http://localhost:8000/.git/logs/HEAD",
-    "http://localhost:8000/.git/hooks/",
-    "http://localhost:8000/.git/logs/",
-    "http://localhost:8000/.git/logs/refs/heads/master",
-    "http://localhost:8000/.git/logs/refs/remotes/origin/HEAD",
-    "http://localhost:8000/.git/packed-refs",
-    "http://localhost:8000/.git/logs/refs/remotes/origin/master",
-    "http://localhost:8000/.git/refs/heads/master",
-    "http://localhost:8000/.git/refs/",
-    "http://localhost:8000/.git/refs/remotes/origin/HEAD",
-    "http://localhost:8000/.git/refs/remotes/origin/master",
-    "http://localhost:8000/.gitignore",
-    "http://localhost:8000/.gitignore/",
-    "http://localhost:8000/.git/objects/",
-    "http://localhost:8000/.travis.yml",
-    "http://localhost:8000/Bin/",
-    "http://localhost:8000/Gemfile",
-    "http://localhost:8000/Gemfile.lock",
-    "http://localhost:8000/LICENSE",
-    "http://localhost:8000/README.md",
-    "http://localhost:8000/Rakefile",
-    "http://localhost:8000/bin/",
-    "http://localhost:8000/readme.md"
-  ],
+  "dir": [],
   "dns": {
     "A": {
       "Status": 3,
@@ -116,16 +84,16 @@ $ ryo all http://localhost:8000 | jq .
       "CD": false,
       "Question": [
         {
-          "name": "localhost.",
+          "name": "127.0.0.1.",
           "type": 1
         }
       ],
       "Authority": [
         {
           "name": ".",
           "type": 6,
-          "TTL": 85076,
-          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018091601 1800 900 604800 86400"
+          "TTL": 48909,
+          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018101001 1800 900 604800 86400"
         }
       ]
     },
@@ -138,16 +106,16 @@ $ ryo all http://localhost:8000 | jq .
       "CD": false,
       "Question": [
         {
-          "name": "localhost.",
+          "name": "127.0.0.1.",
           "type": 28
         }
       ],
       "Authority": [
         {
           "name": ".",
           "type": 6,
-          "TTL": 31568,
-          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018091600 1800 900 604800 86400"
+          "TTL": 53205,
+          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018101001 1800 900 604800 86400"
         }
       ]
     },
@@ -160,16 +128,16 @@ $ ryo all http://localhost:8000 | jq .
       "CD": false,
       "Question": [
         {
-          "name": "localhost.",
+          "name": "127.0.0.1.",
           "type": 5
         }
       ],
       "Authority": [
         {
           "name": ".",
           "type": 6,
-          "TTL": 10464,
-          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018091501 1800 900 604800 86400"
+          "TTL": 86391,
+          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018101101 1800 900 604800 86400"
         }
       ]
     },
@@ -182,16 +150,16 @@ $ ryo all http://localhost:8000 | jq .
       "CD": false,
       "Question": [
         {
-          "name": "localhost.",
+          "name": "127.0.0.1.",
           "type": 15
         }
       ],
       "Authority": [
         {
           "name": ".",
           "type": 6,
-          "TTL": 66679,
-          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018091601 1800 900 604800 86400"
+          "TTL": 86398,
+          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018101101 1800 900 604800 86400"
         }
       ]
     },
@@ -204,16 +172,16 @@ $ ryo all http://localhost:8000 | jq .
       "CD": false,
       "Question": [
         {
-          "name": "localhost.",
+          "name": "127.0.0.1.",
           "type": 2
         }
       ],
       "Authority": [
         {
           "name": ".",
           "type": 6,
-          "TTL": 44604,
-          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018091600 1800 900 604800 86400"
+          "TTL": 47564,
+          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018101001 1800 900 604800 86400"
         }
       ]
     },
@@ -226,16 +194,16 @@ $ ryo all http://localhost:8000 | jq .
       "CD": false,
       "Question": [
         {
-          "name": "localhost.",
+          "name": "127.0.0.1.",
           "type": 6
         }
       ],
       "Authority": [
         {
           "name": ".",
           "type": 6,
-          "TTL": 86397,
-          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018091601 1800 900 604800 86400"
+          "TTL": 63738,
+          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018101100 1800 900 604800 86400"
         }
       ]
     },
@@ -248,24 +216,41 @@ $ ryo all http://localhost:8000 | jq .
       "CD": false,
       "Question": [
         {
-          "name": "localhost.",
+          "name": "127.0.0.1.",
           "type": 16
         }
       ],
       "Authority": [
         {
           "name": ".",
           "type": 6,
-          "TTL": 86397,
-          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018091601 1800 900 604800 86400"
+          "TTL": 86398,
+          "data": "a.root-servers.net. nstld.verisign-grs.com. 2018101101 1800 900 604800 86400"
         }
       ]
     }
   },
   "shodan": {
     "error": "Invalid IP"
   },
-  "subdomain": [],
+  "subdomain": [
+    {
+      "domain": "127.0.0.1",
+      "ip": "N/A"
+    },
+    {
+      "domain": "develop.127.0.0.1",
+      "ip": "N/A"
+    },
+    {
+      "domain": "loja01.127.0.0.1",
+      "ip": "N/A"
+    },
+    {
+      "domain": "www.127.0.0.1",
+      "ip": "N/A"
+    }
+  ],
   "tech": {
     "HTTPServer": [
       {
@@ -298,6 +283,41 @@ $ ryo all http://localhost:8000 | jq .
       }
     ]
   },
+  "urlscan": {
+    "results": [
+      {
+        "task": {
+          "visibility": "public",
+          "method": "api",
+          "time": "2018-09-12T19:15:09.391Z",
+          "source": "api",
+          "url": "http://price-wise.net"
+        },
+        "stats": {
+          "uniqIPs": 6,
+          "consoleMsgs": 0,
+          "dataLength": 401944,
+          "encodedDataLength": 155968,
+          "requests": 8
+        },
+        "page": {
+          "country": "US",
+          "server": "nginx/1.10.2",
+          "city": "Dallas",
+          "domain": "price-wise.net",
+          "ip": "159.65.190.189",
+          "asnname": "DIGITALOCEAN-ASN - DigitalOcean, LLC, US",
+          "asn": "AS14061",
+          "url": "http://price-wise.net/",
+          "ptr": ""
+        },
+        "uniq_countries": 1,
+        "_id": "120e5b5d-4d3e-4c63-ba0f-cf3e5ceb4219",
+        "result": "https://urlscan.io/api/v1/result/120e5b5d-4d3e-4c63-ba0f-cf3e5ceb4219"
+      }
+    ],
+    "total": 1
+  },
   "whois": {
     "globa_stats": {},
     "registrant": {}

diff --git a/lib/ryo.rb b/lib/ryo.rb
@@ -24,6 +24,7 @@ def self.discover(uri, options)
     h[:shodan] = Plugin::Shodan.discover(target.ip) if options[:shodan] || options[:all]
     h[:subdomain] = Plugin::Subdomain.discover(target.fld) if options[:subdomain] || options[:all]
     h[:tech] = Plugin::Tech.discover(target.uri) if options[:tech] || options[:all]
+    h[:urlscan] = Plugin::URLscan.discover(target.domain) if options[:urlscan] || options[:all]
     h[:whois] = Plugin::Whois.discover(target.domain) if options[:whois] || options[:all]
     h
   end

diff --git a/lib/ryo/cli.rb b/lib/ryo/cli.rb
@@ -44,12 +44,19 @@ def whois(url)
       puts hash.to_json
     end
 
+    desc "urlscan URL", "Discover urlscan.io scan results of a given URL"
+    def urlscan(url)
+      hash = run_discovery(url, urlscan: true)
+      puts hash.to_json
+    end
+
     desc "discover URL", "Run discovery plugin(s) against a given URL"
     method_option :cert, type: :boolean, default: false
     method_option :dir, type: :boolean, default: false
     method_option :shodan, type: :boolean, default: false
     method_option :subdomain, type: :boolean, default: false
     method_option :tech, type: :boolean, default: false
+    method_option :urlscan, type: :boolean, default: false
     method_option :whois, type: :boolean, default: false
     def discovery(url)
       hash = run_discovery(url, options)

diff --git a/lib/ryo/plugin.rb b/lib/ryo/plugin.rb
@@ -6,4 +6,5 @@
 require_relative "./plugin/shodan"
 require_relative "./plugin/subdomain"
 require_relative "./plugin/tech"
+require_relative "./plugin/urlscan"
 require_relative "./plugin/whois"
diff --git a/lib/ryo/plugin/urlscan.rb b/lib/ryo/plugin/urlscan.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Ryo
+  module Plugin
+    class URLscan
+      def endpoint
+        "https://urlscan.io/api/v1"
+      end
+
+      def discover(query)
+        q = '"' + query.to_s + '"'
+        res = Client.http.get("#{endpoint}/search", params: { q: q })
+        JSON.parse(res.body)
+      end
+
+      def self.discover(query)
+        new.discover(query)
+      end
+    end
+  end
+end
diff --git a/spec/cli_spec.rb b/spec/cli_spec.rb
@@ -52,6 +52,16 @@
       expect(json).to be_a(Hash)
     end
   end
+  describe "#urlscan" do
+    before {
+      allow(Ryo::Plugin::URLscan).to receive(:discover).and_return({})
+    }
+    it "should output a JSON" do
+      output = capture(:stdout) { subject.start %w(urlscan http://localhost) }
+      json = JSON.parse(output)
+      expect(json).to be_a(Hash)
+    end
+  end
   describe "#discover" do
     context "when given an empty options" do
       it "should return a JSON with an error message" do