v0.1.0

.gitignore

@@ -48,3 +48,5 @@ Gemfile.lock
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
+
+.rspec_status

README.md

@@ -1,6 +1,10 @@
 # Ryo
 
-Ryo is a yet another recon tool powered by Ruby.
+[![Build Status](https://travis-ci.org/ninoseki/ryo.svg?branch=master)](https://travis-ci.org/ninoseki/ryo)
+[![Maintainability](https://api.codeclimate.com/v1/badges/7e5f124034cd51768567/maintainability)](https://codeclimate.com/github/ninoseki/ryo/maintainability)
+[![Coverage Status](https://coveralls.io/repos/github/ninoseki/ryo/badge.svg)](https://coveralls.io/github/ninoseki/ryo)
+
+Ryo is a yet another website recon tool powered by Ruby.
 
 *Note*: I'm working on this just because for fun and study purposes.
 
@@ -21,25 +25,96 @@ Ryo is a yet another recon tool powered by Ruby.
 
 ## Installation
 
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'ryo'
+```sh
+$ gem install ryo
 ```
 
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install ryo
-
 ## Usage
 
-TODO: Write usage instructions here
+```sh
+$ ryo
+Commands:
+  ryo all URL         # Run all discovery plugins against a given URL
+  ryo dir URL         # Discover directories and files belong to a given URL
+  ryo help [COMMAND]  # Describe available commands or one specific command
+  ryo subdomain URL   # Discover subdomains of a given URL
+  ryo tech URL        # Discover used technolgies of a given URL
+```
+
+```sh
+# use Webrick as a local http server
+# $ ruby -rwebrick -e 'WEBrick::HTTPServer.new(:DocumentRoot => "./", :Port => 8000).start'
+$ ryo all http://localhost:8000 | jq .
+{
+  "dir": [
+    "http://localhost:8000/.git/",
+    "http://localhost:8000/.git/branches/",
+    "http://localhost:8000/.git/COMMIT_EDITMSG",
+    "http://localhost:8000/.git/config",
+    "http://localhost:8000/.git/description",
+    "http://localhost:8000/.git/FETCH_HEAD",
+    "http://localhost:8000/.git/HEAD",
+    "http://localhost:8000/.git/hooks/",
+    "http://localhost:8000/.git/index",
+    "http://localhost:8000/.git/info/",
+    "http://localhost:8000/.git/info/exclude",
+    "http://localhost:8000/.git/logs/",
+    "http://localhost:8000/.git/logs/HEAD",
+    "http://localhost:8000/.git/logs/refs/heads/master",
+    "http://localhost:8000/.git/logs/refs/remotes/origin/HEAD",
+    "http://localhost:8000/.git/objects/",
+    "http://localhost:8000/.git/packed-refs",
+    "http://localhost:8000/.git/refs/",
+    "http://localhost:8000/.git/refs/heads/master",
+    "http://localhost:8000/.git/refs/remotes/origin/HEAD",
+    "http://localhost:8000/.gitignore",
+    "http://localhost:8000/.gitignore/",
+    "http://localhost:8000/.travis.yml",
+    "http://localhost:8000/Bin/",
+    "http://localhost:8000/bin/",
+    "http://localhost:8000/Gemfile",
+    "http://localhost:8000/Gemfile.lock",
+    "http://localhost:8000/LICENSE",
+    "http://localhost:8000/Rakefile",
+    "http://localhost:8000/README.md",
+    "http://localhost:8000/readme.md"
+  ],
+  "subdomain": [],
+  "tech": {
+    "HTTPServer": [
+      {
+        "name": "server string",
+        "string": "WEBrick/1.4.2 (Ruby/2.5.1/2018-03-29)",
+        "certainty": 100
+      }
+    ],
+    "Ruby": [
+      {
+        "regexp": [
+          "Ruby"
+        ],
+        "search": "headers[server]",
+        "certainty": 100
+      },
+      {
+        "regexp": [
+          "WEBrick"
+        ],
+        "search": "headers[server]",
+        "certainty": 100
+      }
+    ],
+    "Title": [
+      {
+        "name": "page title",
+        "string": "Index of /",
+        "certainty": 100
+      }
+    ]
+  }
+}
+```
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
-

exe/ryo

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift("#{__dir__}/../lib")
+
+require "ryo"
+
+Ryo::CLI.start

lib/ryo.rb

@@ -4,20 +4,21 @@
 require "oga"
 
 require "ryo/client"
-require "ryo/domain"
 require "ryo/target"
 require "ryo/plugin"
 
+require "ryo/cli"
+
 require "ryo/version"
 
 module Ryo
-  def self.discover(uri)
+  def self.discover(uri, options)
     target = Target.new(uri)
 
     h = {}
-    h[:dir] = Plugin::Dir.discover(target.uri)
-    h[:subdomain] = Plugin::Subdomain.discover(target.domain)
-    h[:tech] = Plugin::Tech.discover(target.uri)
+    h[:dir] = Plugin::Dir.discover(target.uri) if options[:dir] || options[:all]
+    h[:subdomain] = Plugin::Subdomain.discover(target.fld) if options[:subdomain] || options[:all]
+    h[:tech] = Plugin::Tech.discover(target.uri) if options[:tech] || options[:all]
     h
   end
 end

lib/ryo/cli.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Ryo
+  class CLI < Thor
+    desc "dir URL", "Discover directories and files belong to a given URL"
+    def dir(url)
+      hash = discover(url, dir: true)
+      puts hash.to_json
+    end
+
+    desc "subdomain URL", "Discover subdomains of a given URL"
+    def subdomain(url)
+      hash = discover(url, subdomain: true)
+      puts hash.to_json
+    end
+
+    desc "tech URL", "Discover used technolgies of a given URL"
+    def tech(url)
+      hash = discover(url, tech: true)
+      puts hash.to_json
+    end
+
+    desc "all URL", "Run all discovery plugins against a given URL"
+    def all(url)
+      hash = discover(url, all: true)
+      puts hash.to_json
+    end
+
+    no_commands do
+      def discover(url, options)
+        Ryo.discover url, options
+      rescue StandardError => e
+        { error: "Warning: #{e}" }
+      end
+    end
+  end
+end

lib/ryo/plugin/subdomain.rb

@@ -7,10 +7,10 @@
 module Ryo
   module Plugin
     module Subdomain
-      def self.discover(domain)
+      def self.discover(fld)
         subdomains = []
-        subdomains << DNSDumpster.discover(domain)
-        subdomains << FindSubDomains.discover(domain)
+        subdomains << DNSDumpster.discover(fld)
+        subdomains << FindSubDomains.discover(fld)
         subdomains.flatten.uniq.sort_by { |e| e[:domain] }
       end
     end

lib/ryo/plugin/subdomain/base.rb

@@ -4,17 +4,17 @@ module Ryo
   module Plugin
     module Subdomain
       class Base
-        attr_reader :domain
-        def initialize(domain)
-          @domain = Domain.new(domain)
+        attr_reader :fld
+        def initialize(fld)
+          @fld = fld
         end
 
         def endpoint
           raise NotImplementedError, "You must implement #{self.class}##{__method__}"
         end
 
         def fetch_body
-          res = Client.http.get("#{endpoint}/#{domain.fld}")
+          res = Client.http.get("#{endpoint}/#{fld}")
           res.body.to_s
         end
 
@@ -30,8 +30,8 @@ def discover
           parse
         end
 
-        def self.discover(domain)
-          new(domain).discover
+        def self.discover(fld)
+          new(fld).discover
         end
       end
     end

lib/ryo/plugin/subdomain/dnsdumpster.rb

@@ -11,7 +11,7 @@ def endpoint
         def fetch_body
           res = Client.http.get(endpoint)
           csrftoken = res.cookies.find { |c| c.name == "csrftoken" }.value
-          params = { csrfmiddlewaretoken: csrftoken, targetip: domain.fld }
+          params = { csrfmiddlewaretoken: csrftoken, targetip: fld }
 
           res = Client.http.cookies(csrftoken: csrftoken).headers(referer: endpoint).post(endpoint, form: params)
           res.body.to_s

lib/ryo/target.rb

@@ -1,11 +1,45 @@
 # frozen_string_literal: true
 
+require "ipaddr"
+
 module Ryo
   class Target
     attr_reader :uri, :domain
     def initialize(uri)
       @uri = URI.parse(uri)
-      @domain = Domain.new(@uri.host)
+      @domain = @uri.host
+    end
+
+    def fld
+      @fld ||= String.new.tap do |out|
+        removed_tlds_domain = domain.gsub(tlds_regexp, "")
+        # test.com => ["test"]
+        # dev.test.com => ["dev", "test"]
+        parts = removed_tlds_domain.split(".")
+        if parts.length == 1 || ip?
+          out << domain
+        else
+          idx = domain.split(".").index(parts.last)
+          out << domain.split(".")[idx..-1].join(".")
+        end
+      end
+    end
+
+    private
+
+    def tlds
+      File.readlines(File.expand_path("./aux/tlds.txt", __dir__)).map(&:chomp).compact
+    end
+
+    def tlds_regexp
+      Regexp.new tlds.map { |domain| "#{domain.split('.').join('\\.')}$" }.join("|")
+    end
+
+    def ip?
+      IPAddr.new(domain.to_s)
+      true
+    rescue IPAddr::InvalidAddressError => _
+      false
     end
   end
 end

ryo.gemspec

@@ -35,4 +35,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "http", "~> 3.3"
   spec.add_dependency "oga", "~> 2.15"
   spec.add_dependency "simple_whatweb", "~> 0.2"
+  spec.add_dependency "thor", "~> 0.19"
 end