initial commit

Gemfile

@@ -0,0 +1,51 @@
+source 'https://rubygems.org'
+
+# API
+gem 'sinatra'
+
+
+# Application Server
+gem 'puma'
+
+# Background Tasks
+gem 'sidekiq'
+
+# Redis access
+gem 'redis'
+
+# Encoding
+gem 'iconv'
+
+# Web hooks
+gem 'rest-client'
+
+# UI
+gem 'haml'
+gem 'sass'
+
+# Testing
+gem 'rspec'
+gem 'rack-test'
+
+# Debugging
+gem 'pry'
+
+# Tasks
+gem 'net-dns'           # dns_cache_snoop
+gem 'dnsruby'           # dns_zone_transfer
+gem 'whois'             # dns_zone_transfer, whois
+gem 'googleajax'        # search_google
+gem 'anemone'           # web_app_spider
+gem 'geoip'             # geolocate_host
+#gem 'openssl'           # web_app_gather_ssl_cert
+gem 'shodan'            # search_shodan
+gem 'nmap-parser'       # nmap_scan
+gem 'screencap'         # phantom_js
+gem 'twitter'           # twitter_account_following, etc
+
+# Client::Search::*Scraper
+gem 'nokogiri'
+
+# Cli
+gem 'thor'
+gem 'diffy'

Gemfile.lock

@@ -0,0 +1,155 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.3.8)
+    anemone (0.7.2)
+      nokogiri (>= 1.3.0)
+      robotex (>= 1.0.0)
+    buftok (0.2.0)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    coderay (1.1.0)
+    connection_pool (2.1.3)
+    diff-lcs (1.2.5)
+    diffy (3.0.7)
+    dnsruby (1.58.0)
+    domain_name (0.5.23)
+      unf (>= 0.0.5, < 1.0.0)
+    equalizer (0.0.11)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    geoip (1.5.0)
+    googleajax (1.0.1)
+      json (>= 1.0.0)
+    haml (4.0.6)
+      tilt
+    hitimes (1.2.2)
+    http (0.6.4)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    http_parser.rb (0.6.0)
+    i18n (0.7.0)
+    iconv (1.0.4)
+    json (1.8.2)
+    memoizable (0.4.2)
+      thread_safe (~> 0.3, >= 0.3.1)
+    method_source (0.8.2)
+    mime-types (2.4.3)
+    mini_portile (0.6.2)
+    minitest (5.5.1)
+    multipart-post (2.0.0)
+    naught (1.0.0)
+    net-dns (0.8.0)
+    netrc (0.10.3)
+    nmap-parser (0.3.5)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    phantomjs (1.9.8.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    puma (2.11.1)
+      rack (>= 1.1, < 2.0)
+    rack (1.6.0)
+    rack-protection (1.5.3)
+      rack
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    redis (3.2.1)
+    redis-namespace (1.5.2)
+      redis (~> 3.0, >= 3.0.4)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    robotex (1.0.0)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.3)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+    sass (3.4.13)
+    screencap (0.1.2)
+      phantomjs
+    shodan (1.0.0)
+      json (>= 1.4.6)
+    sidekiq (3.3.3)
+      celluloid (>= 0.16.0)
+      connection_pool (>= 2.1.1)
+      json
+      redis (>= 3.0.6)
+      redis-namespace (>= 1.3.1)
+    simple_oauth (0.3.1)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    slop (3.6.0)
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    tilt (2.0.1)
+    timers (4.0.1)
+      hitimes
+    twitter (5.14.0)
+      addressable (~> 2.3)
+      buftok (~> 0.2.0)
+      equalizer (~> 0.0.9)
+      faraday (~> 0.9.0)
+      http (~> 0.6.0)
+      http_parser.rb (~> 0.6.0)
+      json (~> 1.8)
+      memoizable (~> 0.4.0)
+      naught (~> 1.0)
+      simple_oauth (~> 0.3.0)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.6)
+    whois (3.5.9)
+      activesupport (>= 3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  anemone
+  diffy
+  dnsruby
+  geoip
+  googleajax
+  haml
+  iconv
+  net-dns
+  nmap-parser
+  nokogiri
+  pry
+  puma
+  rack-test
+  redis
+  rest-client
+  rspec
+  sass
+  screencap
+  shodan
+  sidekiq
+  sinatra
+  thor
+  twitter
+  whois

Procfile

@@ -0,0 +1 @@
+web: bundle exec ruby core.rb -p $PORT

README.md

@@ -0,0 +1,59 @@
+### Environment and external dependencies
+
+The following are presumed available and configured in your environment
+ - redis
+ - sudo
+ - nmap
+ - zmap
+ - masscan
+ - API keys (copy config/config.yml.default -> config/config.yml)
+
+### To start:
+
+Make sure you have redis installed and running.
+
+```
+bundle exec rackup ## start the UI
+bundle exec sidekiq -r ./core.rb ## Start the background processing
+```
+
+
+
+### curl usage:
+
+Request the task type, specify an entity, and the appropriate options:
+````
+curl -s -X POST -H "Content-Type: application/json" -d '{ "task": "example", "entity": { "type": "Host", "attributes": { "name": "4.4.4.4" } }, "options": {} }' http://localhost:9292/task_runs/
+````
+
+### core-cli interface
+
+A command line utility has been added for convenience
+
+List tasks:
+```
+./core-cli.rb list
+```
+
+Start a task:
+```
+./core-cli.rb start dns_lookup_forward DnsRecord#wow.com
+```
+
+Start a task with options:
+```
+./core-cli.rb start dns_lookup_forward DnsRecord#wow.com resolver#8.8.8.8
+```
+
+Check for a subdomain on iastate.edu:
+```
+INTRIGUE_ENV=production  ./core-cli.rb start_and_wait dns_brute_sub DnsRecord#iastate.edu resolver=8.8.8.8#brute_list=a,b,c,proxy,test,www
+http://core.intrigue.io/task_runs/05d975a7-4527-4f76-bfe3-6b1e8c6fa581
+DnsRecord#www.iastate.edu
+Host#129.186.23.166
+```
+
+Check the top 1000 domains for the existence of security headers:
+```
+for x in `cat data/domains.txt | head -n 1000`; do ./core-cli.rb start_and_wait dns_sub_brute DnsRecord#http://$x;done
+```

Rakefile

@@ -0,0 +1,20 @@
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+task :default => :spec
+
+desc "Run Specs"
+begin
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError
+  # No rspec available
+end
+
+desc "Run Integration Specs"
+begin
+  RSpec::Core::RakeTask.new(:integration) do |t|
+    t.rspec_opts = "--pattern spec/integration/*_spec.rb"
+  end
+rescue LoadError
+  # No rspec available
+end

config.ru

@@ -0,0 +1,6 @@
+#\ -s puma
+
+require './core'
+
+#run Sinatra::Application
+run Rack::URLMap.new('/' => Sinatra::Application, '/sidekiq' => Sidekiq::Web)

config/config.yml.default

@@ -0,0 +1,13 @@
+---
+intrigue_engine_id: XXX               # generated at startup
+intrigue_api_hackerone: XXX           # http://intrigue.io/signup/
+bing_api_key: XXX                     # https://datamarket.azure.com/dataset/bing/search
+corpwatch_api_key: XXX                # http://api.corpwatch.org/register.php
+pipl_api_key: samplekey               # http://dev.pipl.com/choosekey
+shodan_api_key: XXX                   # https://account.shodan.io/login
+data_gov_key: XXX                     # https://api.data.gov/signup/
+google_safebrowsing_lookup_key: XXX   # https://developers.google.com/safe-browsing/lookup_guide#GettingStarted
+twitter_consumer_key: XXX             # https://dev.twitter.com
+twitter_consumer_secret: XXX          # https://dev.twitter.com
+twitter_access_token: XXX             # https://dev.twitter.com
+twitter_access_token_secret: XXX      # https://dev.twitter.com

config/sidekiq.yml

@@ -0,0 +1,6 @@
+:verbose: false
+#:logfile: ./log/intrigue-sidekiq.log
+:pidfile: ./tmp/pids/intrigue-sidekiq.pid
+:concurrency: 50
+# Set timeout to 8 on Heroku, longer if you manage your own systems.
+:timeout: 30