diff --git a/MANIFEST b/MANIFEST new file mode 100644 index 0000000000..a209f03167 --- /dev/null +++ b/MANIFEST @@ -0,0 +1,91 @@ +# file GENERATED by distutils, do NOT edit +setup.py +bin/attackkeys +bin/checkknownkeys +bin/detectdupkeys +bin/getmoduli +bin/ipdata +bin/ipinfo +bin/ipinfohost +bin/httpd-ivre +bin/nmap2db +bin/p0f2db +bin/passiverecon2db +bin/passivereconworker +bin/plotdb +bin/runscans +bin/runscans-agent +bin/scancli +bin/scanstatus +doc/AGENT.md +doc/DOCKER.md +doc/FAST-INSTALL-AND-FIRST-RUN.md +doc/INSTALL.md +doc/LICENSE-EXTERNAL.md +doc/LICENSE.md +doc/README.md +doc/WEBUI.md +docker/agent/Dockerfile +docker/base/Dockerfile +docker/base/ivre.conf +docker/client/Dockerfile +docker/db/Dockerfile +docker/web/Dockerfile +docker/web/doku-conf-acl.auth.php +docker/web/doku-conf-plugins.local.php +docker/web/doku-conf-local.php +docker/web/doku-conf-users.auth.php +docker/web/nginx-default-site +honeyd/sshd +ivre/__init__.py +ivre/config.py +ivre/db.py +ivre/geoiputils.py +ivre/graphroute.py +ivre/keys.py +ivre/mathutils.py +ivre/nmapopt.py +ivre/scanengine.py +ivre/target.py +ivre/utils.py +ivre/xmlnmap.py +passiverecon/passiverecon.bro +passiverecon/passiverecon2db-ignore.example +web/cgi-bin/scanjson.py +web/cgi-bin/scanjsonconfig-sample.py +web/dokuwiki/backlinks.patch +web/dokuwiki/doc/agent.txt +web/dokuwiki/doc/docker.txt +web/dokuwiki/doc/fast-install-and-first-run.txt +web/dokuwiki/doc/install.txt +web/dokuwiki/doc/license-external.txt +web/dokuwiki/doc/license.txt +web/dokuwiki/doc/readme.txt +web/dokuwiki/doc/webui.txt +web/dokuwiki/media/logo.png +web/static/config-sample.js +web/static/favicon-loading.gif +web/static/favicon.png +web/static/index.html +web/static/ivre.css +web/static/ivre.js +web/static/loading.gif +web/static/logo.png +web/static/world-110m.json +web/static/templates/filters.html +web/static/templates/menu.html +web/static/templates/progressbar.html +web/static/templates/subview-host-summary.html +web/static/templates/subview-port-summary.html +web/static/templates/subview-service-summary.html +web/static/templates/view-hosts.html +web/static/templates/view-scripts-only.html +web/static/an/js/angular.js +web/static/bs/css/bootstrap-responsive.css +web/static/bs/css/bootstrap.css +web/static/bs/img/glyphicons-halflings-white.png +web/static/bs/img/glyphicons-halflings.png +web/static/bs/js/bootstrap.js +web/static/d3/js/d3.v3.min.js +web/static/d3/js/topojson.v1.min.js +web/static/jq/jquery.js diff --git a/README.md b/README.md new file mode 100644 index 0000000000..3cdb6f060e --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +This file is part of IVRE. + +Copyright 2011 - 2014 [Pierre LALET](mailto:pierre.lalet@cea.fr) + +# What is it? # + +IVRE (Instrument de veille sur les réseaux extérieurs) or DRUNK +(Dynamic Recon of UNKnown networks) is a network recon framework, +including two modules for passive recon (one p0f-base and one +bro-based) and one module for active recon (mostly nmap-based, with a +bit of zmap). + +The advertising slogans are: + + - (in French): IVRE, il scanne Internet. + - (in English): Know the networks, get DRUNK! + +The names IVRE and DRUNK have been chosen as a tribute to "Le +Taullier". + +# Documentation # + +See [doc/README](doc/README.md) (and `doc/*` files) for more +information. + +On a server with the IVRE web server properly installed with a +Dokuwiki notepad, the `doc/*` files are available under the `doc:` +namespace (e.g., `doc:readme` for the [doc/README](doc/README.md) +file). + +On a client with IVRE installed, you can use a `--help` option with +most IVRE CLI tools, and use `help(ivre.module)` with most IVRE Python +sub-modules. + +# License # + +IVRE is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +IVRE is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +[along with IVRE](doc/LICENSE.md). If not, see [the gnu.org web +site](http://www.gnu.org/licenses/). + +# Support # + +Try `--help` for the CLI tools, `help()` under Python and the "HELP" +button in the web interface. + +Feel free to contact the author and offer him a beer if you need help! + +If you don't like beer, a good scotch or any other good alcoholic +beverage will do (it is the author's unalienable right to decide +whether a beverage is good or not). + +# Contributing # + +Code contributions (pull-requests) are of course welcome! + +The project needs scan results and capture files that can be provided +as examples. If you can contribute some samples, or if you want to +contribute some samples and would need some help to do so, or if you +can provide a server to run scans, please contact the author. diff --git a/agent/agent b/agent/agent new file mode 100755 index 0000000000..5f92215845 --- /dev/null +++ b/agent/agent @@ -0,0 +1,66 @@ +#! /bin/sh + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +NMAPOPTS="-vv -A --host-timeout 15m" +SLEEP="sleep 2" +THREADS=10 +STOREDOWN="true" + +INDIR=./input/ +CURDIR=./cur/ +OUTDIR=./output/ + +if [ "$TERM" != "screen" ] ; then + screen "$0" $@ + exit 0 +fi + +mkdir -p "$INDIR" "$CURDIR" "$OUTDIR" + +if [ -z "$INTHREAD" ] ; then + screen -X setenv INTHREAD 1 + for i in `seq $THREADS` ; do + screen "$0" $@ + done + exit 0 +fi + +while true ; do + [ -f "want_down" ] && break + fname=`ls -rt "$INDIR" | head -1` + if [ -z "$fname" ] ; then + $SLEEP + continue + fi + if ! mv "$INDIR/$fname" "$CURDIR/" ; then + continue + fi + if ! (nmap $NMAPOPTS -iL "$CURDIR/$fname" -oX "$CURDIR/$fname.xml") ; then + rm -f "$CURDIR/$fname.xml" + mv "$CURDIR/$fname" "$INDIR/" + $SLEEP + else + if [ "$STOREDOWN" = "false" ] && + grep -q -F ' +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + + +# This is an equivalent of runscans-agent (the --sync part only) that +# can do not need python. + +RSYNC="rsync" +# Uncomment the following line to use TOR +#RSYNC="torify rsync" + +MAINDIR="./agentsdata" + +# [user@]host:path [[user@]host:path [...]] +AGENTS="user@host:path" + +SLEEP="2" + +function agent_path () { + echo "${MAINDIR}/$(echo $1 | tr ':@' '__')" +} + +# make directories +for a in $AGENTS ; do + for d in input remoteinput remotecur remoteoutput ; do + echo -p "$(agent_path $a)/${d}/" + done +done + +# sync loop +while true ; do + for a in $AGENTS ; do + ${RSYNC} -a "$(agent_path $a)/input/" "$(agent_path $a)/remoteinput/" + ${RSYNC} --remove-source-files "$(agent_path $a)/input/" "${a}/input/" + ${RSYNC} --delete "${a}/input/" "$(agent_path $a)/remoteinput/" + ${RSYNC} --delete "${a}/cur/" "$(agent_path $a)/remotecur/" + ${RSYNC} --remove-source-files "${a}/output/" "$(agent_path $a)/remoteoutput/" + done + sleep ${SLEEP} +done diff --git a/bin/attackkeys b/bin/attackkeys new file mode 100755 index 0000000000..3c55835936 --- /dev/null +++ b/bin/attackkeys @@ -0,0 +1,97 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +"""This tool is based on the paper "Mining your Ps and Qs: Detection +of Widespread Weak Keys in Network Devices" +(https://factorable.net/paper.html). It is *really* slow. You should +probably consider to use the tool getmoduli to extract the moduli from +the databases and then use the tool fastgcd (available here: +https://factorable.net/resources.html). + +""" + +import ivre.keys +import ivre.db +from Crypto.Util.number import GCD +import datetime +import sys + + +def check_keys(k, keys): + """Checks whether the modulus of the new RSA key k has a one common +factor with one of the other keys. +""" + res = False + for kk in keys: + g = GCD(k, kk) + if g != 1 and g != k: + print g, k, kk + sys.stdout.flush() + res = True + if (len(keys) + 1) % 100 == 0: + print "%d unique keys handled in %d seconds" % ( + len(keys) + 1, + int(datetime.datetime.now().strftime('%s')) - starttime, + ) + sys.stdout.flush() + return res + + +def init(): + "Initialize global variables." + global starttime + starttime = int(datetime.datetime.now().strftime('%s')) + + +def test_keys(): + "Run the test with all the SSH and SSL keys we have in our database." + keys = {} + allkeys = [ivre.keys.SSHRSAKey(), ivre.keys.SSLRSAKey(), + ivre.keys.PassiveSSLRSAKey()] + for a in allkeys: + for k in a.get_keys(): + if 'modulus' in k: + kk = k['modulus'] + if kk in keys: + kkk = keys[kk] + if (k['host'], k['port']) not in kkk: + asnum = ivre.db.db.data.get(k['host']) + if asnum is not None and 'as_num' in asnum: + asnum = asnum['as_num'] + else: + asnum = -1 + kkk.add((asnum, k['host'], k['port'])) + keys[kk] = kkk + continue + if check_keys(kk, keys): + print k + sys.stdout.flush() + keys[kk] = set([(k['host'], k['port'])]) + else: + print "BUG ?", k + print "%d unique keys handled in %d seconds" % ( + len(keys), + int(datetime.datetime.now().strftime('%s')) - starttime, + ) + for k in keys: + if len(keys[k]) != 1: + print hex(k), len(keys[k]), keys[k] + +if __name__ == '__main__': + init() + test_keys() diff --git a/bin/checkknownkeys b/bin/checkknownkeys new file mode 100755 index 0000000000..5030d8d3a4 --- /dev/null +++ b/bin/checkknownkeys @@ -0,0 +1,56 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +import ivre.keys +import ivre.utils +import re +import sys + +KNOWN_KEYS = {} + + +def read_known_keys(listname, keytype, keylen, filename): + for l in open(filename): + add_known_key(listname, keytype, keylen, l) + + +def add_known_key(listname, keytype, keylen, key): + key = key.replace(':', '').strip().decode('hex') + k = KNOWN_KEYS.get(key, []) + k.append('%s-%s-%d' % (listname, keytype, keylen)) + KNOWN_KEYS[key] = k + + +def check_ssh_keys(): + for k in ivre.keys.SSHKey().get_hashes(): + if k['hash'] in KNOWN_KEYS: + svcs = "" % () + print 'SERVICE %s:%d has key %s listed as %r' % ( + ivre.utils.int2ip(k['host']), + k['port'], + k['hash'].encode('hex'), + KNOWN_KEYS[k['hash']]) + +if __name__ == '__main__': + read_known_keys('DebianWeakKeys', 'dsa', + 1024, 'data/DebianSSHWeakKeys-dsa-1024') + read_known_keys('DebianWeakKeys', 'rsa', + 2048, 'data/DebianSSHWeakKeys-rsa-2048') + add_known_key('LittleBlackBox', 'rsa', 2048, + '19:c6:8b:c2:1a:8a:ec:2d:d7:7a:f3:eb:d4:bc:68:d0') + check_ssh_keys() diff --git a/bin/detectdupkeys b/bin/detectdupkeys new file mode 100755 index 0000000000..1b4c1b8cd4 --- /dev/null +++ b/bin/detectdupkeys @@ -0,0 +1,48 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +import ivre.utils +import ivre.keys +import re +import sys + + +def scan_rsa_keys(keys, known={}): + for k in keys: + rec = known.get(k['modulus'], []) + rec.append(k) + known[k['modulus']] = rec + return known + + +def check_keys(keys): + for modulus in keys: + if len(keys[modulus]) > 1: + sys.stdout.write('%x' % modulus) + for i in keys[modulus]: + sys.stdout.write(' %s:%d' % (ivre.utils.int2ip(i['host']), + i['port'])) + sys.stdout.write('\n') + +if __name__ == '__main__': + known = {} + allkeys = [ivre.keys.SSHRSAKey(), ivre.keys.SSLRSAKey(), + ivre.keys.PassiveSSLRSAKey()] + for a in allkeys: + known = scan_rsa_keys(a.get_keys(), known=known) + check_keys(known) diff --git a/bin/getmoduli b/bin/getmoduli new file mode 100755 index 0000000000..463a9ea27e --- /dev/null +++ b/bin/getmoduli @@ -0,0 +1,92 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +"""This tool's output can be used with the tool fastgcd (available +here: https://factorable.net/resources.html) to efficiently perform +the attack described in the paper "Mining your Ps and Qs: Detection of +Widespread Weak Keys in Network Devices" +(https://factorable.net/paper.html). This option will really be faster +than the standalone tool attackkeys. + +To do so, you need to strip the output from the information after the +moduli. A simple sed with 's# .*##' will do the trick. + +""" + +import ivre.keys +import ivre.db + +if __name__ == '__main__': + import sys + import getopt + # FIXME: this will not work if .nmap and .passive have different + # backends + flt = ivre.db.db.nmap.flt_empty + bases = set() + progress = None + try: + opts, args = getopt.getopt(sys.argv[1:], + "p:f:", + ['passive-ssl', 'active-ssl', + 'active-ssh', + 'filter=']) + except getopt.GetoptError, err: + sys.stderr.write(str(err) + '\n') + sys.exit(-1) + for o, a in opts: + if o in ['-f', '--filter']: + # FIXME: this will not work if .nmap and .passive have + # different backends + flt = ivre.db.db.nmap.str2flt(a) + elif o == '--passive-ssl': + bases.add(ivre.keys.PassiveSSLRSAKey) + elif o == '--active-ssl': + bases.add(ivre.keys.SSLRSAKey) + elif o == '--active-ssh': + bases.add(ivre.keys.SSHRSAKey) + elif o in ['-p', '--progress']: + progress = int(a) + if progress == 0: + progress = 1 + else: + sys.stderr.write( + '%r %r not undestood (this is probably a bug).\n' % (o, a)) + sys.exit(-1) + moduli = {} + i = 0 + if not bases: + bases = set([ivre.keys.PassiveSSLRSAKey, + ivre.keys.SSLRSAKey, + ivre.keys.SSHRSAKey]) + for a in bases: + for k in a(cond=flt).get_keys(): + if 'modulus' in k: + m = k['modulus'] + h = (ivre.utils.int2ip(k['host']), k['port']) + # if 'subject' in k and 'issuer' in k: + # h = (k['subject'], k['issuer']) + if m not in moduli: + moduli[m] = set([h]) + i += 1 + if progress is not None and i % progress == 0: + sys.stderr.write("Got %d moduli.\r" % i) + elif h not in moduli[m]: + moduli[m].add(h) + sys.stderr.write("Got %d moduli. Completed !\n" % i) + for m in moduli: + sys.stdout.write('%x %d %r\n' % (m, len(moduli[m]), moduli[m])) diff --git a/bin/httpd-ivre b/bin/httpd-ivre new file mode 100755 index 0000000000..eb3c5857fa --- /dev/null +++ b/bin/httpd-ivre @@ -0,0 +1,101 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# This program is part of IVRE. +# +# Copyright 2011 - 2014 Pierre LALET +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +""" +This program is part of IVRE. +Copyright 2011 - 2014 Pierre LALET + +This program runs a simple httpd server to provide an out-of-the-box +access to the web user interface. + +This script should only be used for testing purposes. Production +deployments should use "real" web servers (IVRE has been successfully +tested with both Apache and Nginx). +""" + +from ivre import utils + +import os + +from BaseHTTPServer import HTTPServer +from CGIHTTPServer import CGIHTTPRequestHandler + +BASEDIR, CGIDIR, DOKUWIKIDIR = None, None, None + +class IvreRequestHandler(CGIHTTPRequestHandler): + """Request handler to serve both static files from + [PREFIX]/share/ivre/web/static/ and the CGI from + [PREFIX]/share/ivre/web/cgi-bin/. + + """ + def translate_path(self, path): + if not path: + return path + if path.startswith('/cgi-bin/'): + return os.path.join(CGIDIR, os.path.basename(path)) + if path.startswith('/dokuwiki/'): + path = os.path.basename(path).lower().replace(':', '/') + if '.' not in os.path.basename(path): + path += '.txt' + print os.path.join(DOKUWIKIDIR, path) + return os.path.join(DOKUWIKIDIR, path) + while path.startswith('/'): + path = path[1:] + path = os.path.join(BASEDIR, path) + if path.startswith(BASEDIR): + return path + raise ValueError("Invalid translated path") + +def parse_args(): + """Imports the available module to parse the arguments and return + the parsed arguments. + + """ + try: + import argparse + parser = argparse.ArgumentParser(description=__doc__) + except ImportError: + import optparse + parser = optparse.OptionParser(description=__doc__) + parser.parse_args_orig = parser.parse_args + parser.parse_args = lambda: parser.parse_args_orig()[0] + parser.add_argument('--bind-address', '-b', + help='(IP) Address to bind the server to (defaults ' + 'to 127.0.0.1).', + default="127.0.0.1") + parser.add_argument('--port', '-p', type=int, default=80, + help='(TCP) Port to use (defaults to 80)') + return parser.parse_args() + + +def main(): + """This function is called when __name__ == "__main__".""" + global BASEDIR, CGIDIR, DOKUWIKIDIR + BASEDIR = utils.guess_prefix(directory='web/static') + CGIDIR = utils.guess_prefix(directory='web/cgi-bin') + DOKUWIKIDIR = utils.guess_prefix(directory='dokuwiki') + if BASEDIR is None or CGIDIR is None or DOKUWIKIDIR is None: + raise Exception('Cannot find where IVRE is installed') + args = parse_args() + httpd = HTTPServer((args.bind_address, args.port), IvreRequestHandler) + httpd.serve_forever() + +if __name__ == '__main__': + print __doc__ + main() diff --git a/bin/ipdata b/bin/ipdata new file mode 100755 index 0000000000..521f311904 --- /dev/null +++ b/bin/ipdata @@ -0,0 +1,133 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +"""This tool can be used to manage IP addresses related data, such as +AS number et country information. + +""" + +import ivre.db +import ivre.geoiputils +import ivre.config + +if __name__ == '__main__': + import sys + import os + try: + import argparse + parser = argparse.ArgumentParser( + description='Access and query the IP metadata.') + USING_ARGPARSE = True + except ImportError: + import optparse + parser = optparse.OptionParser( + description='Access and query the IP metadata.') + parser.parse_args_orig = parser.parse_args + + def my_parse_args(): + res = parser.parse_args_orig() + res[0].ensure_value('ip', res[1]) + return res[0] + parser.parse_args = my_parse_args + parser.add_argument = parser.add_option + USING_ARGPARSE = False + TORUN = [] + parser.add_argument('--init', '--purgedb', action='store_true', + help='Purge or create and initialize the database.') + parser.add_argument('--ensure-indexes', action='store_true', + help='Create missing indexes (will lock the database).') + parser.add_argument('--download', action='store_true', + help='Fetch all data files.') + parser.add_argument('--country-csv', metavar='FILE', + help='Import FILE into countries database.') + parser.add_argument('--asnum-csv', metavar='FILE', + help='Import FILE into AS database.') + parser.add_argument('--city-csv', metavar='FILE', + help='Import FILE into cities database.') + parser.add_argument('--location-csv', metavar='FILE', + help='Import FILE into locations database.') + parser.add_argument('--import-all', action='store_true', + help='Import all files into databases.') + parser.add_argument('--quiet', "-q", action='store_true', + help='Quiet mode.') + if USING_ARGPARSE: + parser.add_argument('ip', nargs='*', metavar='IP', + help='Display results for specified IP addresses.') + args = parser.parse_args() + if args.init: + if os.isatty(sys.stdin.fileno()): + sys.stdout.write( + 'This will remove any country/AS information in your ' + 'database. Process ? [y/N] ') + ans = raw_input() + if ans.lower() == 'y': + ivre.db.db.data.init() + if args.ensure_indexes: + if os.isatty(sys.stdin.fileno()): + sys.stdout.write( + 'This will lock your database. Process ? [y/N] ') + ans = raw_input() + if ans.lower() == 'y': + ivre.db.db.data.ensure_indexes() + if args.download: + ivre.geoiputils.download_all(verbose=not args.quiet) + if args.city_csv is not None: + TORUN.append((ivre.db.db.data.feed_geoip_city, + [args.city_csv], + {"feedipdata": [ivre.db.db.passive]})) + if args.country_csv: + TORUN.append((ivre.db.db.data.feed_geoip_country, + [args.country_csv], + {"feedipdata": [ivre.db.db.passive]})) + if args.asnum_csv: + TORUN.append((ivre.db.db.data.feed_geoip_asnum, + [args.asnum_csv], + {"feedipdata": [ivre.db.db.passive]})) + if args.location_csv: + TORUN.append((ivre.db.db.data.feed_city_location, + [args.location_csv], {})) + if args.import_all: + for function, fname, kwargs in [ + (ivre.db.db.data.feed_geoip_city, + 'GeoIPCity-Blocks.csv', + {"feedipdata": [ivre.db.db.passive]}), + (ivre.db.db.data.feed_geoip_country, + 'GeoIPCountry.csv', + {"feedipdata": [ivre.db.db.passive]}), + (ivre.db.db.data.feed_geoip_asnum, + 'GeoIPASNum.csv', + {"feedipdata": [ivre.db.db.passive]}), + (ivre.db.db.data.feed_city_location, + 'GeoIPCity-Location.csv', {}), + ]: + TORUN.append((function, + [os.path.join(ivre.config.GEOIP_PATH, + fname)], + kwargs)) + for r in TORUN: + r[0](*r[1], **r[2]) + for a in args.ip: + if a.isdigit(): + a = int(a) + print a + for i in [ivre.db.db.data.country_byip(a), + ivre.db.db.data.as_byip(a), + ivre.db.db.data.location_byip(a)]: + if i: + for f in i: + print ' ', f, i[f] diff --git a/bin/ipinfo b/bin/ipinfo new file mode 100755 index 0000000000..f81f6a983e --- /dev/null +++ b/bin/ipinfo @@ -0,0 +1,389 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +import ivre.utils +from ivre.db import db + +import re +import time +import functools + + +def disp_rec(h): + print '\t', + if 'port' in h: + print h['port'], + if 'recontype' in h: + print h['recontype'], + if 'source' in h: + print h['source'], + if 'infos' not in h and 'value' in h: + if 'fullvalue' in h: + h['value'] = h['fullvalue'] + print h['value'], + if 'version' in h: + print h['version'], + if 'signature' in h: + print '[' + h['signature'] + ']', + if 'distance' in h: + print "at %s hop%s" % (h['distance'], h['distance'] > 1 and 's' or ''), + if 'count' in h: + print "(%d time%s)" % (h['count'], h['count'] > 1 and 's' or ''), + if 'firstseen' in h and 'lastseen' in h: + print datetime.datetime.fromtimestamp(int(h['firstseen'])), '-', + print datetime.datetime.fromtimestamp(int(h['lastseen'])), + if 'sensor' in h: + print h['sensor'], + print + if 'infos' in h: + for i in h['infos']: + print '\t\t', i + ':', + if i == 'domainvalue': + print h['infos'][i][0] + else: + print h['infos'][i] + + +def disp_recs_std(flt): + oa = None + c = db.passive.get(flt) + for h in c.sort([('addr', 1), ('recontype', 1), ('source', 1), + ('port', 1)]): + if not 'addr' in h or h['addr'] == 0: + continue + if oa != h['addr']: + if oa is not None: + print + oa = h['addr'] + print ivre.utils.int2ip(oa) + c = db.data.infos_byip(oa) + if c: + if 'country_code' in c: + print '\t', + print c['country_code'], + try: + print '[%s]' % db.data.country_name_by_code( + c['country_code']), + except: + pass + print + if 'as_num' in c: + print '\t', + print 'AS%d' % c['as_num'], + if 'as_name' in c: + print '[%s]' % c['as_name'], + print + elif 'as_name' in c: + print '\t', + print 'AS???? [%s]' % c['as_name'], + disp_rec(h) + + +def disp_recs_short(flt): + for addr in db.passive.get(flt).distinct('addr'): + print ivre.utils.int2ip(addr) + + +def disp_recs_distinct(field, flt): + for value in db.passive.get(flt).distinct(field): + print value + + +def disp_recs_count(flt): + print db.passive.get(flt).count() + + +def _disp_recs_tail(flt, field, n): + recs = list(db.passive.get( + flt, sort=[(field, -1)], limit=n)) + recs.reverse() + for r in recs: + if 'addr' in r: + print ivre.utils.int2ip(r['addr']), + else: + if 'fulltargetval' in r: + print r['fulltargetval'], + else: + print r['targetval'], + disp_rec(r) + + +def disp_recs_tail(n): + return lambda flt: _disp_recs_tail(flt, 'firstseen', n) + + +def disp_recs_tailnew(n): + return lambda flt: _disp_recs_tail(flt, 'lastseen', n) + + +def _disp_recs_tailf(flt, field): + # 1. init + firstrecs = list(db.passive.get( + flt, sort=[(field, -1)], limit=10)) + firstrecs.reverse() + # in case we don't have (yet) records matching our criteria + r = {'firstseen': 0, 'lastseen': 0} + for r in firstrecs: + if 'addr' in r: + print ivre.utils.int2ip(r['addr']), + else: + if 'fulltargetval' in r: + print r['fulltargetval'], + else: + print r['targetval'], + disp_rec(r) + # 2. loop + try: + while True: + prevtime = r[field] + time.sleep(1) + for r in db.passive.get( + db.passive.flt_and( + baseflt, {field: {'$gt': prevtime}}), + sort=[(field, 1)]): + if 'addr' in r: + print ivre.utils.int2ip(r['addr']), + else: + if 'fulltargetval' in r: + print r['fulltargetval'], + else: + print r['targetval'], + disp_rec(r) + except KeyboardInterrupt: + pass + + +def disp_recs_tailfnew(): + return lambda flt: _disp_recs_tailf(flt, 'firstseen') + + +def disp_recs_tailf(): + return lambda flt: _disp_recs_tailf(flt, 'lastseen') + + +def disp_recs_explain(flt): + print db.passive.explain(db.passive.get(flt), indent=4) + +if __name__ == '__main__': + import sys + import os + import datetime + baseflt = {} + disp_recs = disp_recs_std + try: + import argparse + parser = argparse.ArgumentParser( + description='Access and query the passive database.') + USING_ARGPARSE = True + except ImportError: + import optparse + parser = optparse.OptionParser( + description='Access and query the passive database.') + parser.parse_args_orig = parser.parse_args + + def my_parse_args(): + res = parser.parse_args_orig() + res[0].ensure_value('ips', res[1]) + return res[0] + parser.parse_args = my_parse_args + parser.add_argument = parser.add_option + USING_ARGPARSE = False + # DB + parser.add_argument('--init', '--purgedb', action='store_true', + help='Purge or create and initialize the database.') + parser.add_argument('--ensure-indexes', action='store_true', + help='Create missing indexes (will lock the database).') + # filters + parser.add_argument('--sensor') + parser.add_argument('--country') + parser.add_argument('--asnum') + parser.add_argument('--torcert', action='store_true') + parser.add_argument('--dns') + parser.add_argument('--dnssub') + parser.add_argument('--cert') + parser.add_argument('--basicauth', action='store_true') + parser.add_argument('--auth', action='store_true') + parser.add_argument('--java', action='store_true') + parser.add_argument('--ua') + parser.add_argument('--ftp', action='store_true') + parser.add_argument('--pop', action='store_true') + parser.add_argument('--timeago', type=int) + parser.add_argument('--timeagonew', type=int) + # display modes + parser.add_argument('--short', action='store_true', + help='Output only IP addresses, one per line.') + parser.add_argument('--tail', metavar='COUNT', type=int, + help='Output latest COUNT results.') + parser.add_argument('--tailnew', metavar='COUNT', type=int, + help='Output latest COUNT new results.') + parser.add_argument('--tailf', action='store_true', + help='Output continuously latest results.') + parser.add_argument('--tailfnew', action='store_true', + help='Output continuously latest results.') + parser.add_argument('--count', action='store_true', + help='Count matched results.') + parser.add_argument('--explain', action='store_true', + help='MongoDB specific: .explain() the query.') + parser.add_argument('--distinct', metavar='FIELD', + help='Output only unique FIELD part of the ' + 'results, one per line.') + parser.add_argument('--delete', action='store_true', + help='DELETE the matched results instead of ' + 'displaying them.') + if USING_ARGPARSE: + parser.add_argument('ips', nargs='*', + help='Display results for specified IP addresses' + ' or ranges.') + args = parser.parse_args() + if args.init: + if os.isatty(sys.stdin.fileno()): + sys.stdout.write( + 'This will remove any passive information in your ' + 'database. Process ? [y/N] ' + ) + ans = raw_input() + if ans.lower() != 'y': + exit(0) + db.passive.init() + exit(0) + if args.ensure_indexes: + if os.isatty(sys.stdin.fileno()): + sys.stdout.write( + 'This will lock your database. Process ? [y/N] ' + ) + ans = raw_input() + if ans.lower() != 'y': + exit(0) + db.passive.ensure_indexes() + exit(0) + if args.sensor is not None: + baseflt = db.passive.flt_and( + baseflt, + db.passive.searchsensor(args.sensor) + ) + if args.asnum is not None: + if args.asnum.startswith('!') or args.asnum.startswith('-'): + baseflt = db.passive.flt_and( + baseflt, + db.passive.searchasnum(int(args.asnum[1:]), neg=True) + ) + else: + baseflt = db.passive.flt_and( + baseflt, + db.passive.searchasnum(int(args.asnum)) + ) + if args.country is not None: + baseflt = db.passive.flt_and( + baseflt, + db.passive.searchcountry(args.country) + ) + if args.torcert: + baseflt = db.passive.flt_and(baseflt, db.passive.searchtorcert()) + if args.basicauth: + baseflt = db.passive.flt_and(baseflt, db.passive.searchbasicauth()) + if args.auth: + baseflt = db.passive.flt_and(baseflt, db.passive.searchhttpauth()) + if args.ua is not None: + baseflt = db.passive.flt_and( + baseflt, + db.passive.searchuseragent(ivre.utils.str2regexp(args.ua)) + ) + if args.java: + baseflt = db.passive.flt_and( + baseflt, + db.passive.searchjavaua() + ) + if args.ftp: + baseflt = db.passive.flt_and(baseflt, db.passive.searchftpauth()) + if args.pop: + baseflt = db.passive.flt_and(baseflt, db.passive.searchpopauth()) + if args.dns is not None: + baseflt = db.passive.flt_and( + baseflt, + db.passive.searchdns( + ivre.utils.str2regexp(args.dns), + subdomains=False)) + if args.dnssub is not None: + baseflt = db.passive.flt_and( + baseflt, + db.passive.searchdns( + ivre.utils.str2regexp(args.dnssub), + subdomains=True)) + if args.cert is not None: + baseflt = db.passive.flt_and( + baseflt, + db.passive.searchcertsubject( + ivre.utils.str2regexp(args.cert))) + if args.timeago is not None: + baseflt = db.passive.flt_and(db.passive.searchtimeago(args.timeago, + new=False)) + if args.timeagonew is not None: + baseflt = db.passive.flt_and(db.passive.searchtimeago(args.timeagonew, + new=True)) + if args.short: + disp_recs = disp_recs_short + elif args.distinct is not None: + disp_recs = functools.partial(disp_recs_distinct, args.distinct) + elif args.tail is not None: + disp_recs = disp_recs_tail(args.tail) + elif args.tailnew is not None: + disp_recs = disp_recs_tailnew(args.tailnew) + elif args.tailf: + disp_recs = disp_recs_tailf() + elif args.tailfnew: + disp_recs = disp_recs_tailfnew() + elif args.count: + disp_recs = disp_recs_count + elif args.delete: + disp_recs = db.passive.remove + elif args.explain: + disp_recs = disp_recs_explain + if not args.ips: + if not baseflt and disp_recs == disp_recs_std: + # default to tail -f mode + disp_recs = disp_recs_tailfnew() + disp_recs(baseflt) + exit(0) + first = True + for a in args.ips: + if first: + first = False + else: + print + flt = baseflt.copy() + if ':' in a: + a = a.split(':', 1) + if a[0].isdigit(): + a[0] = int(a[0]) + if a[1].isdigit(): + a[1] = int(a[1]) + flt = db.passive.flt_and(flt, db.passive.searchrange(a[0], a[1])) + elif '-' in a: + a = a.split('-', 1) + if a[0].isdigit(): + a[0] = int(a[0]) + if a[1].isdigit(): + a[1] = int(a[1]) + flt = db.passive.flt_and(flt, db.passive.searchrange(a[0], a[1])) + elif '/' in a: + flt = db.passive.flt_and(flt, db.passive.searchnet(a)) + else: + if a.isdigit(): + a = ivre.utils.int2ip(int(a)) + flt = db.passive.flt_and(flt, db.passive.searchhost(a)) + disp_recs(flt) diff --git a/bin/ipinfohost b/bin/ipinfohost new file mode 100755 index 0000000000..6bbff31e64 --- /dev/null +++ b/bin/ipinfohost @@ -0,0 +1,110 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +import ivre.utils +from ivre.db import db +import re +import time + +ipaddr = re.compile('^\d+\.\d+\.\d+\.\d+$') + + +def disp_rec(r): + if 'addr' in r: + if r['source'].startswith('PTR-'): + print '%s PTR %s (%s, %s time%s, %s - %s)' % ( + ivre.utils.int2ip(r['addr']), + r['value'], r['source'][4:], r['count'], + r['count'] > 1 and 's' or '', + datetime.datetime.fromtimestamp(int(r['firstseen'])), + datetime.datetime.fromtimestamp(int(r['lastseen']))) + elif r['source'].startswith('A-'): + print '%s A %s (%s, %s time%s, %s - %s)' % ( + r['value'], + ivre.utils.int2ip(r['addr']), + r['source'][2:], r['count'], + r['count'] > 1 and 's' or '', + datetime.datetime.fromtimestamp(int(r['firstseen'])), + datetime.datetime.fromtimestamp(int(r['lastseen']))) + else: + print 'WARNING', r + else: + if r['source'].split('-')[0] in ['CNAME', 'NS', 'MX']: + print '%s %s %s (%s, %s time%s, %s - %s)' % ( + r['value'], + r['source'].split('-')[0], + r['targetval'], + ':'.join(r['source'].split('-')[1:]), + r['count'], + r['count'] > 1 and 's' or '', + datetime.datetime.fromtimestamp(int(r['firstseen'])), + datetime.datetime.fromtimestamp(int(r['lastseen']))) + else: + print 'WARNING', r + +if __name__ == '__main__': + import sys + import datetime + baseflt = {'recontype': 'DNS_ANSWER'} + import getopt + subdomains = False + try: + opts, args = getopt.getopt(sys.argv[1:], + "s:", + [ + # filters + "sensor=", + # subdomains + "sub" + ]) + except getopt.GetoptError, err: + sys.stderr.write(str(err) + '\n') + sys.exit(-1) + for o, a in opts: + if o in ['-s', '--sensor']: + baseflt = db.passive.flt_and(baseflt, db.passive.searchsensor(a)) + elif o == '--sub': + subdomains = True + else: + sys.stderr.write( + '%r %r not undestood (this is probably a bug).\n' % (o, a)) + sys.exit(-1) + first = True + flts = [] + for a in args: + if first: + first = False + else: + print + if ipaddr.match(a) or a.isdigit(): + flts.append(db.passive.flt_and(baseflt, db.passive.searchhost(a))) + else: + flts += [ + db.passive.flt_and( + baseflt, + db.passive.searchdns( + ivre.utils.str2regexp(a), subdomains=subdomains)), + db.passive.flt_and( + baseflt, + db.passive.searchdns( + ivre.utils.str2regexp(a), + reverse=True, subdomains=subdomains)) + ] + for flt in flts: + for r in db.passive.get(flt, sort=[('source', 1)]): + disp_rec(r) diff --git a/bin/nmap2db b/bin/nmap2db new file mode 100755 index 0000000000..39580f6d59 --- /dev/null +++ b/bin/nmap2db @@ -0,0 +1,122 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +import ivre.db +import ivre.xmlnmap + +import xml.sax +import os + + +def storescan(fname, database, **kargs): + parser = xml.sax.make_parser() + try: + contenthandler = { + "test": ivre.xmlnmap.Nmap2Txt, + ivre.db.MongoDBNmap: ivre.xmlnmap.Nmap2Mongo, + }[database](fname, **kargs) + except Exception as exc: + print "WARNING: %s [%r] [fname=%s]" % (exc.message, exc, fname) + else: + parser.setContentHandler(contenthandler) + parser.setEntityResolver(ivre.xmlnmap.NoExtResolver()) + parser.parse(fname) + contenthandler.outputresults() + return True + return False + + +def recursive_filelisting(base_directories): + "Iterator on filenames in base_directories" + + for base_directory in base_directories: + for root, _, files in os.walk(base_directory): + for leaffile in files: + yield os.path.join(root, leaffile) + + +def main(): + try: + import argparse + parser = argparse.ArgumentParser( + description='Parse NMAP scan results and add them in DB.') + parser.add_argument('scan', nargs='*', metavar='SCAN', + help='Scan results') + + except ImportError: + import optparse + parser = optparse.OptionParser( + description='Parse NMAP scan results and add them in DB.') + parser.parse_args_orig = parser.parse_args + + def my_parse_args(): + res = parser.parse_args_orig() + res[0].ensure_value('scan', res[1]) + return res[0] + parser.parse_args = my_parse_args + parser.add_argument = parser.add_option + + parser.add_argument('-c', '--categories', default='', + help='Scan categories.') + parser.add_argument('-s', '--source', default=None, + help='Scan source.') + parser.add_argument('-t', '--test', action='store_true', + help='Test mode.') + parser.add_argument('--port', action='store_true', + help='Need ports.') + parser.add_argument('--never-archive', action='store_true', + help='Never archive.') + parser.add_argument('--archive', '--archive-same-host', action='store_true', + help='Archive results for the same host.') + parser.add_argument('--archive-same-host-and-source', action='store_true', + help='Archive results with both the same host and' + ' source (this is the default).') + parser.add_argument('-r', '--recursive', action='store_true', + help='Import all files from given directories.') + args = parser.parse_args() + categories = [] + source = None + database = ivre.db.db.nmap.__class__ + categories = args.categories.split(',') + if args.test: + database = "test" + if args.never_archive: + def gettoarchive(collection, addr, source): + return [] + elif args.archive: + def gettoarchive(collection, addr, source): + return collection.find({'addr': addr}) + else: #args.archive_same_host_and_source + def gettoarchive(collection, addr, source): + return collection.find({'addr': addr, + 'source': source}) + if args.recursive: + scans = recursive_filelisting(args.scan) + else: + scans = args.scan + count = 0 + for scan in scans: + if storescan(scan, database, + categories=categories, source=args.source, + needports=args.port, gettoarchive=gettoarchive): + count += 1 + print "%d results imported." % count + + +if __name__ == '__main__': + main() diff --git a/bin/p0f2db b/bin/p0f2db new file mode 100755 index 0000000000..31761a1d1d --- /dev/null +++ b/bin/p0f2db @@ -0,0 +1,108 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +import ivre.db +import ivre.utils + +import re +import subprocess +import datetime +import signal + + +def terminate(signum, stack_frame): + p0fprocess.stdout.close() + p0fprocess.terminate() + exit() + +signal.signal(signal.SIGINT, terminate) +signal.signal(signal.SIGTERM, terminate) + + +def process_file(sensor, fname, mode='SYN'): + global p0fprocess + distre = re.compile('distance ([0-9]+),') + if fname.startswith('iface:'): + fname = ['-i', fname[6:]] + else: + fname = ['-s', fname] + mode = { + 'SYN': { + 'options': [], + 'name': 'SYN', + 'filter': 'tcp and tcp[tcpflags] & (tcp-syn|tcp-ack) == 2' + }, + 'SYN+ACK': { + 'options': ['-A'], + 'name': 'SYN+ACK', + 'filter': 'tcp and tcp[tcpflags] & (tcp-syn|tcp-ack) == 18'}, + 'RST+': { + 'options': ['-R'], + 'name': 'RST+', + 'filter': 'tcp and tcp[tcpflags] & (tcp-rst) == 4'}, + 'ACK': { + 'options': ['-O'], + 'name': 'ACK', + 'filter': 'tcp and tcp[tcpflags] & (tcp-syn|tcp-ack) == 16'} + }[mode] + p0fprocess = subprocess.Popen( + ['p0f', '-l', '-S', '-ttt'] + fname + + mode['options'] + [mode['filter']], + stdout=subprocess.PIPE + ) + for l in p0fprocess.stdout: + # try: + l = [l.split(' - ')[0]] + l.split(' - ')[1].split(' -> ') + if l[1].startswith('UNKNOWN '): + sig = l[1][l[1].index('UNKNOWN ') + 8:][1:-1].split(':')[:6] + OS, version, dist = '?', '?', -1 + else: + sig = l[1][l[1].index(' Signature: ') + 12:][ + 1:-1].split(':')[:6] + if ' (up: ' in l[1]: + OS = l[1][:l[1].index(' (up: ')] + else: + OS = l[1][:l[1].index(' Signature: ')] + OS, version = OS.split(' ')[0], ' '.join(OS.split(' ')[1:]) + dist = int(distre.search(l[2]).groups()[0]) + # *** we wildcard any window size which is not Sxxx or Tyyy + if sig[0][0] not in ['S', 'T']: + sig[0] = '*' + spec = { + 'addr': ivre.utils.ip2int(l[0][l[0].index('> ') + + 2:l[0].index(':')]), + 'recontype': 'P0F2-%s' % mode['name'], + 'distance': dist, + 'value': OS, + 'version': version, + 'signature': ":".join(map(str, sig)), + 'sensor': sensor, + } + if mode['name'] == 'SYN+ACK': + spec.update({'port': int(l[0][l[0].index(':') + 1:])}) + ivre.db.db.passive.insert_or_update( + float(l[0][1:l[0].index('>')]), spec) + # except: + # sys.stderr.write('Warning for line [[[%s]]]\n' % str(l)) + +if __name__ == '__main__': + import sys + if len(sys.argv) == 4: + process_file(sys.argv[1], sys.argv[2], mode=sys.argv[3]) + else: + process_file(sys.argv[1], sys.argv[2]) diff --git a/bin/passiverecon2db b/bin/passiverecon2db new file mode 100755 index 0000000000..87a8545bff --- /dev/null +++ b/bin/passiverecon2db @@ -0,0 +1,143 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +import ivre.db +import ivre.utils +import re +import hashlib + +MAXVALLEN = ivre.utils.MAXVALLEN +SymantecUA = re.compile('[a-zA-Z0-9/+]{32,33}AAAAA$') +DigestAuthInfos = re.compile('(username|realm|algorithm|qop)=') + + +def preparerec(spec, ignorenets, neverignore): + # First of all, let's see if we are supposed to ignore this spec, + # and if so, do so. + if ('addr' in spec and + spec.get('source') not in neverignore.get(spec['recontype'], [])): + for n in ignorenets.get(spec['recontype'], ()): + if n[0] <= spec['addr'] <= n[1]: + return None + # Then, let's clean up the records. + # Change Symantec's random user agents (matching SymantecUA) to + # the constant string 'SymantecRandomUserAgent', to store + # different SymantecUA for the same host & sensor in the same + # records. + if spec['recontype'] == 'HTTP_CLIENT_HEADER' and \ + spec.get('source') == 'USER-AGENT': + if SymantecUA.match(spec['value']): + spec['value'] = 'SymantecRandomUserAgent' + # Change any Digest authorization header to remove non-constant + # information. On one hand we loose the necessary information to + # try to recover the passwords, but on the other hand we store + # specs with different challenges but the same username, realm, + # host and sensor in the same records. + if spec['recontype'] in ['HTTP_CLIENT_HEADER', + 'HTTP_CLIENT_HEADER_SERVER'] and \ + spec.get('source') in ['AUTHORIZATION', + 'PROXY-AUTHORIZATION']: + if spec['value'].startswith('Digest'): + try: + # we only keep relevant info + v = filter(DigestAuthInfos.match, + spec['value'][6:].lstrip().split(',')) + spec['value'] = 'Digest ' + ','.join(v) + except: + pass + # Finally we prepare the record to be stored. For that, we make + # sure that no indexed value has a size greater than MAXVALLEN. If + # so, we replace the value with its SHA1 hash and store the + # original value in full[original column name]. + if len(spec['value']) > MAXVALLEN: + spec['fullvalue'] = spec['value'] + spec['value'] = hashlib.sha1(spec['fullvalue']).hexdigest() + if 'targetval' in spec and len(spec['targetval']) > MAXVALLEN: + spec['fulltargetval'] = spec['targetval'] + spec['targetval'] = hashlib.sha1(spec['fulltargetval']).hexdigest() + return spec + + +def handle_rec(sensor, ignorenets, neverignore, + # these argmuments are provided by * + ts, host, port, recon_type, source, value, targetval): + ts = float(ts) + recon_type = recon_type[14:] # skip PassiveRecon:: + if host == '-': + spec = { + 'targetval': targetval, + 'recontype': recon_type, + 'value': value + } + else: + try: + host = ivre.utils.ip2int(host) + except: + pass + spec = { + 'addr': host, + 'recontype': recon_type, + 'value': value + } + if sensor is not None: + spec.update({'sensor': sensor}) + if port != '-': + spec.update({'port': int(port)}) + if source != '-': + spec.update({'source': source}) + spec = preparerec(spec, ignorenets, neverignore) + if spec is not None: + ivre.db.db.passive.insert_or_update( + ts, spec, getinfos=ivre.utils.passive_getinfos + ) + +def main(): + import sys + description = ('Update the database from output of the Bro script ' + '"passiverecon"') + try: + import argparse + parser = argparse.ArgumentParser(description=description) + except ImportError: + import optparse + parser = optparse.OptionParser(description=description) + parser.parse_args_orig = parser.parse_args + parser.parse_args = lambda: parser.parse_args_orig()[0] + parser.add_argument = parser.add_option + parser.add_argument('--sensor', '-s', help='Sensor name') + parser.add_argument('--ignore-spec', '-i', + help='Filename containing ignore rules') + args = parser.parse_args() + ignore_rules = {} + if args.ignore_spec is not None: + execfile(args.ignore_spec, ignore_rules) + for l in sys.stdin: + if not l or l.startswith('#'): + continue + if l.endswith('\n'): + l = l[:-1] + try: + handle_rec(args.sensor, + ignore_rules.get('IGNORENETS', {}), + ignore_rules.get('NEVERIGNORE', {}), + *l.split('\t')) + except: + sys.stderr.write("WARNING: cannot parse line [%s]\n" % l) + +if __name__ == '__main__': + main() diff --git a/bin/passivereconworker b/bin/passivereconworker new file mode 100755 index 0000000000..6bcfaf8e93 --- /dev/null +++ b/bin/passivereconworker @@ -0,0 +1,199 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +"""Handle passiverecon2db files.""" + +import re +import os +import sys +import shutil +import time +import subprocess +import gzip +import signal + + +SENSORS = {} # shortname: fullname +DIRECTORY = "/ivre/passiverecon/" +PROGNAME = "passiverecon2db" +FILEFORMAT = "^(?P%s)\\.(?P[0-9-]+)\\.log(?:\\.gz)?$" +SLEEPTIME = 2 +CMDLINE = "%(progname)s -s %(sensor)s" +WANTDOWN = False + + +def shutdown(signum, _): + """Sets the global variable `WANTDOWN` to `True` to stop + everything after the current files have been processed. + + """ + global WANTDOWN + print 'SHUTDOWN: got signal %d, will halt after current file.' % signum + WANTDOWN = True + + +def getnextfiles(directory=None, sensor=None, count=1): + """Returns a list of maximum `count` filenames to process, given + the `directory` (or, if it is `None`, the global variable + `DIRECTORY` is used ) and the `sensor` (or, if it is `None`, from + any sensor). + + """ + if directory is None: + directory = DIRECTORY + if sensor is None: + fmt = re.compile(FILEFORMAT % "[^\\.]*") + else: + fmt = re.compile(FILEFORMAT % re.escape(sensor)) + files = [fmt.match(f) for f in os.listdir(directory)] + files = [f for f in files if f is not None] + files.sort(key=lambda x: map(int, x.groupdict()['datetime'].split('-'))) + return [f for f in files[:count]] + + +def create_process(progname, sensor): + """Creates the insertion process for the given `sensor` using + `progname`. + + """ + return subprocess.Popen( + CMDLINE % { + "progname": progname, + "sensor": SENSORS.get(sensor, sensor) + }, + shell=True, stdin=subprocess.PIPE + ) + + +def worker(progname=None, directory=None, sensor=None, debug=False): + """This function is the main loop, creating the processes when + needed and feeding them with the data from the files. + + """ + if directory is None: + directory = DIRECTORY + if progname is None: + progname = PROGNAME + try: + os.makedirs(os.path.join(directory, "current")) + except OSError: + pass + procs = {} + while not WANTDOWN: + # We get the next file to handle + fname = getnextfiles(directory=directory, sensor=sensor, count=1) + # ... if we don't, we sleep for a while + if not fname: + if debug: + print "Sleeping for %d s" % SLEEPTIME, + sys.stdout.flush() + time.sleep(SLEEPTIME) + if debug: + print "DONE" + continue + fname = fname[0] + fname_sensor = fname.groupdict()['sensor'] + if fname_sensor in procs: + proc = procs[fname_sensor] + else: + proc = create_process(progname, fname_sensor) + procs[fname_sensor] = proc + fname = fname.group() + # Our "lock system": if we can move the file, it's ours + try: + shutil.move(os.path.join(directory, fname), + os.path.join(directory, "current")) + except shutil.Error: + continue + if debug: + print "Handling %s" % fname, + sys.stdout.flush() + fname = os.path.join(directory, "current", fname) + if fname.endswith('.gz'): + fdesc = gzip.open(fname) + else: + fdesc = open(fname) + handled_ok = True + for line in fdesc: + try: + proc.stdin.write(line) + except ValueError: + proc = create_process(progname, fname_sensor) + procs[fname_sensor] = proc + # Second (and last) try + try: + proc.stdin.write(line) + except ValueError: + handled_ok = False + fdesc.close() + if handled_ok: + os.unlink(fname) + if debug: + if handled_ok: + print "OK" + else: + print "KO!" + # SHUTDOWN + for sensor in procs: + procs[sensor].stdin.close() + procs[sensor].wait() + + +def main(): + """Parses the arguments and call worker()""" + global DIRECTORY, PROGNAME + try: + import argparse + parser = argparse.ArgumentParser(description=__doc__) + except ImportError: + # Python 2.6 compatibility + import optparse + parser = optparse.OptionParser(description=__doc__) + parser.parse_args_orig = parser.parse_args + parser.parse_args = lambda: parser.parse_args_orig()[0] + parser.add_argument = parser.add_option + parser.add_argument('--sensor', metavar='SENSOR[:SENSOR]', + help='sensor to check, optionally with a long name, ' + 'defaults to all.') + parser.add_argument('--directory', metavar='DIR', + help='base directory (defaults to %s).' % DIRECTORY, + default=DIRECTORY) + parser.add_argument('--progname', metavar='PROG', + help='Program to run (defaults to %s).' % PROGNAME, + default=PROGNAME) + args = parser.parse_args() + DIRECTORY = args.directory + PROGNAME = args.progname + if args.sensor is not None: + SENSORS.update(dict([args.sensor.split(':', 1) + if ':' in args.sensor + else [args.sensor, args.sensor]])) + sensor = args.sensor.split(':', 1)[0] + else: + sensor = None + #worker(sensor=sensor, debug=True) + worker(sensor=sensor) + + +if __name__ == '__main__': + # Set the signal handler + for s in [signal.SIGINT, signal.SIGTERM]: + signal.signal(s, shutdown) + signal.siginterrupt(s, False) + # Start + main() diff --git a/bin/plotdb b/bin/plotdb new file mode 100755 index 0000000000..caad700109 --- /dev/null +++ b/bin/plotdb @@ -0,0 +1,113 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +from ivre import db +import struct +import socket +import numpy +import math +import matplotlib +import matplotlib.pyplot +# from mpl_toolkits.mplot3d import Axes3D +from mpl_toolkits.mplot3d import axes3d, Axes3D + + +def graphhost(ap): + if 'ports' not in ap: + return [], [] + hh, pp = [], [] + an = ap['addr'] + ap = ap['ports'] + for p in ap: + pn = p['port'] + if p['state_state'] == 'open': + hh.append(an) + pp.append(pn) + return hh, pp + + +def getgraph(flt=db.db.nmap.flt_empty): + h, p = [], [] + allhosts = db.db.nmap.get(flt) + for ap in allhosts: + hh, pp = graphhost(ap) + h += hh + p += pp + return h, p + + +def graph3d(mainflt=db.db.nmap.flt_empty, alertflt=None): + h, p = getgraph(flt=mainflt) + fig = matplotlib.pyplot.figure() + if matplotlib.__version__.startswith('0.99'): + ax = Axes3D(fig) + else: + ax = fig.add_subplot(111, projection='3d') + ax.plot(map(lambda x: x / 65535, h), map(lambda x: x % + 65535, h), map(lambda x: math.log(x, 10), p), '.') + if alertflt is not None: + h, p = getgraph(flt=db.db.nmap.flt_and(mainflt, alertflt)) + if h: + ax.plot(map(lambda x: x / 65535, h), map(lambda x: x % + 65535, h), map(lambda x: math.log(x, 10), p), '.', c='r') + matplotlib.pyplot.show() + + +def graph2d(mainflt=db.db.nmap.flt_empty, alertflt=None): + h, p = getgraph(flt=mainflt) + fig = matplotlib.pyplot.figure() + ax = fig.add_subplot(111) + ax.semilogy(h, p, '.') + if alertflt is not None: + h, p = getgraph(flt=db.db.nmap.flt_and(mainflt, alertflt)) + if h: + ax.semilogy(h, p, '.', c='r') + matplotlib.pyplot.show() + +if __name__ == '__main__': + import sys + try: + import argparse + parser = argparse.ArgumentParser( + description='Plot scan results.', + parents=[db.db.nmap.argparser]) + except ImportError: + import optparse + parser = optparse.OptionParser(description='Plot scan results.') + for args, kargs in db.db.nmap.argparser.args: + parser.add_option(*args, **kargs) + parser.parse_args_orig = parser.parse_args + parser.parse_args = lambda: parser.parse_args_orig()[0] + parser.add_argument = parser.add_option + parser.add_argument('--2d', '-2', action='store_const', + dest='graph', + const=graph2d, + default=graph3d) + parser.add_argument('--3d', '-3', action='store_const', + dest='graph', + const=graph3d) + parser.add_argument('--alert-445', action='store_const', + dest='alertflt', + const=db.db.nmap.searchxp445(), + default=db.db.nmap.searchhttpauth()) + parser.add_argument('--alert-nfs', action='store_const', + dest='alertflt', + const=db.db.nmap.searchnfs()) + args = parser.parse_args() + args.graph(mainflt=db.db.nmap.parse_args(args), + alertflt=args.alertflt) diff --git a/bin/runscans b/bin/runscans new file mode 100755 index 0000000000..8092f0f891 --- /dev/null +++ b/bin/runscans @@ -0,0 +1,430 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +""" +This module is part of IVRE. +Copyright 2011 - 2014 Pierre LALET + +This program runs scans and produces output files importable with +nmap2db. +""" + +import subprocess +import resource +import multiprocessing +import shlex +import shutil +import select +import re +import os +import sys +import fcntl +import time +import termios + +import ivre.geoiputils +import ivre.utils +import ivre.target +import ivre.nmapopt + +if sys.version_info >= (2, 7): + import functools + USE_PARTIAL = True +else: + # Python version <= 2.6: + # see http://bugs.python.org/issue5228 + # multiprocessing not compatible with functools.partial + USE_PARTIAL = False + +STATUS_NEW = 0 +STATUS_DONE_UP = 1 +STATUS_DONE_DOWN = 2 +STATUS_DONE_UNKNOWN = 3 + +NMAP_LIMITS = {} + + +def setnmaplimits(): + """Enforces limits from NMAP_LIMITS global variable.""" + for limit, value in NMAP_LIMITS.iteritems(): + resource.setrlimit(limit, value) + + +class XmlProcess: + addrrec = re.compile('') + + def target_status(self, target): + return STATUS_NEW + + +class XmlProcessTest(XmlProcess): + + def process(self, fdesc): + data = fdesc.read() + if data == '': + return False + for addr in self.addrrec.finditer(data): + print "Read adddress", addr.groups()[0] + return True + + +class XmlProcessWritefile(XmlProcess): + statusline = re.compile('\n') + status_up = ']') + status_paths = { + 'up': STATUS_DONE_UP, + 'down': STATUS_DONE_DOWN, + 'unknown': STATUS_DONE_UNKNOWN, + } + + def __init__(self, path, fulloutput=False): + self.path = path + self.starttime = int(time.time() * 1000000) + self.data = '' + self.isstarting = True + self.startinfo = '' + ivre.utils.makedirs(self.path) + self.scaninfo = open('%sscaninfo.%d' % (self.path, + self.starttime), + 'w') + if fulloutput: + self.has_fulloutput = True + self.fulloutput = open('%sfulloutput.%d' % (self.path, + self.starttime), + 'w') + else: + self.has_fulloutput = False + + def process(self, fdesc): + newdata = fdesc.read() + # print "READ", len(newdata), "bytes" + if newdata == '': + self.scaninfo.write(self.data) + self.scaninfo.close() + if self.has_fulloutput: + self.fulloutput.close() + return False + if self.has_fulloutput: + self.fulloutput.write(newdata) + self.fulloutput.flush() + self.data += newdata + while '' in self.data: + hostbeginindex = self.data.index( + self.hostbegin.search(self.data).group()) + self.scaninfo.write(self.data[:hostbeginindex]) + self.scaninfo.flush() + if self.isstarting: + self.startinfo += self.statusline.sub( + '', self.data[:hostbeginindex]) + self.isstarting = False + self.data = self.data[hostbeginindex:] + hostrec = self.data[:self.data.index('') + 7] + try: + addr = self.addrrec.search(hostrec).groups()[0] + except Exception as exc: + print exc + print hostrec + if self.status_up in hostrec: + status = 'up' + elif self.status_down in hostrec: + status = 'down' + else: + status = 'unknown' + outfile = self.path + status + \ + '/' + addr.replace('.', '/') + '.xml' + ivre.utils.makedirs(os.path.dirname(outfile)) + with open(outfile, 'w') as fdesc: + # fdesc.write('\n' % starttime) + fdesc.write(self.startinfo) + fdesc.write(hostrec) + fdesc.write('\n\n') + self.data = self.data[self.data.index('') + 7:] + if self.data.startswith('\n'): + self.data = self.data[1:] + return True + + def target_status(self, target): + for status, statuscode in self.status_paths.iteritems(): + try: + os.stat(os.path.join(self.path, status, + target.replace('.', '/') + '.xml')) + return statuscode + except OSError: + pass + return STATUS_NEW + + +def restore_echo(): + """Hack for https://stackoverflow.com/questions/6488275 equivalent + issue with Nmap (from + http://stackoverflow.com/a/8758047/3223422) + + """ + fdesc = sys.stdin.fileno() + attrs = termios.tcgetattr(fdesc) + attrs[3] = attrs[3] | termios.ECHO + termios.tcsetattr(fdesc, termios.TCSADRAIN, attrs) + + +def call_nmap(options, xmlprocess, targets, + accept_target_status=None): + if accept_target_status is None: + accept_target_status = [STATUS_NEW] + options += ['-oX', '-', '-iL', '-'] + proc = subprocess.Popen(options, preexec_fn=setnmaplimits, + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + procout = proc.stdout.fileno() + procoutfl = fcntl.fcntl(procout, fcntl.F_GETFL) + fcntl.fcntl(procout, fcntl.F_SETFL, procoutfl | os.O_NONBLOCK) + toread = [proc.stdout] + towrite = [proc.stdin] + targiter = targets.__iter__() + while toread: + # print "ENTERING SELECT" + rlist, wlist = select.select(toread, towrite, [])[:2] + # print "LEAVING SELECT", rlist, wlist + for rfdesc in rlist: + # print "PROCESSING DATA" + if not xmlprocess.process(rfdesc): + print "NO MORE DATA TO PROCSESS" + rfdesc.close() + toread.remove(rfdesc) + for wfdesc in wlist: + try: + naddr = ivre.utils.int2ip(targiter.next()) + while xmlprocess.target_status( + naddr) not in accept_target_status: + naddr = ivre.utils.int2ip(targiter.next()) + print "ADDING TARGET", + print targiter.nextcount, + if hasattr(targets, "targetcount"): + print '/', targets.targetscount, + print ":", naddr + wfdesc.write(naddr + '\n') + wfdesc.flush() + except StopIteration: + print "WROTE ALL TARGETS" + wfdesc.close() + towrite.remove(wfdesc) + except IOError: + print "ERROR: NMAP PROCESS IS DEAD" + return -1 + proc.wait() + return 0 + + +def _call_nmap_single(maincategory, options, + accept_target_status, target): + target = ivre.utils.int2ip(target) + outfile = 'scans/%s/%%s/%s.xml' % (maincategory, target.replace('.', '/')) + if STATUS_DONE_UP not in accept_target_status: + try: + os.stat(outfile % 'up') + return + except OSError: + pass + if STATUS_DONE_DOWN not in accept_target_status: + try: + os.stat(outfile % 'down') + return + except OSError: + pass + if STATUS_DONE_UNKNOWN not in accept_target_status: + try: + os.stat(outfile % 'unknown') + return + except OSError: + pass + ivre.utils.makedirs(os.path.dirname(outfile % 'current')) + subprocess.call(options + ['-oX', outfile % 'current', target], + preexec_fn=setnmaplimits) + resdata = open(outfile % 'current').read() + if ' +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +import ivre.target +import ivre.utils +import ivre.scanengine +import os +import sys +import time +import subprocess + +MAINDIR = "./agentsdata" + +ACTION_SYNC = 1 +ACTION_FEED = 2 +ACTION_BOTH = 3 + +if __name__ == '__main__': + import sys + try: + import argparse + parser = argparse.ArgumentParser( + description='Sends targets to a remote agent.', + parents=[ivre.target.argparser]) + USING_ARGPARSE = True + except ImportError: + import optparse + parser = optparse.OptionParser( + description='Sends targets to a remote agent.', + usage='Usage: runscans-agent [options] AGENT [AGENT ...]') + for args, kargs in ivre.target.argparser.args: + parser.add_option(*args, **kargs) + parser.parse_args_orig = parser.parse_args + + def parse_args(): + args, agents = parser.parse_args_orig() + args._update_loose({'agents': agents}) + return args + parser.parse_args = parse_args + parser.add_argument = parser.add_option + USING_ARGPARSE = False + parser.add_argument('--category', metavar='CAT', default="MISC", + help='tag scan results with this category') + parser.add_argument('--max-waiting', metavar='TIME', type=int, default=60, + help='maximum targets waiting') + parser.add_argument('--sync', dest='action', action='store_const', + const=ACTION_SYNC, default=ACTION_BOTH) + parser.add_argument('--dont-store-down', dest='storedown', + action='store_const', + const=False, default=True) + parser.add_argument('--feed', dest='action', action='store_const', + const=ACTION_FEED) + if USING_ARGPARSE: + parser.add_argument('agents', metavar='AGENT', nargs='+', + help='agents to use (rsync address)') + args = parser.parse_args() + if args.category is None: + args.categories = [] + else: + args.categories = [args.category] + agents = [ + ivre.scanengine.Agent.from_string(a, localbase=MAINDIR, + maxwaiting=args.max_waiting) + for a in args.agents + ] + for a in agents: + a.create_local_dirs() + if args.action == ACTION_SYNC: + camp = ivre.scanengine.Campaign([], args.category, agents, + os.path.join(MAINDIR, 'output'), + visiblecategory='MISC', + storedown=args.storedown) + ivre.scanengine.syncloop(agents) + elif args.action in [ACTION_FEED, ACTION_BOTH]: + if args.action == ACTION_BOTH: + # we make sure we're in screen + if os.environ['TERM'] != 'screen': + subprocess.call(['screen'] + sys.argv) + sys.exit(0) + # we run the sync process in another screen window + subprocess.call(['screen'] + sys.argv + ['--sync']) + targets = ivre.target.target_from_args(args) + if targets is None: + parser.error( + 'one argument of --country/--asnum/--range/--network/' + '--routable/--file/--test is required' + ) + camp = ivre.scanengine.Campaign(targets, args.category, agents, + os.path.join(MAINDIR, 'output'), + visiblecategory='MISC') + try: + camp.feedloop() + except KeyboardInterrupt: + sys.stderr.write('Stop feeding.\n') + sys.stderr.write('Use "--state %s" to resume.\n' % + ' '.join(map(str, camp.targiter.getstate()))) + except Exception as e: + sys.stderr.write('ERROR: %r.\n' % e) + sys.stderr.write('Use "--state %s" to resume.\n' % + ' '.join(map(str, camp.targiter.getstate()))) + else: + sys.stderr.write('No more targets to feed.\n') + if os.environ['TERM'] != 'screen': + sys.stderr.write('Press enter to exit.\n') + raw_input() + raw_input() diff --git a/bin/scancli b/bin/scancli new file mode 100755 index 0000000000..e8dd525f98 --- /dev/null +++ b/bin/scancli @@ -0,0 +1,568 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +from ivre import utils, db, graphroute, config + +import sys +import os +from datetime import datetime + + +def displayhost(dic, showscripts=True, showtraceroute=True, showos=True, + out=sys.stdout): + try: + h = "Host %s" % utils.int2ip(dic['addr']) + except: + h = "Host %s" % dic['addr'] + if 'hostnames' in dic and dic['hostnames']: + h += " (%s)" % '/'.join(x['name'] for x in dic['hostnames']) + if 'source' in dic: + h += ' from %s' % dic['source'] + if 'categories' in dic: + h += ' (%s)' % ', '.join(dic['categories']) + if 'state' in dic: + h += ' (%s' % dic['state'] + if 'state_reason' in dic: + h += ': %s' % dic['state_reason'] + h += ')\n' + out.write(h) + if 'infos' in dic: + infos = dic['infos'] + if 'country_code' in infos or 'country_name' in infos: + out.write("\t%s - %s" % (infos.get('country_code', '?'), + infos.get('country_name', '?'))) + if 'city' in infos: + out.write(' - %s' % infos['city']) + out.write('\n') + if 'as_num' in infos or 'as_name' in infos: + out.write("\tAS%s - %s\n" % (infos.get('as_num', '?'), + infos.get('as_name', '?'))) + if 'starttime' in dic and 'endtime' in dic: + out.write("\tscan %s - %s\n" % + (dic['starttime'], dic['endtime'])) + if 'extraports' in dic: + d = dic['extraports'] + for k in d: + out.write("\t%d ports %s (%s)\n" % + (d[k][0], k, ', '.join(['%d %s' % (d[k][1][kk], kk) + for kk in d[k][1].keys()]))) + if 'ports' in dic: + d = dic['ports'] + d.sort(key=lambda x: (x['protocol'], x['port'])) + for k in d: + reason = "" + if 'state_reason' in k: + reason = " (%s" % k['state_reason'] + for kk in filter(lambda x: x.startswith('state_reason_'), + k.keys()): + reason += ", %s=%s" % (kk[13:], k[kk]) + reason += ')' + srv = "" + if 'service_name' in k: + srv = "" + k['service_name'] + if 'service_method' in k: + srv += ' (%s)' % k['service_method'] + for kk in ['service_product', 'service_version', + 'service_extrainfo', 'service_ostype', + 'service_hostname']: + if kk in k: + srv += ' %s' % k[kk] + out.write("\t%-10s%-8s%-22s%s\n" % + ('%s/%d' % (k['protocol'], k['port']), + k['state_state'], reason, srv)) + if showscripts and 'scripts' in k: + for s in k['scripts']: + if 'output' not in s: + out.write('\t\t' + s['id'] + ':\n') + else: + o = filter( + lambda x: x, map(lambda x: x.strip(), + s['output'].split('\n'))) + if len(o) == 0: + out.write('\t\t' + s['id'] + ':\n') + elif len(o) == 1: + out.write('\t\t' + s['id'] + ': ' + o[0] + '\n') + elif len(o) > 1: + out.write('\t\t' + s['id'] + ': \n') + for oo in o: + out.write('\t\t\t' + oo + '\n') + if showscripts and 'scripts' in dic: + out.write('\tHost scripts:\n') + for s in dic['scripts']: + if 'output' not in s: + out.write('\t\t' + s['id'] + ':\n') + o = [x.strip() for x in s['output'].split('\n') if x] + if len(o) == 0: + out.write('\t\t' + s['id'] + ':\n') + elif len(o) == 1: + out.write('\t\t' + s['id'] + ': ' + o[0] + '\n') + elif len(o) > 1: + out.write('\t\t' + s['id'] + ': \n') + for oo in o: + out.write('\t\t\t' + oo + '\n') + if showtraceroute and 'traces' in dic and dic['traces']: + for k in dic['traces']: + proto = k['protocol'] + if proto in ['tcp', 'udp']: + proto += '/%d' % k['port'] + out.write('\tTraceroute (using %s)\n' % proto) + hops = k['hops'] + hops.sort(key=lambda x: x['ttl']) + for i in hops: + try: + out.write('\t\t%3s %15s %7s\n' % + (i['ttl'], utils.int2ip(i['ipaddr']), + i['rtt'])) + except: + out.write('\t\t%3s %15s %7s\n' % + (i['ttl'], i['ipaddr'], i['rtt'])) + if showos and 'os' in dic and 'osclass' in dic['os'] and \ + dic['os']['osclass']: + o = dic['os']['osclass'] + maxacc = str(max([int(x['accuracy']) for x in o])) + o = filter(lambda x: x['accuracy'] == maxacc, o) + out.write('\tOS fingerprint\n') + for oo in o: + out.write( + '\t\t%(osfamily)s / %(type)s / %(vendor)s / ' + 'accuracy = %(accuracy)s\n' % oo) + + +HONEYD_ACTION_FROM_NMAP_STATE = { + 'resets': 'reset', + 'no-responses': 'block', +} +HONEYD_DEFAULT_ACTION = 'block' +HONEYD_STD_SCRIPTS_BASE_PATH = '/usr/share/honeyd' +HONEYD_SSL_CMD = 'honeydssl --cert-subject %(subject)s -- %(command)s' + + +def display_honeyd_preamble(out=sys.stdout): + out.write("""create default +set default default tcp action block +set default default udp action block +set default default icmp action block + +""") + + +def getscript(port, sname): + for s in port.get('scripts', []): + if s['id'] == sname: + return s + return None + + +def nmap_port2honeyd_action(port): + if port['state_state'] == 'closed': + return 'reset' + elif port['state_state'] != 'open': + return 'block' + if 'service_tunnel' in port and port['service_tunnel'] == 'ssl': + sslrelay = True + else: + sslrelay = False + if 'service_name' in port: + if port['service_name'] == 'tcpwrapped': + return '"true"' + elif port['service_name'] == 'ssh': + s = getscript(port, 'banner') + if s is not None: + banner = s['output'] + else: + banner = 'SSH-%s-%s' % ( + port.get('service_version', '2.0'), + '_'.join([k for k in + port.get('service_product', 'OpenSSH').split() + if k != 'SSH']), + ) + return '''"%s %s"''' % ( + os.path.join(config.HONEYD_IVRE_SCRIPTS_PATH, 'sshd'), + banner + ) + return 'open' + + +def display_honeyd_conf(host, honeyd_routes, honeyd_entries, out=sys.stdout): + addr = utils.int2ip(host['addr']) + hname = "host_%s" % addr.replace('.', '_') + out.write("create %s\n" % hname) + defaction = HONEYD_DEFAULT_ACTION + if 'extraports' in host: + extra = host['extraports'] + defstate = extra[max(extra, key=lambda x: extra[x][0])][1] + defaction = HONEYD_ACTION_FROM_NMAP_STATE.get( + max(defstate, key=lambda x: defstate[x]), + HONEYD_DEFAULT_ACTION + ) + out.write('set %s default tcp action %s\n' % (hname, defaction)) + for p in host.get('ports', []): + out.write('add %s %s port %d %s\n' % ( + hname, p['protocol'], p['port'], + nmap_port2honeyd_action(p)) + ) + if 'traces' in host and len(host['traces']) > 0: + trace = max(host['traces'], key=lambda x: len(x['hops']))['hops'] + if trace: + trace.sort(key=lambda x: x['ttl']) + curhop = trace[0] + honeyd_entries.add(curhop['ipaddr']) + for t in trace[1:]: + key = (curhop['ipaddr'], t['ipaddr']) + latency = max(t['rtt'] - curhop['rtt'], 0) + route = honeyd_routes.get(key) + if route is None: + honeyd_routes[key] = { + 'count': 1, + 'high': latency, + 'low': latency, + 'mean': latency, + 'targets': set([host['addr']]) + } + else: + route['targets'].add(host['addr']) + honeyd_routes[key] = { + 'count': route['count'] + 1, + 'high': max(route['high'], latency), + 'low': min(route['low'], latency), + 'mean': (route['mean'] * route['count'] + + latency) / (route['count'] + 1), + 'targets': route['targets'], + } + curhop = t + out.write('bind %s %s\n\n' % (addr, hname)) + return honeyd_routes, honeyd_entries + + +def display_honeyd_epilogue(honeyd_routes, honeyd_entries, out=sys.stdout): + for r in honeyd_entries: + out.write('route entry %s\n' % utils.int2ip(r)) + out.write('route %s link %s/32\n' % (utils.int2ip(r), + utils.int2ip(r))) + out.write('\n') + for r in honeyd_routes: + out.write('route %s link %s/32\n' % (utils.int2ip(r[0]), + utils.int2ip(r[1]))) + for t in honeyd_routes[r]['targets']: + out.write('route %s add net %s/32 %s latency %dms\n' % ( + utils.int2ip(r[0]), utils.int2ip(t), + utils.int2ip(r[1]), + int(round(honeyd_routes[r]['mean'])), + )) + + +def display_xml_preamble(out=sys.stdout): + out.write('\n' + '\n') + + +def display_xml_host(h, out=sys.stdout): + scan = db.db.nmap.getscan(h['scanid']) + if 'scaninfos' in scan and scan['scaninfos']: + for k in scan['scaninfos'][0]: + scan['scaninfo.%s' % k] = scan['scaninfos'][0][k] + del scan['scaninfos'] + for k in ['version', 'start', 'startstr', 'args', 'scanner', + 'xmloutputversion', 'scaninfo.type', 'scaninfo.protocol', + 'scaninfo.numservices', 'scaninfo.services']: + if k not in scan: + scan[k] = '' + out.write('\n' + '\n' + '\n' + '\n' + '\n' % scan) + out.write('') + if 'state' in h: + out.write('') + out.write('\n') + if 'addr' in h: + try: + out.write('
\n' % + utils.int2ip(h['addr'])) + except: + out.write('
\n' % h['addr']) + for t in h.get('addresses', []): + for a in h['addresses'][t]: + out.write('
\n' % (a, t)) + if 'hostnames' in h: + out.write('\n') + for hostname in h['hostnames']: + out.write('\n') + out.write('\n') + out.write('') + for k in h.get('extraports', []): + out.write('\n' % ( + k, h['extraports'][k][0])) + for kk in h['extraports'][k][1]: + out.write('\n' % ( + kk, h['extraports'][k][1][kk])) + out.write('\n') + for p in h.get('ports', []): + out.write('') + if 'service_name' in p: + out.write('') + # XXX TODO table/elem + out.write('') + else: + out.write('/>') + out.write('\n') + out.write('\n') + + +def main(): + try: + import argparse + parser = argparse.ArgumentParser( + description='Access and query the active scans database.', + parents=[db.db.nmap.argparser]) + USING_ARGPARSE = True + except ImportError: + import optparse + parser = optparse.OptionParser( + description='Access and query the active scans database.') + for args, kargs in db.db.nmap.argparser.args: + parser.add_option(*args, **kargs) + parser.parse_args_orig = parser.parse_args + parser.parse_args = lambda: parser.parse_args_orig()[0] + parser.add_argument = parser.add_option + USING_ARGPARSE = False + parser.add_argument('--init', '--purgedb', action='store_true', + help='Purge or create and initialize the database.') + parser.add_argument('--ensure-indexes', action='store_true', + help='Create missing indexes (will lock the database).') + parser.add_argument('--short', action='store_true', + help='Output only IP addresses, one per line.') + parser.add_argument('--json', action='store_true', + help='Output results as JSON documents.') + parser.add_argument('--honeyd', action='store_true', + help='Output results as a honeyd config file.') + parser.add_argument('--nmap-xml', action='store_true', + help='Output results as a nmap XML output file.') + parser.add_argument('--graphroute-dot', action='store_true', + help='Output a Graphviz "dot" file with traceroute ' + 'results.') + if graphroute.HAVE_DBUS: + parser.add_argument('--graphroute-rtgraph3d', action='store_true', + help='Send traceroute results to rtgraph3d.') + parser.add_argument('--graphroute-dont-reset', action='store_true', + help='Do NOT reset graph (only for ' + '--graphroute-rtgraph3d)') + parser.add_argument('--graphroute-include-last-hop', action='store_true', + help='Include the last hop to graphroute output.') + parser.add_argument('--graphroute-include-target', action='store_true', + help='Include the target to graphroute output ' + '(implies --include-last-hop).') + parser.add_argument('--count', action='store_true', + help='Count matched results.') + parser.add_argument('--explain', action='store_true', + help='MongoDB specific: .explain() the query.') + parser.add_argument('--distinct', metavar='FIELD', + help='Output only unique FIELD part of the ' + 'results, one per line.') + parser.add_argument('--delete', action='store_true', + help='DELETE the matched results instead of ' + 'displaying them.') + if USING_ARGPARSE: + parser.add_argument('--sort', metavar='FIELD / ~FIELD', nargs='+', + help='Sort results according to FIELD; use ~FIELD ' + 'to reverse sort order.') + else: + parser.add_argument('--sort', metavar='FIELD / ~FIELD', + help='Sort results according to FIELD; use ~FIELD ' + 'to reverse sort order.') + parser.add_argument('--limit', type=int, + help='Ouput at most LIMIT results.') + parser.add_argument('--skip', type=int, + help='Skip first SKIP results.') + args = parser.parse_args() + + out = sys.stdout + + def displayfunction(x): + for h in x: + displayhost(h, out=out) + if os.isatty(out.fileno()): + raw_input() + else: + out.write('\n') + + hostfilter = db.db.nmap.parse_args(args) + sortkeys = [] + limit = None + skip = None + if args.init: + if os.isatty(sys.stdin.fileno()): + sys.stdout.write( + 'This will remove any scan result in your database. ' + 'Process ? [y/N] ') + ans = raw_input() + if ans.lower() != 'y': + sys.exit(-1) + db.db.nmap.init() + sys.exit(0) + if args.ensure_indexes: + if os.isatty(sys.stdin.fileno()): + sys.stdout.write( + 'This will lock your database. ' + 'Process ? [y/N] ') + ans = raw_input() + if ans.lower() != 'y': + sys.exit(-1) + db.db.nmap.ensure_indexes() + sys.exit(0) + if args.json: + import json + + def displayfunction(x): + if os.isatty(sys.stdout.fileno()): + indent = 4 + else: + indent = None + for h in x: + print json.dumps(h, indent=indent, + default=db.db.nmap.serialize) + elif args.short: + def displayfunction(x): + for h in x.distinct('addr'): + try: + out.write(utils.int2ip(h) + '\n') + except: + out.write(str(h) + '\n') + elif args.honeyd: + def displayfunction(x): + display_honeyd_preamble(out) + honeyd_routes = {} + honeyd_entries = set() + for h in x: + honeyd_routes, honeyd_entries = display_honeyd_conf( + h, + honeyd_routes, + honeyd_entries, + out + ) + display_honeyd_epilogue(honeyd_routes, honeyd_entries, out) + elif args.nmap_xml: + def displayfunction(x): + display_xml_preamble(out) + for h in x: + display_xml_host(h, out) + elif args.graphroute_dot or (graphroute.HAVE_DBUS and + args.graphroute_rtgraph3d): + def displayfunction(cursor): + graph, entry_nodes = graphroute.buildgraph( + cursor, + include_last_hop=args.graphroute_include_last_hop, + include_target=args.graphroute_include_target, + ) + if args.graphroute_dot: + from sys import stdout + graphroute.writedotgraph(graph, stdout) + elif graphroute.HAVE_DBUS and args.graphroute_rtgraph3d: + g = graphroute.display3dgraph( + graph, + reset_world=not args.graphroute_dont_reset + ) + for n in entry_nodes: + g.glow(utils.int2ip(n)) + elif args.count: + def displayfunction(x): + out.write(str(x.count()) + '\n') + elif args.distinct is not None: + distinctfield = args.distinct + + def displayfunction(x): + out.write(', '.join(map(str, x.distinct(distinctfield))) + '\n') + elif args.explain: + def displayfunction(x): + out.write(db.db.nmap.explain(x, indent=4) + '\n') + elif args.delete: + def displayfunction(x): + for h in x: + db.db.nmap.remove(h) + if args.sort is not None: + sortkeys = [(field.startswith('~') and field[1:] or field, + field.startswith('~') and -1 or 1) + for field in args.sort] + if args.limit is not None: + limit = args.limit + if args.skip is not None: + skip = args.skip + + cursor = db.db.nmap.get(hostfilter) + if sortkeys: + cursor = cursor.sort(sortkeys) + if limit is not None: + cursor = cursor.limit(limit) + if skip is not None: + cursor = cursor.skip(skip) + displayfunction(cursor) + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/bin/scanstatus b/bin/scanstatus new file mode 100755 index 0000000000..adc4ae768e --- /dev/null +++ b/bin/scanstatus @@ -0,0 +1,84 @@ +#! /usr/bin/env python + +# This file is part of IVRE. +# Copyright 2011 - 2014 Pierre LALET +# +# IVRE is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# IVRE is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +# License for more details. +# +# You should have received a copy of the GNU General Public License +# along with IVRE. If not, see . + +import sys +import re +import datetime + +statusline = re.compile( + 'begin|end|progress) task="(?P[^"]*)" ' + 'time="(?P