exploitutils.py

##ImmunityHeader v1
###############################################################################
## File       :  exploitutils.py
## Description:
##            :
## Created_On :  Fri Jul 17 10:48:26 2009
## Created_By :  Rich
## Modified_On:  Fri Jul 17 11:03:21 2009
## Modified_By:  Rich
##
## (c) Copyright 2009, Immunity Inc all rights reserved.
###############################################################################
#! /usr/bin/env python

#Proprietary CANVAS source code - use only under the license agreement
#specified in LICENSE.txt in your CANVAS distribution
#Copyright Immunity, Inc, 2002-2006
#http://www.immunityinc.com/CANVAS/ for more information

#from __future__ must be at beginning of file...
#from __future__ import generators

from __future__ import with_statement

# 64 bit compatibility, wrap struct.pack and struct unpack
import os, sys
import commands
from canvaserror import *

import logging


class structWrap:
    def __init__(self, f = None):
        self.f = f
        return

    def amdsixquatro(self, a, k):
        #print "ALTERING: Args, Kwargs: ", a, k
        aList = list(a) # unfreeze the tuple to list
        sList = []
        for c in a[0]:
            if c == 'L': c = 'I'
            if c == 'l': c = 'i'
            sList.append(c)
        aList[0] = "".join(sList)
        a = tuple(aList) # freeze the list to tuple
        #print "ALTERED?: Args, Kwargs: ", a, k
        return a, k

    def __call__(self, *args, **kwargs):
        args, kwargs = self.amdsixquatro(args, kwargs)
        ret = self.f(*args, **kwargs)
        return ret

import struct
from struct import calcsize

#we can use this to know if we are a 64 bit Python or not
original_long_size = struct.calcsize("<L")
threadchecknonMain_we_are_commandline = False
# end of 64 bit compatibility code

import sys, string
#thanks Tim O'Malley! (BSDish license)
import timeoutsocket
timeoutsocket.setDefaultSocketTimeout(5)
import socket
from MOSDEF.mosdefutils import *
from internal import * # a lot of files only include exploitutils and use internal functions via this include.
import threading
import random
import locale
from libs.tlslite.api import *
import time

# platform_is_win32
import os
global platform_is_win32
platform_is_win32 = False
if sys.platform in [ "win32" , "cygwin" ] or os.name == "nt":
    platform_is_win32 = True
#del os #why would we delete os?
import socket

def utf16toascii(buf):
    #replace all non-ascii characters with a ?
    if buf == "":
        return ""
    try:
        ret = buf.decode("utf-16-le").encode("ascii","replace")
    except UnicodeDecodeError:
        ret = prettyprint(buf)

    if ret[-1] == "\x00":
        ret = ret[:-1]
    return ret

def getsocklistener(listenport, listenhost="0.0.0.0", ipv6=False, protocol=None):
    if protocol in [None, "TCP"]:
        protocol = socket.SOCK_STREAM
    elif protocol in ["UDP"]:
        protocol = socket.SOCK_DGRAM

    if ipv6:
        s = socket.socket(socket.AF_INET6, protocol)
    else:
        s = socket.socket(socket.AF_INET, protocol)
    s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)

    try:
        s.bind((listenhost,listenport))
    except socket.error:
        s.close()
        #import traceback
        #traceback.print_exc()
        return None

    if protocol == socket.SOCK_STREAM:
        s.listen(5)
    return s

def getudplistener(listenport, listenhost="0.0.0.0", ipv6=False):
    return getsocklistener(listenport, listenhost, ipv6, "UDP")

def gettcplistener(listenport, listenhost="0.0.0.0", ipv6=False):
    """
    Returns None on failure

    """
    return getsocklistener(listenport, listenhost, ipv6, "TCP")

import cStringIO
import gzip
def gzipBuffer(buffer):
    """
    Gzip encodes a buffer, returning a string of compressed data. This will add the header and footer information
    needed by the HTTP spec. Normal zlib.compress() will not, so you can't use it for Content-Encoded: gzip format.
    """
    zbuf = cStringIO.StringIO()
    zfile = gzip.GzipFile(mode = 'wb',  fileobj = zbuf, compresslevel = 9)
    zfile.write(buffer)
    zfile.close()
    return zbuf.getvalue()

#simpler routines for simpler functions
import StringIO
def gunzipstring(data):
    """
    Gunzips a string, or throws an exception
    """
    datastream = StringIO.StringIO(data)
    g = gzip.GzipFile(fileobj=datastream)
    data = g.read()
    return data

def gzipstring(data):
    """
    Gzips a string to another string using StringIO
    """
    datastream = StringIO.StringIO()
    g = gzip.GzipFile(mode="wb",fileobj=datastream)
    g.write(data)
    g.close()
    ret = datastream.getvalue()
    return ret

# read: 0x61626364 == offset 0x1234 .. low nibble for each byte
def offsets(size, endian='<', mask=0x60606060):
    dwords = []
    ret = ""
    align = size % 4
    for i in range(0, size/4): # /sizeof(dword)
        nibbles = []
        nibbles.append(((i*4 + align) & 0xf000) >> 12)
        nibbles.append(((i*4 + align) & 0x0f00) >> 8)
        nibbles.append(((i*4 + align) & 0x00f0) >> 4)
        nibbles.append(((i*4 + align) & 0x000f))
        masked = mask
        masked |= nibbles[0] << 24
        masked |= nibbles[1] << 16
        masked |= nibbles[2] << 8
        masked |= nibbles[3]
        print "[+] masked: %.8X" % masked
        dwords.append(masked)
    for dword in dwords:
        ret += struct.pack(endian + 'L', dword)
    print "[+] returning .. %s" % ret
    return align * 'A' + ret

def iso8859toascii(buf):
    """
    00 -> 7F: ASCII
    A0 -> FF: ISO8859
    """
    table = {0x82:'e',
             0xa1:'!', 0xa2:'c', 0xa3:'L', 0xa5:'Y', 0xa6:'|', 0xa7:'S', 0xa8:'"', 0xa9:'(c)', 0xab:'<<',
             0xad:'-', 0xae:'(R)', 0xb0:'o', 0xb1:'+/-', 0xb2:'^2', 0xb3:'^3', 0xb4:'\'', 0xb5:'u', 0xb6:'P',
             0xb8:',', 0xb9:'^1', 0xba:'o', 0xbb:'>>', 0xbc:'1/4', 0xbd:'1/2', 0xbe:'1/3', 0xbf:'?',
             0xc0:'A', 0xc1:'A', 0xc2:'A', 0xc3:'A', 0xc4:'A', 0xc5:'A', 0xc6:'AE', 0xc7:'C',
             0xc8:'E', 0xc9:'E', 0xca:'E', 0xcb:'E', 0xcc:'I', 0xcd:'I', 0xce:'I', 0xcf:'I',
             0xd0:'D', 0xd1:'N', 0xd2:'O', 0xd3:'O', 0xd4:'O', 0xd5:'O', 0xd6:'O', 0xd7:'x',
             0xd8:'O', 0xd9:'U', 0xda:'U', 0xdb:'U', 0xdc:'U', 0xdd:'Y', 0xde:'b', 0xdf:'B',
             0xe0:'a', 0xe1:'a', 0xe2:'a', 0xe3:'a', 0xe4:'a', 0xe5:'a', 0xe6:'ae', 0xe7:'c',
             0xe8:'e', 0xe9:'e', 0xea:'e', 0xeb:'e', 0xec:'i', 0xed:'i', 0xee:'i', 0xef:'i',
             0xf0:'d', 0xf1:'ni', 0xf2:'o', 0xf3:'o', 0xf4:'o', 0xf5:'o', 0xf6:'o', 0xf7:'%',
             0xf8:'o', 0xf9:'u', 0xfa:'u', 0xfb:'u', 0xfc:'u', 0xfd:'y', 0xfe:'b', 0xff:'y'}
    ret = ""
    for c in buf:
        ordc = ord(c)
        if table.has_key(ordc):
            #devlog('iso8859toascii::replace', "0x%02x -> %s" % (ordc, table[ordc]))
            ret += table[ordc]
        else:
            if ordc > 0x7f and ordc < 0xa0:
                #devlog('iso8859toascii::missing', "not keeping non-ASCII char 0x%02x: %s" % (ordc, c))
                ret += "?"
            else:
                #devlog('iso8859toascii::keep', "keeping ISO8859 char 0x%02x -> %s" % (ordc, c))
                ret += c
    return ret

letras = string.ascii_letters + string.ascii_uppercase + string.digits

def randomstring(length):
    import random
    return "".join([random.choice(letras) for a in range(0, length)])

def randomletters(size=6):
    return ''.join(random.choice(string.ascii_letters) for x in range(size))

def xorstrings(a, b):
    """
    Xors two string/list of chars together and returns the string result

    b is the key, and if it is not long enough, it will be reused from the beginning
    """
    tmp = []
    for i in xrange(len(a)):
        a1 = str(a[i])
        b1 = str(b[i % len(b)])
        c1 = ord(a1) ^ ord(b1)
        tmp += [chr(c1)]
    return "".join(tmp)

def hexify(astr):
    """
    from \x41 to "\x41"
    """
    ret = ""
    for c in astr:
        ret += "\\x%2.2x"%ord(c)
    return ret

def prettydictprint(adict):
    """
    print out a dictionary how I like it printed out ...
    """
    keys = adict.keys()
    keys.sort()
    ret2 = "{"
    for key in keys:
        ret2 += "\"%s\" : \"%s\", "%(hexify(key),hexify(adict[key]))
    ret2 += "}"
    return ret2

def prettyhexprint(s,length=8):
    """
    A nicely displayed hexdump as a string
    """
    # we are expecting a string here
    if not type(s) == type(""):
        logging.error("Can not hexdump type %s" % type(s))
    tmp = []
    i = 1
    for c in s:
        tmp += ["%2.2x "%ord(c)]
        if i % length == 0:
            tmp += ["\n"]
        i += 1
    return "".join(tmp)


def cleanhexprint(s):
    """
    turns ABCD
    into
    41424344
    """
    tmp = []
    for c in s:
        tmp += ["%2.2x" % ord(c)]
    return "".join(tmp)


# Note: this is a 5m lame function
def hexdump(buf):
    tbl = []
    tmp = ""
    h   = ""
    i   = 0
    for a in buf:
        h += "%02X " % ord(a)
        i += 1
        if ord(a) >= 0x20 and ord(a) < 0x7f:
            tmp += a
        else:
            tmp += "."
        if i % 16 == 0:
            tbl.append((h, tmp))
            h = ""
            tmp = ""
    tbl.append((h, tmp))
    return tbl

def onlystrings(instr):
    """
    prints out all the strings - ununicodes, etc
    """
    instr = instr.replace("\x00","")
    tmp = ""
    space = 0
    for ch in instr:
        if (ch.isalnum() or ch in goodchars) and ord(ch) < 127:
            tmp += ch
            space = 0
        else:
            if not space:
                tmp += " "
                space = 1
    return tmp

def iisuni(inint):
    """
    Transforms an integer into %uXXYY%uZZAA - little endia order for IIS use
    """
    ret = ""
    ret += "%%u%4.4x%%u%4.4x" % (inint & 0xffff,((inint & 0xffff0000L) >> 16) & 0xffff)
    return ret

def iisunibuf(buf):
    """
    covert an entire buffer into IIS unicode format (note: makes 12 characters out of 4)
    """
    ret = ""
    mod = 4 - len(buf) % 4
    if mod != 4:
        buf += "\x00" * mod

    while buf != "":
        ret += "%%u%2.2x%2.2x%%u%2.2x%2.2x" % (ord(buf[1]), ord(buf[0]), ord(buf[3]), ord(buf[2]))
        buf = buf[4:]

    return ret

def htmlencode(buf):
    """
    takes in a string and encodes it to change all HTML entities (& " etc) into their HTML escape codes
    http://www.w3schools.com/xml/xml_cdata.asp
    """
    buf = str(buf)
    translation = { "<": "&lt;" , ">": "&gt;", "&": "&amp;" }
    for key in translation:
        buf = buf.replace(key,translation[key])
    return buf

def xmlencode(buf):
    """
    takes in a string and encodes it to change all XML entities (& " etc) into their XML escape codes
    http://www.w3schools.com/xml/xml_cdata.asp
    """
    buf = str(buf)
    translation = { "<": "&lt;" , ">": "&gt;", "&": "&amp;", "'": "&apos;", "\"": "&quot;" }
    for key in translation:
        buf = buf.replace(key,translation[key])
    return buf

def xmldecode(buf):
    """
    opposite of xmlencode()
    """
    from xml.sax.saxutils import unescape as xmlunescape
    return xmlunescape(buf, {"&apos;": "'", "&quot;": '"'})

def urlencode(buf):
    tmp = ""
    for c in buf:
        tmp += "%%%2.2x" % ord(c)
    return tmp

def urluencode(buf):
    tmp = []
    bufsize = len(buf)
    for c in range(0, bufsize - bufsize%2, 2):
        tmp.append("%%u%02x%02x" % (ord(buf[c+1]), ord(buf[c]) )  )
    if bufsize %2:
        tmp.append( "%%u%02x41" %  ord(buf[bufsize-1]) )
    return string.joinfields(tmp, "")

def uuencode(buf):
    import binascii
    s = ""
    l = 0
    while l < len(buf):
        s += binascii.b2a_uu(buf[l:l+45])
        l += 45
    return s

def uudecode(buf):
    import binascii
    # XXX do it for each line\n of buffer ?
    return binascii.a2b_uu(buf)

def uuencode_file(buf, filename="file", filemode=0644):
    return "begin %o %s\n%s`\nend" % (filemode & 0777, filename, uuencode(buf))

def b64encode(buf):
    import binascii
    return binascii.b2a_base64(buf)

def b64decode(buf):
    import binascii
    return binascii.a2b_base64(buf)

def indexable(var):
    return hasattr(var, '__getitem__')

def sizeable(var):
    return hasattr(var, '__len__')

def allowAddress(addr, badbytelist):
    for a in range(0, 32, 8):
        tmp = (addr >> a) & 0xff
        if tmp in badbytelist:
            return 0
    return 1

def intel_hex(instring):
    """
    reverses a string in hex
    """
    instring = binstring(instring)
    inlist = list(instring)
    inlist.reverse()
    instring = "".join(inlist)
    return instring

def getbyte(buf):
    """
    Gets one byte from the stream and returns it as an integer value
    """
    return ord(buf[0]), buf[1:]

def getint(buf):
    """For reading an intel order integer out of a buffer
    returns integer_in_intel_order, buf[4:]"""
    return istr2int(buf), buf[4:]

def getshort(buf):
    """
    Get an intel-orderd 16 bit integer from the buffer
    returns <the integer>,  <buffer[2:]>
    """
    return istr2halfword(buf), buf[2:]

#return 1 if there is a bad character in an integer word (converted to a str).
#of course, if you pass a string in, we just handle it as a string
def hasbadchar(word,badchars):
    try:
        wordstr = intel_order(word)
    except:
        wordstr = str(word)
    for ch in badchars:
        if wordstr.count(ch):
            return 1
    return 0

#returns a string with every other byte in it
def everyother(buffer):
    result = ""
    i = 0
    while i < len(buffer):
        result += buffer[i]
        i += 2
    return result

def csub(a,b):
    """
    Do subtraction like C does, with wrapover
    don't automatically convert to a LONG integer,
    which screws everything up
    """
    #print "A=%8.8x - B=%8.8x"%(a,b)
    c = uint32(a-b)
    #print "C= %8.8x %s"%(c,str(type(c)))
    return c

def istr2double(astring):
    firsthalf = str2littleendian(astring)
    secondhalf = str2littleendian(astring[4:])
    return long(firsthalf) + (long(secondhalf) << 32)

def reversedword(myint): #just here for namespace
    return reverseword(myint)

def reverseword(long):
    data = intel_order(long)
    ret = str2bigendian(data)
    return ret

def reverseshort(myshort):
    data = halfword2istr(myshort)
    newshort = nstr2halfword(data)
    return newshort

def s_intelword(myint):
    return intel_order(myint)

def UTF16toUTF8(utf16buf):
    """Unicode to UTF-8 encoder, should be Windows compliant."""
    utf8buf = ''
    if (len(utf16buf) % 2) == 1:
        utf16buf += '\x00'
    for i in range(0, len(utf16buf), 2):
        import struct
        H = struct.unpack('<H',utf16buf[i:i + 2])[0]
        #print '%d'%(H)
        if H < 0x80:
            utf8buf += utf16buf[i]
        elif H < 0x800:
            utf8buf += chr(0xc0 | ((ord(utf16buf[i + 1]) & 0xf) << 2) | (ord(utf16buf[i]) >> 6))
            utf8buf += chr(0x80 | (ord(utf16buf[i]) & 0x3f))
        else:
            utf8buf += chr(0xe0 | (ord(utf16buf[i + 1]) >> 4))
            utf8buf += chr(0x80 | ((ord(utf16buf[i + 1]) & 0xf) << 2) | (ord(utf16buf[i]) >> 6))
            utf8buf += chr(0x80 | (ord(utf16buf[i]) & 0x3f))
    return utf8buf

#returns a binary version of the string
def binstring(instring):
    result = []
    #erase all whitespace
    tmp = instring.replace(" ","")
    tmp = tmp.replace("\n","")
    tmp = tmp.replace("\t","")
    tmp = tmp.replace("\r","")
    tmp = tmp.replace(",","")

    if len(tmp) % 2 != 0:
        logging.error("Tried to binstring something of illegal length: %d: *%s*" % (len(tmp), prettyprint(tmp)))
        return ""

    while tmp != "":
        two = tmp[:2]
        #account for 0x and \x stuff
        if two != "0x" and two != "\\x":
            result += [chr(int(two, 16))]
        tmp = tmp[2:]
    result = "".join(result)
    return result

def pathunique(newpath):
    if newpath not in sys.path:
        sys.path.append(newpath)

def s_binary(instring):
    """
    for spike compatability
    """
    return binstring(instring)

#overwrites a string in place...hard to do in python
def stroverwrite(instring, overwritestring, offset):
    """
    We use this in our exploits as a way to keep them clean.
    str += str
    str += str2 is bad form.

    This api is more of a memcpy() which allows for easy setting of eip offsets and such.

    """
    head = instring[:offset]
    #print head
    tail = instring[offset + len(overwritestring):]
    #print tail
    result = head + overwritestring + tail
    return result

def getnullstring(instring):
    null = instring.find("\x00")
    if null == -1:
        return instring
    return instring[:null]

def sunrpcstr(instring, nonull=0):
    ret = big_order(len(instring)) + sunstring(instring, nonull=nonull)
    return ret

def pad4(instr):
    return sunstring(instr, nonull=1)

def sunstring(instring, nonull=0):
    """
    pads out a string with a null and to %4
    """

    #print "instring=%s"%instring
    ret = instring
    if not nonull:
        instring += chr(0)

    padlen = 4 - len(ret) % 4
    if padlen == 4:
        padlen = 0
    #print "Len=%d mod4=%d"%(len(ret),padlen)
    ret += chr(0) * (padlen)
    #print "hexofit=%s"%hexprint(ret)
    return ret

def mod4string(astr):
    return sunstring(astr)

def small_prettyprint(instring):
    prtstring = prettyprint(instring)
    if len(prtstring) > 100:
        prtstring = prtstring[:50] + "..." + prtstring[-50:]
    return prtstring

def cprint(instring):
    """
    prints out a  buffer the way C would like to see it
    """

    tmp = ""
    tmp = "\"" #add start quote
    for ch in instring:
        value = "\\x%2.2x" % ord(ch)
        tmp += value
    tmp += "\"" #add finish quote
    return tmp

def getsitebase(url):
    """
    Used by spkproxy.SPIDER() to get the site base of a url.
    """
    return "".join(url.split("/")[2:3]) + "/"

#returns None on error
#returns a dictionary of a string split like a normal HTTP argument list
def splitargs(argstring, orderlist=[]):
    resultDict = {}
    templist = argstring.split("&")
    for pair in templist:
        if pair != "":
            templist2 = pair.split("=")
            if len(templist2) < 2:
                #print "Failed to parse the URL arguments because of
                #invalid number of equal signs in one argument in:
                #\""+pair+"\" len="+str(len(templist2))
                return None
            else:
                #add this argument to the Dict
                orderlist.append(templist2[0])
                resultDict[templist2[0]] = "=".join(templist2[1:])
    return resultDict

def msunistring(instr):
    """Does a backwards string from a standard ASCII string and then adds two nulls"""
    if instr in [0,None]:
        # a null pointer ...
        return None
    ret = ""
    ret += backwardsunistring(instr)
    ret += "\x00\x00"
    return ret

def hashit(obj):
    """
    Returns a hashable object no matter what!
    """

    stringit=False
    try:
        if obj in {}:
            pass
    except TypeError:
        stringit = True

    if stringit:
        #unhashable object
        return "%r" % (obj,)
    else:
        return obj

def str_from_object(obj, blacklist=None):
    """
    Converts any object into a string representation
    in a recursive fashion
    """
    if blacklist == None:
        blacklist = {}
    ret = ""
    if hashit(obj) in blacklist:
        return "<blacklisted obj>"

    #prevent recursive decent
    if type(obj) in [type([]), type({})]:
        blacklist[hashit(obj)] = True

    if type(obj) == type([]):
        #we have a list
        ret += "["
        for o in obj:
            ret += str_from_object(o, blacklist)
            ret += ","
        ret = ret[:-1]
        ret += "]"
    elif type(obj) == type({}):
        #dictionary
        ret += "{"
        for o in obj.keys():
            ret += "%s: "%o
            ret += str_from_object(obj[o], blacklist)
            ret += ","
        ret = ret[:-1]
        ret += "}"
    else:
        ret += str(obj)

    return ret

def int32toIpstr(uint, swap=False):
    return socket.inet_ntoa(int2str32(uint, swap=swap))

def getNetworkInterfaces():
    """Returns a list of the network interface names on the current machine"""
    ret = []
    for i in getInterfaceData():
        ret += [i[0]]
    return ret

def parserouteprint(data):
    """
    Instead of parsing the ipconfig /all which relies on English
    text, we can pattern match a "route print" command much better.
    With Adam's blessing we will deprecate parseipconfig()
    """
    final_list      = []
    interface_count = 1
    split_string    = "=" * 75

    route_table = data.split(split_string)[3].split("\n")
    clean_route_table = route_table[3:len(route_table) - 2]

    # Now walk through the table and obtain all of the
    # interfaces and IP addresses
    found_ips = {}

    for route in clean_route_table:
        entry     = route.split(" ")
        temp_list = []

        for ip in entry:
            if ip:
                temp_list.append(ip)

        if len(temp_list):
            # Build our final list

            ip_addr = temp_list[3].replace("\t","").replace("\n","")

            if temp_list[1] != "255.255.255.255" and temp_list[1] != "0.0.0.0":
                if not found_ips.has_key(ip_addr):
                    netmask =  str2bigendian(socket.inet_aton(temp_list[1]))
                    final_list.append(["#%d" % interface_count, ip_addr, netmask])
                    found_ips[ip_addr] = netmask
                    interface_count += 1

    return final_list

def parseIPV6IFS(data):
    """
    Parses IPV6IFS.exe output in the form of 'IFID ADDRESS%SCOPE'
    """

    import struct
    ret = []
    for a in data.split("\n"):
        if a == "":
            continue
        b = a.split(" ")
        devlog('parseIPV6IFS', b)
        ifid = "#" + b[0]
        addr = b[1]
        logging.info("Win32 IPv6 parsed .. ifid %s -> addr %s" % (ifid, addr))
        mask = "0xffffffffffffffff" # bogus mask for now BETA
        if addr.find("error") != 0:
            ret.append( [ifid, addr, mask] )
    #print ret
    return ret

def getIPv6FromInterfaceLinux(interface):
    " Uses /proc/net/if_inet6 to get ipv6 addresses to interfaces "
    #print "[!] checking for Linux IPv6 interfaces ..."

    try:
        fd = open("/proc/net/if_inet6", "r")
    except IOError:
        #print "[!] no Linux IPv6 information found"
        return []
    #print "[!] parsing if_inet6 for: %s"% interface

    import struct
    import string

    addrlist = []
    try:
        # address, index, plen, scope, flags, interface
        # fe80000000000000020c29fffe18aa1c 02 40 20 80     eth0
        for line in fd.readlines():
            iflist = line.split() # split on spaces
            paddress = iflist[0] # the address in raw hex
            plen = string.atoi(iflist[2], base=16) # XXX: netmask ?
            pif = iflist[5] # the interface it's on
            address = struct.unpack("4s4s4s4s4s4s4s4s", paddress) # we want a string
            if interface in pif:
                address = ':'.join(address) # to : form
                import socket
                # compress the address (google for the win!)
                address = socket.inet_ntop(socket.AF_INET6, socket.inet_pton(socket.AF_INET6, address))
                #print "[!] IPv6: %s -> %s/%s"% (interface, address, plen)
                addrlist += [ [ address, plen ] ]
    except:
        import traceback
        traceback.print_exc(file=sys.stderr)
        #addrlist needs to be returned, but we're done parsing
        pass
    # return a list of found ipv6 addresses for that interface
    return addrlist

def getInterfaceData(log_output = True):
    """
    Get the Network adapters that  are local to this machine
    Linux/OS X: IPv4/IPv6/HWaddr
    Win32: IPv4/HWaddr (IPv6 comming soon for boxen > WinXP)

    returns a list of lists, each list has:
                 [iface_id (str), ip(str), netmask(str]
    """
    from libs import get_interfaces
    get_interfaces = get_interfaces.GetInterfaces()
    get_interfaces()

    interface_list = []

    ##From our container of all interfaces get each in turn
    for iface in get_interfaces.interfaces:
        ##Then for each interface get each NetIface object
        for ni_obj in get_interfaces.interfaces.get(iface):

            try:
                ##This hack is required at c6 has never differentiated
                ## between address families and so has always been buggy
                ##NASTY solution as we have to use these crappy lists
                ## rather than use a NetIface object is to suffix the
                ## name
                if ni_obj.family_name == "AF_INET6":
                    if_name = "%s-ipv6"%(ni_obj.name)
                else:
                    if_name = ni_obj.name

                interface_list.append([if_name,
                                       ni_obj.addr['address'],
                                       ni_obj.netmask['address']])
            except:
                continue
    if log_output:
        logging.info("Discovered interfaces:")
        for iface in interface_list:
            logging.info(" - %s" % iface)
    return interface_list

def xordata(data,key):
    ret = []
    for ch in data:
        ch = chr(ord(ch) ^ key)
        ret += [ch]
    return "".join(ret)

def get_source_ip(target):
    """
    Returns an IP address to send out from given a target ip address or host
    name
    """
    #We fake a connection here just to find the callback interface
    randport = random.randint(1,65534)
    find_me = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    #devlog("canvasengine","Connecting to %s:%d"%(target,randport))
    try:
        find_me.connect((target,randport))
    except socket.error:
        #could not assign requested address - i.e. we tried to connect
        #to a 192.168.1.0 and the computer complained
        #I dunno why it does this. It's super annoying.
        #This will always happen on udpscan, for example
        return None
    callback_ip = find_me.getsockname()[0]
    find_me.close()
    return callback_ip

def getAllLocalIPs():
    return getInterfaceData()

def getLocalIP():
    """
    Gets the local IP via some dumb tricks with popen()
    No good multiplatform way to do this though
    """
    IP = getAllLocalIPs()[0][1]
    return IP

def getIPfromARPTable():
    """
    Read hosts out of the arp cache and add them to our list
    """
    import os
    ipList = []
    if sys.platform == "win32" or os.name == "nt":
        try:
            if 0:
                #canvas_win32api only exists for Python24
                #so let's not do this
                import canvas_win32api
                arplist = canvas_win32api.GetARPEntryList()
                for entry in arplist:
                    ipList += [int32toIpstr(entry['Addr'], swap=True)]
        except:
            pass
    else:
        try: # Linux at least
            arplist = file("/proc/net/arp").readlines()[1:]
            for entry in arplist:
                if entry.count("00:00:00:00:00:00"):
                    continue
                ipList += [entry.split(" ")[0]]
        except IOError:
            pass
    return ipList

def read_bigendianword_packet(s):
    data = s.recv(4)
    #print "Data=%s"%prettyprint(data)
    size = str2bigendian(data)
    #print "Size=*%d*\n"%size
    data = s.recv(size)
    return data

def send_bigendian_halfword_packet(s, packet, additive=0):
    length = len(packet) + additive
    size = chr(length / 256 & 0xff)
    size += chr(length & 0xff)
    packet = size + packet
    return s.send(packet)

def rrecv(sock, size):
    retdata = ""
    newsize = 0
    while newsize < size:
        data = sock.recv(1)
        if data == "":
            logging.error("Connection ended?!")
            return retdata
        retdata += data
        newsize += len(data)
    return retdata

def recv_bigendian_halfword_packet(sock, additive=0):
    data = rrecv(sock,2)
    #print "size is "+prettyprint(data)
    size = ord(data[0])<<8
    size += ord(data[1])
    #print "Size of packet: %x"%size
    data2 = rrecv(sock, size + additive)
    return data2

def intelword_recv(s):
    """
    This function recieves an intel_order 32 bit integer,
    and then recieves that much data from the socket

    It may throw the timeout exception if it did not get all
    the data it expects
    """
    data = s.recv(4)
    #print "Data is "+prettyprint(data)
    size = istr2int(data)
    data = s.recv(size)
    return data

def intelhalfword_recv(s):
    """
    This function is used for some network protocols
    that send a sixteen bit intel_order (unsigned) number
    and then a block of data of that size

    It may throw a timeout exception if the data does not
    arrive in time
    """
    data = s.recv(2)
    size = istr2halfword(data)
    data = s.recv(size)
    return data

def badchartest(start, end, badchars):
    """
    used for generating strings to test for bad characters
    Generates strings such as:
    \x00\x01\x02...\xff that won't contain any characters in badchars argument, which should
    be a string (from self.badchars, typically).
    """
    tst = ""
    for c in range(start, end + 1):
        if chr(c) not in badchars:
            tst += chr(c)
    return tst

class IPList:
    """
    Class used for efficient production of a long list of
    IP's from an IP and netmask
    """
    def __init__(self, ip, netmask):
        """netmask is the /24, not the ffffff00"""
        #print "Netmask=%d"%netmask
        self.ip = str2bigendian(socket.inet_aton(ip))
        self.netmask = uint32(netmask)
        #print "netmask=%s"%netmask
        self.numberofips = 2 ** (32 - uint32(netmask)) #how many ip's total
        #print "number of ips=%s"%self.numberofips
        self.ip = self.ip & (~(self.numberofips - 1)) #need to mask it out so we don't do wacky things
        return

    def __len__(self):
        return int(self.numberofips)

    def __iter__(self):
        """
        We need to use yield here or else we will have to store
        huge amounts of ips
        """
        start = 0
        end = 1
        if self.netmask == 32:
            yield socket.inet_ntoa(big_order(self.ip))
        elif self.netmask >= 24:
            start = 1

        for i in xrange(start,self.numberofips - end):
            yield socket.inet_ntoa(big_order(self.ip + i))

def getIPList(ip):
    """
    Returns a list of IP's from 128.8.*.*
    or 128.8.0.0/16
    """

    #if ip is a name/netmask
    ret = []
    if ip.count("*"):
        number = ip.count("*")
        if number > 3:
            logging.error("Too many *'s in IP address")
            return None

        digits = ip.split(".")
        netmask = 32 - number * 8
        digits = digits[:4 - number] #adjust to account for *'s
        while len(digits) < 4:
            digits.append("0") #insert zeros
        ip = ".".join(digits)

    elif ip.count("/"):
        (ip, netmask) = ip.split("/")
        ip = socket.gethostbyname(ip) #we may have a hostname, so convert it
        netmask = int(netmask)

    else:
        # print "[+] Single IP %s" % ip
        ip = socket.gethostbyname(ip)
        netmask = 32

    #at this point we hvae a ip and a netmask
    #print "Network is %s" % ip
    #print "netmask is %d" % netmask
    return IPList(ip, netmask)

def orderIPlist(list_of_mixed_ips):
    """
    Little function to correctly order ip addresses in dotted octet notation
    supports ipv4 and ipv6
    Rich: this is the only way I can think of doing this? Anyone with any better
          ideas feel free to contribute :)
    """
    ipv4_dot_seperated_ips = []
    ipv6_dot_seperated_ips = []

    for ip_any in list_of_mixed_ips:
        if ":" in ip_any:
            ipv6_dot_seperated_ips.append(ip_any)

        elif "." in ip_any:
            ipv4_dot_seperated_ips.append(ip_any)

        else:
            logging.error("%s NOT a valid IP address" % (ip_any))

    a_ip_list = []

    #IPv4
    ##Convert to int repr of the address
    n_ip_list = [socket.inet_aton(a_ip) for a_ip in ipv4_dot_seperated_ips]
    ##Sort it
    n_ip_list.sort()
    ##And back to our dotted repr
    a_ip_list += [socket.inet_ntoa(n_ip) for n_ip in n_ip_list]

    #IPv6 - no ipv6 equivilent for ntoa/aton so just append best effort sorted list
    ipv6_dot_seperated_ips.sort()
    a_ip_list += ipv6_dot_seperated_ips

    return a_ip_list

def rawhexstr(fp):
    """ makes a raw hex string out of another string """
    return "%2.2x%2.2x%2.2x%2.2x" % (ord(fp[0]), ord(fp[1]), ord(fp[2]), ord(fp[3]))

def testip():
    """
    A small function that tests our getIPlist function
    """
    for i in getIPList("192.168.1.255/24"):
        print i

def ber_encoded_string(instr):
    """
    Returns a ASN1 BER encoded string (if the string is < 255)
    (for long strings, we return nothing!!)
    """
    tmp = ""
    tmp += s_binary("13")
    if len(instr) < 255:
        tmp += chr(len(instr))
        tmp += instr
    return tmp

# FIXME doesnt read until "\r\n\r\n" but only 1000bytes
def getWWWHeader(host, port, ssl = 0, sock = None):
    if sock == None:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    else:
        s = sock

    try:
        logging.info("Connecting to %s:%d" % (host, port))
        s.connect(host, port)
    except:
        return ""

    # setup function pointers
    if (ssl):
        sslsock = socket.ssl(s._sock)
        trecv = sslsock.recv
        twrit = sslsock.write
    else:
        trecv = s.recv
        twrit = s.send

    sploitstrings = [ "GET / HTTP/1.0\r\n\r\n", "HEAD / HTTP/1.0\r\n\r\n" ]
    sploitstrings += ["GET / HTTP/1.1\r\nHost: localhost\r\n\r\n" ]

    for sploitstring in sploitstrings:
        logging.info("Trying with: %s" % sploitstring)
        twrit(sploitstring)

        result = ""
        while(True):
            if "400 Bad Request" in result and "Server" not in result:
                continue
            # We cap at 20kb to avoid reading huge files
            if (len(result) > 20000):
                break
            try:
                tres = trecv(1000)
                if (tres == None or tres == ""):
                    #got something, so let's return?
                    break
                result += tres
            except:
                continue

            time.sleep(0.001)

        # We got a response from get don't bother with head
        if result:
            break

    return result

def uniquelist(alist):
    """
    Looks at all the elements in a list, and returns a new list
    that contains only the uniq elements.

    On Win32, you don't want the path to contain duplicates, as this
    greatly slows down module loading, so it's a good idea
    to use this function on sys.path after sys.path.append()
    """
    return dict((repr(x), x) for x in alist).values()

def heapIndex(base, size):
    """Used for finding the lookahead buffer in win32 heap overflows
    """
    size += 0x0f
    size &= 0xfffffff8L
    size = size >> 3
    index = size*0x30
    index += 0x688
    index += base
    return index

def reversestring(data):
    #reverse data - we can't assume 2.3 here, so we have to do this the hard way
    l = []
    l.extend(data)
    l.reverse()
    data2 = "".join(l)
    return data2

def isprint(str):
    for i in str:
        if not i in string.printable:
        #if not ord(i) in range(0x20,0x7f):
            return 0
    return 1

def getsearchpatternoffset(registerval, mychr="\xcc", startchar=0x01, badchars=""):
    """
    If you use searchpattern and you end up with
    eip=0xcccccc45, this function will return
    4*4 (aka 16)

    registerval is either an int, long or a string value
    """
    if type(registerval) in [type(1), type(1L)]:
        findme = intel_order(registerval)
    elif type(registerval) == type(""):
        findme = registerval
    else:
        logging.error("Don't know how to deal with type: %s" % type(registerval))
    length = 4 * 256 * 2 #max length, essentially
    ret = searchpattern(length, mychr, startchar, badchars).find(findme)
    return ret

def new_searchpattern(length):
    """
    TODO:
    generate a string from only A-Z

    total = 0

    i2strlist = []
    for c in xrange(ord("A"),ord("Z")):
        i2strlist += [chr(c)]
    #so now, i2strlist[4] == "D", etc
    #we can represent any decimal number by such:
    #1000 = BAAA. Arg. This won't work. Need to avoid
    #duplicates. so each block of 4 bytes is unique.
    #need to literally convert into base 26
    Better might be to do a random method for generating
    the string - have to create our own really simple
    random number generator  that is very predictable
    md5sum("CANVAS") and then md5sum of that, over and over
    mod 26
    """

from hashlib import md5
def md5sum_file(filename):
    """
    Gets an md5sum on a file, returns it as a string
    """
    fd = file(filename,"rb")
    mymd5 = md5()
    while 1:
        try:
            data = fd.read(1000)
        except:
            break
        if data == "":
            break
        mymd5.update(data)
    ret = mymd5.hexdigest()
    return ret

def searchpattern(length, mychr="\xcc", startchar=0x01, badchars=""):
    """
    Length is in bytes.

    Creates a pattern that looks like this:
    intel_order(0xcccccc41)+
    intel_order(0xcccccc42)+
    intel_order(0xcccccc43)+ ...

    And so on. This is useful for finding out where in your
    attack string a particular register has been loaded from.
    """

    i = 0
    ret = ""
    length = int(length)
    if type(startchar) == type(""):
        #if the user passed in a character, we convert it
        startchar = ord(startchar)

    while len(ret) < length:
        j = 0
        while j < (256 - startchar):
            if chr(startchar + j) not in badchars:
                ret += chr(startchar+j)+mychr*3
            j += 1
            if len(ret) >= length:
                break
        i += 1
        mychr = chr(ord(mychr) + 1)
    #in case length was not %4, we add padding here
    ret += mychr * (length % 4)
    ret = ret[:length]
    return ret

def reliablerecv(s, length):
    datalist = []
    readlength = 0
    while readlength < length:
        tmp = s.recv(length - readlength)
        if tmp == "":
            break
        readlength += len(tmp)
        datalist.append(tmp)
    data = "".join(datalist)
    return data

class reporterDict(dict):
    """
    Prints attempted accesses
    """
    def __getitem__(self, key):
        logging.info("Argument: %s" % key)
        return dict.__getitem__(self, key)

    def get(self, key, default=None):
        #import traceback
        #traceback.print_stack(file=sys.stderr)
        logging.info("Argument: %s with default %s" % (key, default))
        return dict.get(self, key, default)

def usage(app):
    """
    Prints out a quick usage note
    """
    import sys
    logging.info("Mandatory options:")
    logging.info("Standard options: -v <version> -t <host> [ -p <port> ]")
    logging.info("Callback options: -l <localhost> -d <localport>")
    logging.info("Additional options:")
    logging.info("Toggle test mode: -T")
    logging.info("Set covertness: -C <covertness>")
    logging.info("Custom options: (use -O option:value)")
    try:
        for option in app.options:
            print option
    except:
        pass

    if hasattr(app, "getArgs"):
        #print "Checking getArgs function for arguments"
        app.argsDict = reporterDict()
        app.getarg = app.fake_getarg
        app.getArgs()

    elif hasattr(app, "get_args"):
        #print "Checking get_args function for arguments"
        app.argsDict = reporterDict()
        app.getarg = app.fake_getarg
        app.get_args()

    elif hasattr(app, "getargs"):
        #print "Checking getargs functiobn for arguments"
        app.argsDict = reporterDict()
        app.getarg = app.fake_getarg
        app.getargs()

    else:
        logging.warning("This module does not have a getargs() function")

    # print "=" * 20 + "versions" + "=" * 26
    try:
        app.displayVersions()
    except AttributeError:
        pass
    try:
        app.engine.shutdown()
    except:
        pass

#used by canvasntlm and msrpc.py
def unicode2ascii(buf):
    result = ""
    for a in range(0, len(buf)):
        if not a % 2:
            result += buf[a]
    return result

def nounizeros(buf):
    if not buf:
        return ""
    ret = buf.replace("\x00", "")
    return ret

class Fortune:
    def __init__(self, file = "fortunes.txt"):
        try:
            self.__lines = open(file).readlines()
        except:
            self.__lines = [""]
        self.__index = -1
        import random
        random.shuffle(self.__lines)
    def __str__(self):
        self.__index += 1
        self.__index %= len(self.__lines)
        return self.__lines[self.__index].replace('&', '\n')[:-1]

def randomlist(randomlist):
    """
    Out of this list, pick a random one and return it
    """
    index = random.randint(0, len(randomlist) - 1)
    return randomlist[index]

def get_random_letters(length):
    letters = ["a", "b", "c", "d", "e", "A"]
    ret = ""
    for i in range(0, length):
        ret += randomlist(letters)
    return ret

def filesize_backup(crashname, crashname_backup, maxsize):
    """
    backs up a crashlog if necessary
    """
    try:
        f = file(crashname, "r")
        f.seek(0,2) #seek to end
        size = f.tell()
        f.close()
        if size > maxsize:
            logging.warning("crash file hit the threshold")
            logging.info("Removing old crash file %s" % crashname_backup)
            try:
                os.remove(crashname_backup)
                # print "[+] Crash file %s removed" % crashname_backup
            except:
                pass
            logging.info("Renaming %s to %s" % (crashname, crashname_backup))
            os.rename(crashname, crashname_backup)
            # print "[+] Renamed %s to %s" % (crashname, crashname_backup)
    except OSError, msg:
        #print "OSError: %s" % msg
        #no file to remove (windows error?)
        pass
    except IOError, msg:
        #print "IOError: %s" % msg
        #no file to remove (linux error)
        pass
    return

def bugreport(print_traceback_stderr = True):
    """
    Looks to see if our CRASH.log is > a large file should be
    and if it is, then moves it to the CRASH.bak file, removing that
    if it exists already.

    Writes (appends) a log of everything that happened in the crash to
    CRASH.log

    Obeys the canvas_root_directory global so you can store these
    correctly even if using canvas in some way we haven't intended yet.
    """
    # TODO: lock on file?
    import os, sys, datetime, traceback
    from versionsinfos import versionsinfos

    #from engine.config import canvas_root_directory
    #crashname=os.path.join(canvas_root_directory, "CRASH.log")
    #crashname_backup=os.path.join(canvas_root_directory, "CRASH.bak")

    from engine.config import canvas_root_directory
    crashname = os.path.join(canvas_root_directory, "CRASH.log")
    crashname_backup = os.path.join(canvas_root_directory, "CRASH.bak")

    maxsize = 1000000

    #check to see if we need to back our file up (prevents
    #the disk from filling up, esp. on SILICA)
    filesize_backup(crashname, crashname_backup, maxsize)

    #now open our crashname
    try:
        f = file(crashname, "a")
    except IOError:
        logging.warning("Cannot load CRASH.log - using CRASH.log.%s instead" % (str(os.getpid())))
        f = file(crashname + "." + str(os.getpid()), "a")
    f.write('-' * 80 + "\n\n[0] date:\n\n    %s\n\n" % datetime.datetime.now().ctime())
    f.write("[1] path:\n\n    %s\n\n" % sys.path)
    f.write("[2] modules loaded:\n\n")
    modkeys = sys.modules.keys()
    modkeys.sort()
    for modname in modkeys:
        f.write(" module %s:\n  %s\n" % (modname, sys.modules[modname]))
    try:
        infos = versionsinfos()
        f.write("\n[3] versions:\n\n")
        for info in infos:
            f.write("    %s\n" % infos[info])
    except:
        # will that happen ? (check versionsinfos::versionsinfos)
        if hasattr(os, 'uname'):
            uname_str = str(os.uname())
        else:
            uname_str = os.name
        f.write("\n[3] uname:\n\n    %s\n" % uname_str)
    f.write("\n[4] argv:\n\n    %s\n\n" % sys.argv)
    f.write("[5] traceback\n\n")

    #sometimes we are called from an exception handler, but if we are
    #not, then we really want the stack, not the exception...
    #this bug took a while to find. :<
    traceback.print_stack(file = f)
    traceback.print_exc(file = f)
    #also print to stderr, for the user to see.
    if print_traceback_stderr:
        traceback.print_stack(file = sys.stderr)
        traceback.print_exc(file = sys.stderr)

    ##If userdata exists append that
    try:
        u_fd = open("userdata", "rb")
        ud   = u_fd.read()
        u_fd.close()
    except IOError:
        ud = "\nNO userdata file\n"
    f.write("\n[6] userdata\n\n")
    f.write(ud)

    f.write("\n\n")
    f.close()

def file_utf8_encoding(data):
    ansi = locale.getdefaultlocale()[1]

    if data.startswith("\xEF\xBB\xBF"): # UTF-8 "BOM"
        encodings = ["utf-8-sig"]
    elif data.startswith(("\xFE\xFF","\xFF\xFE")):   # UTF-16 BOMs
        encodings = ["utf-16"]
    else:
        encodings = ["ascii","utf-8", ansi]

    for enc in encodings:
        try:
            udata = data.decode(enc).encode("utf-8","replace")
            break
        except UnicodeDecodeError:
            pass
    else:
        raise Exception("Unknown File Encoding")

    return udata


def filterdict(dontuse, adict):
    """
    Copies a dictionary to another dictionary minus a few keys
    useful for pickle's __getstate__
    """
    newdict = {}
    for key in adict:
        #will be none if it was removed while we are running this loop
        value = adict.get(key,None)
        if key not in dontuse:
            newdict[key] = value
    return newdict

def upperdict(dict):
    """
    for key in dictionary, make every key uppercase in a new dict
    """
    newdict = {}
    for key in dict:
        newdict[str(key).upper()] = dict[key]
    return newdict

def bugtracker(trackedfunction, *funcargs):
    """
    should be called before starting any new thread
    """
    try:
        return trackedfunction(*funcargs)
    # imo SyntaxError should not happen on customer side, and we dont want to report that on dev side.
    # KeyboardInterrupt shouldn't be reported as well.
    except KeyboardInterrupt:
        if debug_enabled:
            raise
        logging.error("Aborted by user")
        raise SystemExit
    except SystemExit:
        if debug_enabled:
            logging.error("SystemExit raised")
        raise
    except: # we DO want bugreport of SyntaxError :/
        bugreport()
        if debug_enabled:
            import internal
            internal.logging.display("\n\n%s\n\n" % Fortune('misfortunes.txt'), internal.logging.colors.YELLOW, "")
            raise
        logging.error("An error occurred, please send us your CRASH.log file (support@immunityinc.com)")
        raise SystemExit

def threadcheckMain(astr="Unknown"):
    """
    Reports a bug if we are not in the main thread, which means
    we are doing GUI actions in the wrong thread
    """
    currentthread = str(threading.currentThread())
    if "Main" not in currentthread:
        devlog("threading", "Serious bug - the wrong thread called: %s" % astr)
        bugreport()
    return

def threadchecknonMain(astr="Unknown"):
    """
    Reports a bug if we are in the main thread, since we should NEVER
    be in the main thread in certain functions/classes (only the
    GUI should be the main thread)
    """
    global threadchecknonMain_we_are_commandline
    if threadchecknonMain_we_are_commandline:
        #it's ok to run in the main thread as a commandline...
        #since that will be the only thread most likely
        return

    currentthread = str(threading.currentThread())
    if "Main" in currentthread:
        devlog("threading","Serious bug - the wrong thread called: %s"%astr)
        bugreport()
    return

def standard_callback_commandline(app,node=None,args=None, fromcommandline=True, quoteparse=False):
    """
    Calls our command line parser and sets a global to let our thread-checker
    know it's ok to execute things in the main thread.

    On the other hand, we want to eventually thread off our commandline interface
    at which point we should remove that global
    """
    global threadchecknonMain_we_are_commandline
    threadchecknonMain_we_are_commandline = True

    ##For the version,license and parsetable checks - Rich
    import canvasengine
    canvasengine.license_check()

    return bugtracker(_standard_callback_commandline, app, node, args, fromcommandline, quoteparse)


def quoteparse_split(args):
    """
    Transforms a string of "-O bob:1 -O bob2:"1 1"" into a list of arguments,
    with the "1 1" as a single argument.
    We could use lexx/yacc here, but it seemed like overkill.
    """
    quoted = False #true if we are in a quoted string
    escape = False #true if previous character was a \ and we were not in escape mode
    #go character by character and split this up
    current = "" #this holds the current argument
    ret = []
    for c in args:
        if c == " " and not quoted:
            #we finshed an argument
            ret.append(current)
            current=""
        elif c == "\\" :
            #handle backslashes carefully since they are our escape character
            if not escape:
                #we are now in escape mode
                escape = True
            else:
                #we escaped a backslash
                current += "\\"
        elif c == "\"":
            #quote characters are also special
            if escape:
                current += "\""
            elif quoted:
                #quoted, so we're done with that string - the next space should let us save an argument
                quoted = False
            else:
                #not quoted, so start a quote now
                quoted = True
        else:
            current += c
    #final string
    ret.append(current)
    #ret has our arguments now
    return ret

def commandline_fromengine(app, node, args, register_engine=None):
    """
    Used to call the commandline function from the canvas engine
    """
    _standard_callback_commandline(app, node=node, args=args, fromcommandline=False, quoteparse=True, register_engine=register_engine)
    return

# XXX: need to clean this whole function up
def _standard_callback_commandline(app, node=None, args=None, fromcommandline=True, quoteparse=False, register_engine=None):
    """
    Parses arguments and runs a module with those arguments
    node can be a list of nodes as well (used when called from the engine)
    """
    import getopt
    import canvasengine

    def disp_usage(msg = None):
        if msg:
            logging.info(msg)
        if hasattr(app, 'usage'):
            app.usage()
        else:
            devlog('all', "Commandline Error: Exploit class doesn't have a usage() method")
            usage(app)

        # Have to exit like this so as we kill all the threads that can throw exceptions is we sys.exit()
        # this means changing every usage() method in /exploits/* :(
        #print "CALLING EXIT NOW"
        #os._exit(0)

    # XXX license_check() here? || canvas_startup() (common gui/cmdline)

    doexit = 0
    if args == None and fromcommandline:
        args = sys.argv[1:]
        doexit = 1
    else:
        try:
            args2 = args + "\n"
            #simple parser here
            if not quoteparse:
                #just split on spaces (we are getting this from the bash shell, which already did our parsing)
                args = args.split(" ")
            else:
                #do quotes ourselves.
                #TODO
                args = quoteparse_split(args)
        except:
            #not a string
            pass
    #print "Args=%s"%args
    try:
        opts, args = getopt.getopt(args, "sv:t:p:l:d:TC:m:U:F:O:P:D")
        #print "opts=%s arts=%s"%(opts,args)
    except getopt.GetoptError, info:

        disp_usage("[-] Error in arguments: %s" % info)
        if doexit and app.engine:
            if hasattr(app.engine, 'shutdown'):
                app.engine.shutdown()
                ##Have to exit like this so as we kill all the threads that can throw exceptions if we sys.exit() - this means changing every usage() method in /exploits/* :(
                os._exit(0)
            else:
                print app.engine
                devlog('all', "Commandline Error: Exploit class doesn't have a shutdown() method")
        return 0

    #print "Starting parser"
    test = 0
    i = 0
    version = 0
    covertness = 1
    target = "1"
    port = None
    localhost = None
    localport = None
    #localhost=""
    #if hasattr(app, 'localhost'):
    #    localhost = app.localhost
    argsDict = app.argsDict
    for o, a in opts:
        #print "Parsing arg: %s"%o
        if o == "-v":
            version = int(a)
        if o == "-t":
            i += 1
            target = a
        if o == "-p":
            port = int(a)
        if o == "-T":
            i += 1
            test = 1
        if o == "-l":
            i += 1
            localhost = a
        if o == "-d":
            i += 1
            localport = int(a)
        if o == "-C":
            covertness=int(a)
        if o == "-m":
            argsDict["method"] = int(a)
        if o == "-U":
            argsDict["user"] = a
        if o == "-P":
            argsDict["password"] = a
        if o == "-F":
            argsDict["filename"] = a
        if o == "-Q":
            argsDict["helperhost"] = a
        if o == "-T":
            argsDict["command"] = a
        if o == "-O":
            i = a.find(":")
            name = a[:i]
            value = a[i+1:]
            argsDict[name] = value
        if o == "-D":
            #set this to do a clientd in a httpclientside exploit
            argsDict["Daemonize"] = True

    if register_engine:
        app.engine = register_engine

    if app.engine == None:
        app.engine = canvasengine.canvasengine(None)

    try:
        if localhost and localport:
            # if it needs resolving, resolve it
            try:
                localhost = socket.gethostbyname(localhost)
            except:
                pass
            logging.info("localhost = %s" % localhost)
            from listenerLine import fakeListenerLine
            app.callback = fakeListenerLine(localhost, localport)
    except UnboundLocalError:
        disp_usage("[-] Don't forget the localhost/localport parameters (-l and -d as in dog)")

        if doexit:
            if app.engine:
                app.engine.shutdown()
        return 0

    # autoListener from commandline if needed use the register_engine to start it ...
    if (hasattr(app, 'neededListenerTypes') == True
        and register_engine
        and not localport):

        autoFind = True
        if hasattr(app, 'autoFind'):
            autoFind = app.autoFind

        if 'HTTPPROXYPORT' in app.listenerArgsDict:
            HTTPPROXYPORT = int(app.listenerArgsDict['HTTPPROXYPORT'])
        else:
            HTTPPROXYPORT = 0

        neededlistenertypes = app.neededListenerTypes()

        listener = None

        if neededlistenertypes != []:
            register_engine.log("Running CLI autolistener for exploit that wants listener: %s" % repr(neededlistenertypes))
            target_host = None
            if app.target:
                target_host = app.target.interface
            listener = register_engine.autoListener(app, neededlistenertypes[0], host=target_host, autoFind=autoFind, HTTPPROXYPORT=HTTPPROXYPORT)
            if not listener:
                return 0, None

            if not hasattr(app, "fromcreatethread") or not app.fromcreatethread:
                listener.argsDict["fromcreatethread"] = False

            if listener.type != canvasengine.UNIVERSAL_MOSDEF:
                listener.argsDict = app.listenerArgsDict
                listener.current_exploit = app

        app.callback = listener

    #check to see if we were passed a list of nodes - this will
    #happen if called from the engine
    nodes = None
    if type(node) == type([]):
        nodes = node
        node = node[0]

    if node == None:
        if app.engine:
            #we don't want to create our OWN local node if we already have one
            node = app.engine.getLocalNode()
        else:
            from localNode import localNode
            node = localNode()

    from hostKnowledge import hostKnowledge, knowledgeContainer
    container = knowledgeContainer(node)
    app.target = hostKnowledge(target,container)

    #for debugging our pickle stuff
    #app.target.save_to_file()
    if nodes:
        argsDict["passednodes"] = nodes
    else:
        argsDict["passednodes"] = [node]
    if port:
        argsDict["port"] = port
    if localhost:
        argsDict["localhost"] = localhost
    if localport:
        argsDict["localport"] = localport
    app.argsDict = argsDict
    app.version = version
    app.covertness = covertness

    if test == 1:
        if not hasattr(app, 'test'):
            logging.warning("Module does not have a test() function")
        else:
            #print "Running test"
            if app.test():
                logging.info("This server seems vulnerable")
            else:
                logging.warning("Server does not seem vulnerable")
        if doexit:
            app.engine.shutdown()
        return 1

    if i < 1 and fromcommandline: # XXX
        disp_usage("Not enough arguments (%d)\n" % i)

        #print "Doexit=%d"%doexit
        if doexit:
            app.engine.shutdown()

        return 0
    try:
        app.createShellcode()
    except AttributeError:
        logging.error("Shellcode could not be created. No callback specified")
        import traceback
        traceback.print_exc(file = sys.stderr)
        return 0

    if app.callback:
        #from canvasexploit.py
        listenerfilename = "listener-%s" % str(app.callback.port)
        try:
            os.unlink(listenerfilename)
        except OSError:
            #didn't exist
            pass
        #we check for success with that filename, so we remove it before we run

    #
    # this is in order to replicate the same functionality of the old logging
    # mechanism with self.loggedInformation displayed in module_log_window
    #
    # We have to put this here and not in canvasexploit because of the way
    # we currently run modules, which is not through a standard manager
    #
    root = logging.getLogger()

    app.log_capture_io = StringIO.StringIO()
    app.sh = logging.StreamHandler(app.log_capture_io)
    app.sh.setLevel(logging.INFO)

    f = logging.Formatter("%(asctime)s [%(filename)24s] - %(levelname)s - %(message)s")
    app.sh.setFormatter(f)
    root.addHandler(app.sh)

    shell = app.run()

    if len(app.cleanup_files):
        app.perform_cleanup()

    if hasattr(app, "summary") and len(app.summary):
        logging.info("Summary for %s" % app.name)
        for n in app.argsDict['passednodes']:
            if app.summary[n] == 1:
                logging.warning("%s [%s]: Success" % (app.name, n))
            else:
                logging.error("%s [%s]: Fail" % (app.name, n))

    if app.succeeded or shell:
        #
        # This is for retrocompatibility to old modules
        #
        if not hasattr(app, "summary") and not len(app.summary):
            logging.info("Module succeeded!")

        if register_engine:
            # register the shell with the specified engine ...
            if hasattr(shell, 'nodetype'):
                shell.parentnode = register_engine.getLocalNode()
                register_engine.getLocalNode().newNode(shell)
                register_engine.newNode(shell, app)
            elif hasattr(shell, 'node'):
                shell.node.parentnode = register_engine.getLocalNode()
                register_engine.getLocalNode().newNode(shell.node)
                register_engine.newNode(shell.node, app) # deal with inconsistencies in API returns
    else:
        logging.error("Module did not report success")
    if localhost != None:
        logging.info("done -- connectback set to %s:%d" % (localhost, localport))
    if doexit:
        devlog("canvasengine", "standard_callback: shutting down engine")
        app.engine.shutdown()
        devlog("canvasengine", "standard_callback: shut down engine")
    if shell not in [1, 0, None, -1]:
        devlog('engine', "Setting engine to %s" % app.engine)
        if type(shell) == type(""):
            devlog('engine',"Error, module returned a string: %s!"%shell)
            shell = 0
        else:
            shell.engine = app.engine

    # Close and remove logging StreamHandler
    app.sh.close()
    root.removeHandler(app.sh)

    #return either a shellserver or a return code
    return shell

def getlines(astr, bugline):
    """
    prints the 3 lines before and after the line our bug is on
    """
    lines = astr.split("\n")
    lines = lines[bugline - 3:bugline + 3]
    return "\n".join(lines)

def printlines(astr, bugline):
    data = getlines(astr, bugline)
    print "-" * 80 + "\n" + data + "\n" + "-" * 80
    return

def snort_websend(s, pkt, timeout=60.0):
    """
    sends one byte at a time to annoy snort-like IDS's
    basically IDS's can't afford to store the same state as IIS. Funny, huh?
    """
    logging.error("Sending %d packets to attempt to bypass ids (delay between packets: %f)" % (len(pkt), timeout))
    start = pkt.find("/")
    if start == -1:
        start = 4
    header = pkt[start:].find(" ")
    if header == -1:
        header = 30

    logging.info("Packets from %d to %d will be sent with a delay between them" % (start, header))

    s.sendall(pkt[:start])
    for c in pkt[start:header]:
        s.sendall(c)
        time.sleep(timeout)
    s.sendall(pkt[header:])
    logging.info("Done sending")
    return

def snort_sendall(s, pkt, timeout=61.0):
    """
    sends one byte at a time to annoy snort-like IDS's
    """
    logging.info("Sending %d packets to attempt to bypass ids (delay between packets: %f)" % (len(pkt), timeout))
    start = 0
    header = len(pkt)
    logging.info("Packets from %d to %d (out of %d) will be sent with a delay between them" % (start, header, len(pkt)))
    i = start
    s.sendall(pkt[:start])
    for c in pkt[start:header]:
        s.sendall(c)
        logging.info("Sent packet %d" % i)
        i += 1
        time.sleep(timeout)
    s.sendall(pkt[header:])
    logging.info("Done sending")
    return

def strip_leading_path(source):
    "directory/file.exe to file.exe"
    i = len(source)-1
    try:
        while source[i] != '\\' and source[i] != '/':
            i -= 1
    except IndexError:
        i = -1
    source = source[i+1:]
    return source

def intersection(str1, str2):
    """
    for finding which bad chars you have
    """
    ret = []
    for c in str1:
        if c in str2:
            ret += [c]
    return "".join(ret)

def lazy_recv(fd, length):
    buf = "A"
    fullbuf = []
    wanted = length
    while buf != "" and wanted > 0:
        try:
            buf = fd.recv(wanted)
        except timeoutsocket.Timeout:
            break
        fullbuf += [buf]
        wanted -= len(buf)

    return "".join(fullbuf)

def host_is_interesting(host):
    """
    Hosts are not interesting if they are loopback or multicast, etc.
    """
    ip = socket.inet_aton(host)
    lip = str2int32(ip)
    if lip == 0xffffffffL: # broadcast
        return False
    a = lip >> 24
    if a == 127: # loopback
        return False
    if a == 224: # multicast
        return False
    if a == 240: # badclass
        return False

    # does that interest someone?
    if (lip & 0xe0000000L) == 0xe0000000L: # experimental
        return False
    return True

def select_stdin_and_socket_for_reading(sockfd, timeout = 0.2):
    """
    checks to see if the user has entered any data, otherwise looks
    to see if the remote side has caused an error of any kind
    """
    import select

    try:
        rd = []
        wr = []
        ex = []
        # so we can buffer stdin over xmlrpc proxy
        if hasattr(sys.stdin, 'isactive'):
            if sys.stdin.isactive() == True:
                rd += [sys.stdin.fileno()]
            if sockfd:
                r = []
                r, wr, ex = select.select([sockfd], [], [], timeout)
                rd += r
        else:
            if sockfd == None:
                # remote: HTTP/ICMP/ETC. .. no sockfd
                rd, wr, ex = select.select([sys.stdin.fileno()], [], [], timeout)
            else:
                rd, wr, ex = select.select([sockfd, sys.stdin.fileno()], [], [], timeout)
    except TypeError:
        logging.error("TypeError on select: sockfd is %s type" % type(sockfd))
        raise
    except select.error, (errcode, errmsg):
        if errcode == 10038: # win32 ENOTSOCK
            import os
            if os.name != 'nt':
                raise
            import msvcrt
            while True:
                rd = []
                wr = []
                ex = []
                # if remote == HTTP, do not try to select on sockfd
                if sockfd != None:
                    rd, wr, ex = select.select([sockfd], [], [], timeout)
                if msvcrt.kbhit():
                    rd += [sys.stdin.fileno()]
                if rd != []:
                    return (rd, [], [])
            raise select.error
    return (rd, [], [])

def tlssock(sock):
    """
    Creates a TLS Socket from a normal socket (returns object)
    """
    #HANDLE SSL HERE
    settings = HandshakeSettings()
    settings.minKeySize = 512 #some servers have a very small key
    settings.maxVersion = (3,1) #servers hate it when you are TLSv1.1
    connection = TLSConnection(sock)
    connection.handshakeClientCert(settings = settings)
    return connection

def writeflush(msg, file = sys.stdout):
    file.write(msg)
    file.flush()

def writeflush_for_status(msg, file = sys.stdout):
    """
    writeflush that leaves space right after msg for showing the end result
    of the operation (ok, fail)
    """
    entry = "%s%s" % (msg, (' ' * (72 - len(msg))))
    file.write(entry)
    file.flush()

def no_double_spaces(astr):
    """
    Replaces all strings of spaces with one space
    Not a fast function.
    """
    oldstr = ""
    while oldstr != astr:
        oldstr = astr
        astr = astr.replace("  ", " ") #two spaces to one
    return astr

#We do this once to get the user data
try:
    expiredate, contactemail = file("userdata","r").readlines()[:2]
except:
    # print "Did not find userdata file"
    expiredate = "0/0/0"
    contactemail = "bob@example.com"

###DNS Routines to find SMTP servers (used by smtp_fingerprinter and such)
#we do some tricks here to return the longest host first
def lens(a, b):
    return cmp(len(a), len(b))

def split_host(host):
    """
    Returns list of hosts from one host
    """
    tmp = []
    hosts = host.split(".")
    for a in range(len(hosts)):
        tmp += [".".join(hosts[-a:])]
    tmp.sort(cmp = lens, reverse=True)
    return tmp

def getIfaceMAC(iface):
    """
    Get the hwaddr of specified interface - works well for Linux/OS X
    for Win32 interface id's are more of an enigma so psuedo id's (0,1..)
    used in canvas - if you pass a psuedo id here all will be good.
    """
    from libs import get_interfaces
    get_interfaces = get_interfaces.GetInterfaces()
    get_interfaces()

    ##Look for hardware interface address
    return get_interfaces.interfaces.get_hwaddr(iface)

# Check if an IP is in the local reserved range
def check_reserved(ip):
    reserved = ["10.", "127.", "192.168.", "0."]
    # Append A class subrange for 172.
    for x in range(16,32):
        reserved.append("172."+ str(x) + ".")

    for x in reserved:
        if x == ip[:len(x)]:
            return True

    return False

# Converts IP netmask format to the CIDR format
def nmask2cidr(nmask):
    ip = struct.unpack(">L", socket.inet_aton(nmask))[0]
    result = 0
    while(1):
        # Bitwise least significant shifting
        if ip == 0 or (ip & 1):
            break
        ip >>= 1
        result += 1

    # we go opposite in big endian
    return 32 - result

def get_mx(host):
    """
    Linux only - gets the MX if possible for a host
    """
    #TODO: Add Windows support here.

    sin, sout_err = os.popen4("which host")
    data = sout_err.read()
    #print "which host data:%s"%data
    if "host" in data:
        #print "We have host command"
        #we have host command
        sin,sout_err = os.popen4("host -t mx %s"%host)
        data = sout_err.readlines()
        #print "host data: %s"%data
        #note, the following check is broken by non-English versions of Linux :<
        #we should be able to say "LANG=C" I believe to normalize this?
        if "is handled by " in data:
            #dave@ubuntu:~$ host -t mx immunityinc.com
            #immunityinc.com mail is handled by 0 a.mx.immunityinc.com.
            mail_server = data.split("is handled by")[1].split(" ")[1]
            return mail_server

    sin, sout_err = os.popen4("which dig")
    data = sout_err.read()
    if "dig" in data:
        #we have dig
        sin, sout_err = os.popen4("dig mx %s" % host)
        data = sout_err.readlines()
        answer = False
        for line in data:
            if answer or line.count(";; ANSWER"):
                answer = True
            else:
                continue
            if line.count("MX"):
                host = line.split(" ")
                #print "Host split: %s"%host
                host = host[-1].strip()
                #print "MX found: %s"%host
                return host

    #print "MX for %s not found"%host
    return None

def read_modules_file(filename=None, name=None):
    """Read txt file of exploit names
    For use in new exploits, favorite exploits etc
    """
    if not filename:
        import canvasengine
        if name == 'new':
            filename = os.path.join(canvasengine.canvas_resources_directory, "newmodules.txt")
        elif name == 'favorite':
            filename = os.path.join(canvasengine.canvas_resources_directory, "favmodules.txt")
        else:
            raise ValueError('unknown name for modules file: %s' % name)

    with open(filename) as infile:
        mod_list = []
        for t in infile:
            line = t.rstrip()
            if not line:
                continue
            elif line == "#":
                continue
            elif line == "\x20":
                continue
            elif line == "\n":
                continue
            else:
                mod_list.append(line)
    return mod_list

if __name__=="__main__":
    print "testing exploitutils code (uncomment what you want to test)..."
    #print "Str_from_object test=%s"%str_from_object(["HELLO", {"HELLO2": 1}])
    #hl=IPList("1.1.1.1",bits(0xff000000L))
    #for h in hl:
    #    print h


    searchstring=searchpattern(200,badchars="\r\n")
    print "Searchstring=%s"%repr(searchstring)
    off=0x10ccccccL
    print "Offset of %x = %d"%(off,getsearchpatternoffset(dInt(off),badchars=""))