spider.py

#!/usr/bin/env python
"""
	Spider Module for Grabber v0.1
	Copyright (C) 2006 - Romain Gaucher - http://rgaucher.info
"""
import urllib
import time
import re,sys,os,string
from BeautifulSoup import BeautifulSoup,SoupStrainer
from urllib2 import URLError, HTTPError
COOKIEFILE = 'cookies.lwp'          # the path and filename that you want to use to save your cookies in
import os.path
cj = None
ClientCookie = None
cookielib = None

import cookielib
import urllib2
urlopen = urllib2.urlopen
cj = cookielib.LWPCookieJar()       # This is a subclass of FileCookieJar that has useful load and save methods
Request = urllib2.Request
txdata = None
refererUrl = "http://google.com/?q=you!"
txheaders = {'User-agent' : 'Grabber/0.1 (X11; U; Linux i686; en-US; rv:1.7)', 'Referer' : refererUrl}

allowed=['php','html','htm','xml','xhtml','xht','xhtm',
         'asp','aspx','msp','mspx','php3','php4','php5','txt','shtm',
	    'shtml','phtm','phtml','jhtml','pl','jsp','cfm','cfml','do','py',
		'js', 'css']
database     = {}
database_url = []
database_css = []
database_js  = []
database_ext = [] # database of unsecure external links
local_url    = []
dumb_params  = [] # if there is no parameters associated with a given URL, associate this list of "whatever looks like"
root = "http://localhost"


outSpiderFile = None

"""
	database = {
	 u"URL" : {'GET' : {'param1':value}, 'POST' : { 'param2' : value }},
	 u"URL" : {'GET' : {'param1':value}, 'POST' : { 'param2' : value }},
	 u"URL" : {'GET' : {'param1':value}, 'POST' : { 'param2' : value }}
	}
"""
_urlEncode = {}
for i in range(256):
	_urlEncode[chr(i)] = '%%%02x' % i
for c in string.letters + string.digits + '_,.-/':
	_urlEncode[c] = c
_urlEncode[' '] = '+'


def urlEncode(s):
	""" 
		Returns the encoded version of the given string, safe for using as a URL. 
	"""
	return string.join(map(lambda c: _urlEncode[c], list(s)), '')



def urlDecode(s):
	""" 
		Returns the decoded version of the given string. Note that invalid URLs will throw exceptons. 
		For example, a URL whose % coding is incorrect. 
	"""
	mychr = chr
	atoi = string.atoi
	parts = string.split(string.replace(s, '+', ' '), '%')
	for i in range(1, len(parts)):
		part = parts[i]
		parts[i] = mychr(atoi(part[:2], 16)) + part[2:]
	return string.join(parts, '')



def htmlencode(s):
	"""
		Escaping the HTML special characters
	"""
 	s = s.replace("&", "&")
	s = s.replace("<", "&lt;")
	s = s.replace(">", "&gt;")
	s = s.replace("\"","&quot;")
	s = s.replace("'", "&apos;")
	return s



def htmldecode(s):
	"""
		Unescaping the HTML special characters
	"""
	s = s.replace("&lt;", "<")
	s = s.replace("&gt;", ">")
	s = s.replace("&quot;", "\"")
	s = s.replace("&apos;","'")
	s = s.replace("&amp;", "&")
	return s



def getContentDirectURL_GET(url, string):
	"""
		Get the content of the url by GET method
	"""
	ret = ""
	try:
		if len(string) > 0:
			url = url + "?" + (string)
		opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
		urllib2.install_opener(opener)
		req = Request(url, None, txheaders) # create a request object
		ret = urlopen(req)                     # and open it to return a handle on the url
	except HTTPError, e:
		return
	except URLError, e:
		return
	except IOError:
		return
	return ret



def scan(currentURL):
	"""
		The Scanner is the first part of Grabber.
		It retrieves every information of the HTML page
		TODO:
			Reading in every href='' element for CSS and src='' for JavaScript / Image
	"""
	try:
		archives_hDl = getContentDirectURL_GET(currentURL,'')
	except IOError:
		log <= ("IOError @ %s" % currentURL)
	try:
		htmlContent= archives_hDl.read()
	except IOError, e:
		print "Cannot open the file,",(e.strerror)
		return
	except AttributeError:
		print ("Grabber cannot retrieve the given url: %s" % currentURL)
		return
	parseHtmlLinks (currentURL,htmlContent)
	parseHtmlParams(currentURL,htmlContent)

def allowedExtensions(plop):
	for e in allowed:
		if '.'+e in plop:
			return True
	return False



def makeRoot(urlLocal):
	if allowedExtensions(urlLocal):
		return urlLocal[0:urlLocal.rfind('/')+1]
	return urlLocal



def giveGoodURL(href, urlLocal):
	"""
		It should return a good url...
		href = argument retrieven from the href...
	"""
	if 'javascript' in href:
		return htmldecode(urlLocal)
	if 'http://' in href or 'https://' in href:
		if urlLocal in href:
			return htmldecode(href)
		else:
			return urlLocal
	if len(href) < 1:
		return htmldecode(urlLocal)
	if href[0] == '?' and '?' not in urlLocal and not allowedExtensions(urlLocal):
		for e in allowed:
			if '.'+e in urlLocal:
				return htmldecode(urlLocal + href)
		return htmldecode(urlLocal + '/' + href)
	else:
		# simple name
		if allowedExtensions(urlLocal) or '?' in urlLocal:
			return htmldecode(urlLocal[0:urlLocal.rfind('/')+1] + href)
		else:
			return htmldecode(urlLocal + '/' + href)
	return htmldecode(href)


def dl(fileAdress, destFile):
	"""
		Download the file
	"""
	try:
		f =  urllib.urlopen(fileAdress)
		g = f.read()
		file = open(os.path.join('./', destFile), "wb")
	except IOError:
		return False
	file.write(g)
	file.close()
	return True


def removeSESSID(urlssid):
	"""
		Remove the phpsessid information... don't care about it now
	"""
	k = urlssid.find('PHPSESSID')
	if k > 0:
		return urlssid[0:k-1]
	k = urlssid.find('sid')
	if k > 0:
		return urlssid[0:k-1]
	return urlssid

def parseHtmlLinks(currentURL,htmlContent):
	global database_url,database_js,database_css
	"""
		Parse the HTML/XHTML code to get JS, CSS, links etc.
	"""
	links = SoupStrainer('a')
	# listAnchors = [tag['href'] for tag in BeautifulSoup(htmlContent, parseOnlyThese=links)]
	listAnchors = []
	for tag in BeautifulSoup(htmlContent, parseOnlyThese=links):
		try:
			string = str(tag).lower()
			if string.count("href") > 0:
				listAnchors.append(tag['href'])
		except TypeError:
			continue
		except KeyError:
			continue

	for a in listAnchors:
		goodA = giveGoodURL(a,currentURL)
		goodA = removeSESSID(goodA)
		if (root in goodA) and (goodA not in database_url):
			database_url.append(goodA)

	# parse the CSS and the JavaScript
	script = SoupStrainer('script')
	#listScripts = [tag['src'] for tag in BeautifulSoup(htmlContent, parseOnlyThese=script)]
	listScripts = []
	for tag in BeautifulSoup(htmlContent, parseOnlyThese=script):
		try:
			string = str(tag).lower()
			if string.count("src") > 0 and string.count(".src") < 1:
				listScripts.append(tag['src'])
		except TypeError:
			continue
		except KeyError:
			continue

	for a in listScripts:
		sc = giveGoodURL(a,currentURL)
		if sc not in database_js:
			database_js.append(sc)
		if sc == currentURL:
			# remote script
			database_ext.append(sc)
	parseJavaScriptCalls()

	link = SoupStrainer('link')
	# listLinks = [tag['href'] for tag in BeautifulSoup(htmlContent, parseOnlyThese=link)]
	listLinks = []
	for tag in BeautifulSoup(htmlContent, parseOnlyThese=link):
		try:
			string = str(tag).lower()
			if string.count("href") > 0:
				listLinks.append(tag['href'])
		except TypeError:
			continue
		except KeyError:
			continue

	for a in listLinks:
		sc = giveGoodURL(a,currentURL)
		if sc not in database_css:
			database_css.append(sc)
	return True

jsChars = ["'",'"']

def rfindFirstJSChars(string):
	b = [string.rfind(k) for k in jsChars]
	return max(b)

regDumbParam = re.compile(r'(\w+)')
regDumbParamNumber = re.compile(r'(\d+)')

jsParams = ["'",'"','=','+','%','\\',')','(','^','*','-']

def cleanListDumbParams(listDumb):
	newDumbList = []
	for w in listDumb:
		w = w.replace(' ','')
		w = w.replace('\n','')
		#l = [c for c in jsParams if c in w] # no jsParams
		if len(w) > 0 and regDumbParam.match(w) and not regDumbParamNumber.match(w):
			newDumbList.append(w)
	return newDumbList

def unique(L):
	noDupli=[]
	[noDupli.append(i) for i in L if not noDupli.count(i)]
	return noDupli

def flatten(L):
	if type(L) != type([]):
		return [L]
	if L == []:
		return L
	return reduce(lambda L1,L2:L1+L2,map(flatten,L))


def parseJavaScriptContent(jsContent):
	global database_url, database_ext, dumb_params
	"""
		Parse the content of a JavaScript file
	"""
	for l in jsContent.readlines():
		for e in allowed:
			if l.count('.'+e) > 0:
				# we found an external a call
				if l.count('http://') > 0 and l.count(root) < 1:
					# External link
					et= '.'+e
					b1 = l.find('http://')
					b2 = l.find(et) + len(et)
					database_ext.append(l[b1:b2])
				else:
					# Internal link
					et= '.'+e
					b2 = l.find(et) + len(et)
					b1 = rfindFirstJSChars(l[:b2])+1
					database_url.append(giveGoodURL(l[b1:b2],root))
		# try to get a parameter
		k = l.find('?')
		if k > 0:
			results = l[k:].split('?')
			plop = []
			for a in results:
				plop.append(cleanListDumbParams(regDumbParam.split(a)))
			dumb_params.append(flatten(plop))
		k = l.find('&')
		if k > 0:
			results = l[k:].split('&')
			plop = []
			for a in results:
				plop.append(cleanListDumbParams(regDumbParam.split(a)))
			plop = flatten(plop)
			dumb_params.append(flatten(plop))
	dumb_params = unique(flatten(dumb_params))

def parseJavaScriptCalls():
	global database_js
	"""
		Parse the JavaScript and download the files
	"""
	for j in database_js:
		jsName = j[j.rfind('/')+1:]
		if not os.path.exists('local/js/' + jsName):
			# first download the file
			dl(j,'local/js/' + jsName)
			try:
				jsContent = open('local/js/' + jsName, 'r')
			except IOError:
				continue
			parseJavaScriptContent(jsContent)
			jsContent.close()

def splitQuery(query_string):
	"""
		Split the num=plop&truc=kikoo&o=42 into
		a dictionary
	"""
	try:
		d = dict([x.split('=') for x in query_string.split('&') ])
	except ValueError:
		d = {}
	return d

def dict_add(d1,d2):
	"""
		Flatten 2 dictionaries
	"""
	d={}
	if len(d1):
		for s in d1.keys():
			d[s] = d1[s]
	if len(d2):
		for s in d2.keys():
			d[s] = d2[s]
	return d

def dict_add_list(d1,l1):
	d={}
	if len(d1):
		for s in d1.keys():
			d[s] = d1[s]
	if len(l1):
		for s in l1:
			d[s] = 'bar'
	return d

def parseHtmlParams(currentURL, htmlContent):
	global database, database_css, database_js
	"""
		Parse html to get args
	"""
	for url in database_url:
		k = url.find('?')
		if k > 0:
			keyUrl = url[0:k-1]
			query = url[k+1:]
			if not keyUrl in database:
				database[keyUrl] = {}
				database[keyUrl]['GET']  = {}
				database[keyUrl]['POST'] = {}
			lG = database[keyUrl]['GET']
			lG = dict_add(lG,splitQuery(query))
			database[keyUrl]['GET']  = lG
		elif len(dumb_params) > 0:
			keyUrl = url
			# no params in the URL... let's assign the dumb_params
			if not keyUrl in database:
				database[keyUrl] = {}
				database[keyUrl]['GET']  = {}
				database[keyUrl]['POST'] = {}
			lG = database[keyUrl]['GET']
			lP = database[keyUrl]['POST']
			lG = dict_add_list(lG,dumb_params)
			lP = dict_add_list(lP,dumb_params)
			database[keyUrl]['GET']  = lG
			database[keyUrl]['POST'] = lP

	# then, parse the forms
	forms = SoupStrainer('form')
	input = SoupStrainer('input')
	listForm = [tag for tag in BeautifulSoup(htmlContent, parseOnlyThese=forms)]
	for f in listForm:
		method = 'GET'
		if 'method' in f or 'METHOD' in f:
			method = f['method'].upper()
		action = currentURL
		if 'action' in f or 'ACTION' in f:
			action = f['action']
		keyUrl = giveGoodURL(action,currentURL)
		listInput = [tag for tag in BeautifulSoup(str(f), parseOnlyThese=input)]
		for i in listInput:
			if not keyUrl in database:
				database[keyUrl] = {}
				database[keyUrl]['GET']  = {}
				database[keyUrl]['POST'] = {}
			try:
				value = i['value']
			except KeyError:
				value = '42'
			try:
				name = i['name']
			except KeyError:
				name = 'foo'
				value= 'bar'
				continue
			lGP = database[keyUrl][method]
			lGP = dict_add(lGP,{name : value})
			database[keyUrl][method] = lGP
	return True


def runSpiderScan(entryUrl, depth = 0):
	global outSpiderFile
	print "runSpiderScan @ ", entryUrl, " |   #",depth
	if outSpiderFile:
		outSpiderFile.write("\t\t<entryURL>%s</entryURL>\n" % entryUrl)
	scan(entryUrl)
	if depth > 0 and len(database_url) > 0:
		for a in database_url:
			runSpiderScan(a, depth-1)
		return False
	return True


def spider(entryUrl, depth = 0):
	global root,outSpiderFile
	"""
		Retrieve every links
	"""
	if depth > 0:
		root = makeRoot(entryUrl)
	else:
		root = entryUrl
	
	# test if the spider has already be done on this website
	try:
		f = open("local/spiderSite.xml", 'r')
		firstLine = f.readline()
		f.close()
		if firstLine.count(root) > 0:
			alreadyScanned = True
		else:
			alreadyScanned = False
	except IOError:
		alreadyScanned = False

	print "Start scanning...", root
	if depth == 0:
		scan(root)
	else:
		if not alreadyScanned:
			outSpiderFile = open("local/spiderSite.xml","w")
			outSpiderFile.write("<spider root='%s' depth='%d'>\n" % (root,depth) )
			runSpiderScan(root, depth)
			if len(dumb_params) > 0:
				outSpiderFile.write("<dumb_parameters>\n")
				for d in dumb_params:
					outSpiderFile.write("\t<dumb>%s</dumb>\n" % (d))
				outSpiderFile.write("</dumb_parameters>\n")
			outSpiderFile.write("\n</spider>")
			outSpiderFile.close()
		else:
			print "Loading the previous spider results from 'local/spiderSite.xml'"
			# load the XML file
			regUrl = re.compile(r'(.*)<entryURL>(.*)</entryURL>(.*)',re.I)
			regDmb = re.compile(r'(.*)<dumb>(.*)</dumb>(.*)',re.I)

			f = open("local/spiderSite.xml", 'r')
			for l in f.readlines():
				if regUrl.match(l):
					out = regUrl.search(l)
					url = out.group(2)
					database_url.append(url)
				if regDmb.match(l):
					out = regDmb.search(l)
					param = out.group(2)
					dumb_params.append(param)
			f.close()

			# scan every url
			for currentURL in database_url:
				try:
					archives_hDl = getContentDirectURL_GET(currentURL,'')
				except IOError:
					log <= ("IOError @ %s" % currentURL)
					continue
				try:
					htmlContent= archives_hDl.read()
				except IOError, e:
					continue
				except AttributeError, e:
					continue
				parseHtmlParams(currentURL,htmlContent)


	outSpiderFile = open("results/touchFiles.xml","w")
	outSpiderFile.write("<spider root='%s'>\n" % root)
	for i in database_url:
		outSpiderFile.write("\t<url type='anchor'>%s</url>\n" % i)
	for i in database_js:
		outSpiderFile.write("\t<url type='JavaScript'>%s</url>\n" % i)
	for i in database_css:
		outSpiderFile.write("\t<url type='MetaLink'>%s</url>\n" % i)
	outSpiderFile.write("</spider>")
	outSpiderFile.close()

	if len(database_ext) > 0:
		# alert of External calls
		outSpiderFile = open("results/externalCalls.xml","w")
		outSpiderFile.write("<external>\n")
		for i in database_ext:
			outSpiderFile.write("\t<call severity='high'>%s</call>\n" % i)
		outSpiderFile.write("</external>")
		outSpiderFile.close()