Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
1 contributor

Users who have contributed to this file

214 lines (183 sloc) 7.03 KB
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 18/4/15 上午2:07
# @Author : 重剑无锋
# @Email : TideSecPlus@gmail.com
import random,os,mysql.connector
import urllib2,re,requests
import time,hashlib,urllib
import sys
mysql_config = {
'host': '127.0.0.1',
# 'host': '192.168.1.201',
'port': 3306,
'db_name': 'finger',
'username': 'root',
'password': '123456'
}
def url_protocol(url):
domain = re.findall(r'.*(?=://)', url)
if domain:
return domain[0]
else:
return url
def get_domain(target):
try:
url = target
if url[0:4] == 'http':
proto, rest = urllib.splittype(url)
host, rest = urllib.splithost(rest)
if host[0:3] == 'www':
host = host[4:]
elif url[0:3] == 'www':
host = url[4:]
else:
host = url
if ':' in host:
host = host.split(':')[0]
if '/' in host:
host = host.split('/')[0]
return host
except:
return target
def get_main_domain(domain):
double_exts = ['.com.cn','.edu.cn','.gov.cn','.org.cn','.net.cn']
main_domain = domain
for ext in double_exts:
if ext in domain:
if len(domain.split('.')) > 3:
# print "yuanshi",domain
domain_split = domain.split('.')
domain_new = "%s.%s.%s" % (domain_split[-3], domain_split[-2], domain_split[-1])
# print "exact",domain
main_domain = domain_new
else:
main_domain = domain
break
else:
if len(domain.split('.')) > 2:
domain_split = domain.split('.')
domain_new = "%s.%s" % (domain_split[-2], domain_split[-1])
main_domain = domain_new
else:
main_domain = domain
return main_domain
def requests_headers():
'''
Random UA for every requests && Use cookie to scan
'''
user_agent = ['Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.8.1) Gecko/20061010 Firefox/2.0',
'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.6 Safari/532.0',
'Mozilla/5.0 (Windows; U; Windows NT 5.1 ; x64; en-US; rv:1.9.1b2pre) Gecko/20081026 Firefox/3.1b2pre',
'Opera/10.60 (Windows NT 5.1; U; zh-cn) Presto/2.6.30 Version/10.60','Opera/8.01 (J2ME/MIDP; Opera Mini/2.0.4062; en; U; ssr)',
'Mozilla/5.0 (Windows; U; Windows NT 5.1; ; rv:1.9.0.14) Gecko/2009082707 Firefox/3.0.14',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
'Mozilla/5.0 (Windows; U; Windows NT 6.0; fr; rv:1.9.2.4) Gecko/20100523 Firefox/3.6.4 ( .NET CLR 3.5.30729)',
'Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16',
'Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5']
UA = random.choice(user_agent)
headers = {
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'User-Agent':UA,'Upgrade-Insecure-Requests':'1','Connection':'keep-alive','Cache-Control':'max-age=0',
'Accept-Encoding':'gzip, deflate, sdch','Accept-Language':'zh-CN,zh;q=0.8',
"Referer": "http://www.baidu.com/link?url=www.so.com&url=www.soso.com&&url=www.sogou.com"}
return headers
excludeext = ['.png', '.ico', '.gif','.svg', '.jpeg','js','css','xml','txt']
def getPageLinks(url):
try:
headers = requests_headers()
content = requests.get(url, timeout=5, headers=headers, verify=False).text.encode('utf-8')
links = []
tags = ['a', 'A', 'link', 'script', 'area', 'iframe', 'form'] # img
tos = ['href', 'src', 'action']
if url[-1:] == '/':
url = url[:-1]
try:
for tag in tags:
for to in tos:
link1 = re.findall(r'<%s.*?%s="(.*?)"' % (tag, to), str(content))
link2 = re.findall(r'<%s.*?%s=\'(.*?)\'' % (tag, to), str(content))
for i in link1:
links.append(i)
for i in link2:
if i not in links:
links.append(i)
except Exception, e:
print e
print '[!] Get link error'
pass
return links
except:
return []
def getMD5(c):
m = hashlib.md5()
m.update(c)
psw = m.hexdigest()
return psw
def request_url(url):
try:
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:59.0) Gecko/20100101 Firefox/59.0'
}
r = requests.get(url=url, headers=requests_headers(),timeout=5,verify=False,)
r.encoding = 'utf-8'
if r.status_code==200:
return r.content
else:
return ''
except Exception,e:
# print e
return ''
def update_mysql(conn,url,filename,md5):
try:
cur=conn.cursor()
sql = "select * from file_md5 where md5 = '"+str(md5)+"'"
print sql
cur.execute(sql)
exist_md5 = cur.fetchone()
if exist_md5:
url = exist_md5[2]+','+url
hint = exist_md5[3]+1
sql = "update file_md5 set url = '%s',hint='%s' where md5 = '%s'" % (url,hint,md5)
else:
sql = "INSERT INTO file_md5 (url,filename,md5,hint) VALUES ('%s','%s','%s','%s')" % (url,filename,md5,1)
print sql
cur.execute(sql)
conn.commit()
except Exception,e:
print e
pass
conn=mysql.connector.connect(user=mysql_config['username'],password=mysql_config['password'],host=mysql_config['host'],database=mysql_config['db_name'],charset='utf8')
# url = 'http://www.sddlr.gov.cn/'
for url in open('url.txt','r').readlines():
url= url.strip()
print url
urlprotocol = url_protocol(url)
domain_url = get_domain(url)
pageLinks = getPageLinks(url)
print pageLinks
true_url =[]
for suburl in pageLinks:
for ext in excludeext:
if ext in suburl[-4:]:
# print ext,' ',suburl
if re.findall(r'/', suburl):
if re.findall(r':', suburl):
true_url.append(suburl)
else:
true_url.append(urlprotocol + '://' + domain_url + '/' + suburl)
else:
true_url.append(urlprotocol + '://' + domain_url + '/' + suburl)
url_tmp1 = set(list(true_url))
print "原长度: ",len(url_tmp1)
url_tmp2 = []
for x in url_tmp1:
url_tmp2.append(x)
print "去重后: ",len(url_tmp2)
for url_tmp in url_tmp2:
url_content = request_url(url_tmp)
filename = os.path.basename(url_tmp)
md5 =getMD5(url_content)
print url_tmp,' ',filename,md5
update_mysql(conn,url_tmp,filename,md5)
You can’t perform that action at this time.