Permalink
Switch branches/tags
Nothing to show
Find file
eb3eb2c Feb 15, 2016
@Janhouse @emilhem
executable file 711 lines (563 sloc) 25.4 KB
#!/usr/bin/env python2
#
# Copyright 2012-2013 Janis Jansons (janis.jansons@janhouse.lv)
#
import argparse
args=argparse.Namespace()
args.suppress=None
args.store=None
from SocksiPy import socks
import socket
# Magic!
def getaddrinfo(*args):
return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
socket.getaddrinfo = getaddrinfo
socket.setdefaulttimeout(None)
import urllib
import urllib2
import gzip
import sys
from multiprocessing import Process, Pipe, Manager
from lxml import etree
import time
from math import radians, cos, sin, asin, sqrt
from StringIO import StringIO
# Using StringIO with callback to measure upload progress
class CallbackStringIO(StringIO):
def __init__(self, num, th, d, buf = ''):
# Force self.buf to be a string or unicode
if not isinstance(buf, basestring):
buf = str(buf)
self.buf = buf
self.len = len(buf)
self.buflist = []
self.pos = 0
self.closed = False
self.softspace = 0
self.th=th
self.num=num
self.d=d
self.total=self.len*self.th
def read(self, n=10240):
next = StringIO.read(self, n)
#if 'done' in self.d:
# return
self.d[self.num]=self.pos
down=0
for i in range(self.th):
down=down+self.d.get(i, 0)
if self.num==0:
percent = float(down) / (self.total)
percent = round(percent*100, 2)
print_debug("Uploaded %d of %d bytes (%0.2f%%) in %d threads\r" %
(down, self.total, percent, self.th))
#if down >= self.total:
# print_debug('\n')
# self.d['done']=1
return next
def __len__(self):
return self.len
class TeSpeed:
def __init__(self, server = "", numTop = 0, servercount = 3, store = False, suppress = False, unit = False, chunksize=10240, downloadtests=15, uploadtests=10):
self.headers = {
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64; rv:11.0) Gecko/20100101 Firefox/11.0',
'Accept-Language' : 'en-us,en;q=0.5',
'Connection' : 'keep-alive',
'Accept-Encoding' : 'gzip, deflate',
#'Referer' : 'http://c.speedtest.net/flash/speedtest.swf?v=301256',
}
self.num_servers=servercount;
self.servers=[]
if server != "":
self.servers=[server]
self.server=server
self.down_speed=-1
self.up_speed=-1
self.latencycount=10
self.bestServers=5
self.units="Mbit"
self.unit=0
self.chunksize=chunksize
self.downloadtests=downloadtests
self.uploadtests=uploadtests
if unit:
self.units="MiB"
self.unit=1
self.store=store
self.suppress=suppress
if store:
print_debug("Printing CSV formated results to STDOUT.\n")
self.numTop=int(numTop)
#~ self.downList=['350x350', '500x500', '750x750', '1000x1000',
#~ '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000',
#~ '3500x3500', '4000x4000', '4000x4000', '4000x4000', '4000x4000']
#~ self.upSizes=[1024*256, 1024*256, 1024*512, 1024*512,
#~ 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2,
#~ 1024*1024*2, 1024*1024*2]
self.downList=[
'350x350', '350x350', '500x500', '500x500', '750x750', '750x750', '1000x1000', '1500x1500', '2000x2000', '2500x2500',
'3000x3000','3500x3500','4000x4000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000',
'1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000',
'2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000',
'2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000',
'4000x4000', '4000x4000', '4000x4000', '4000x4000', '4000x4000'
]
#'350x350', '500x500', '750x750', '1000x1000',
# '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000',
# '3500x3500', '4000x4000', '4000x4000', '4000x4000', '4000x4000'
#]
self.upSizes=[
1024*256, 1024*256, 1024*512, 1024*512, 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*512,
1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256,
1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512,
1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512,
1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256,
1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512,
1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2,
1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2,
# 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2,
# 1024*1024*2, 1024*1024*2]
]
self.postData=""
self.TestSpeed()
def Distance(self, one, two):
#Calculate the great circle distance between two points
#on the earth specified in decimal degrees (haversine formula)
#(http://stackoverflow.com/posts/4913653/revisions)
# convert decimal degrees to radians
lon1, lat1, lon2, lat2 = map(radians, [one[0], one[1], two[0], two[1]])
# haversine formula
dlon = lon2 - lon1
dlat = lat2 - lat1
a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
c = 2 * asin(sqrt(a))
km = 6367 * c
return km
def Closest(self, center, points, num=5):
# Returns object that is closest to center
closest={}
for p in range(len(points)):
now = self.Distance(center, [points[p]['lat'], points[p]['lon']])
points[p]['distance']=now
while True:
if now in closest:
now=now+00.1
else:
break
closest[now]=points[p]
n=0
ret=[]
for key in sorted(closest):
ret.append(closest[key])
n+=1
if n >= num and num!=0:
break
return ret
def TestLatency(self, servers):
# Finding servers with lowest latency
print_debug("Testing latency...\n")
po = []
for server in servers:
now=self.TestSingleLatency(server['url']+"latency.txt?x=" + str( time.time() ))*1000
now=now/2 # Evil hack or just pure stupidity? Nobody knows...
if now == -1 or now == 0:
continue
print_debug("%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n" %
(now, server['url'], server['sponsor'], server['name'], server['country'], server['distance']))
server['latency']=now
# Pick specified ammount of servers with best latency for testing
if int(len(po)) < int(self.num_servers):
po.append(server)
else:
largest = -1
for x in range(len(po)):
if largest < 0:
if now < po[x]['latency']:
largest = x
elif po[largest]['latency'] < po[x]['latency']:
largest = x
#if cur['latency']
if largest >= 0:
po[largest]=server
return po
def TestSingleLatency(self, dest_addr):
# Checking latency for single server
# Does that by loading latency.txt (empty page)
request=self.GetRequest(dest_addr)
averagetime=0
total=0
for i in range(self.latencycount):
error=0
startTime = time.time()
try:
response = urllib2.urlopen(request, timeout = 5)
except (urllib2.URLError, socket.timeout), e:
error=1
if error==0:
averagetime = averagetime + (time.time() - startTime)
total=total+1
if total==0:
return False
return averagetime/total
def GetRequest(self, uri):
# Generates a GET request to be used with urlopen
req = urllib2.Request(uri, headers = self.headers)
return req
def PostRequest(self, uri, stream):
# Generate a POST request to be used with urlopen
req = urllib2.Request(uri, stream, headers = self.headers)
return req
def ChunkReport(self, bytes_so_far, chunk_size, total_size, num, th, d, w):
# Receiving status update from download thread
if w==1:
return
d[num]=bytes_so_far
down=0
for i in range(th):
down=down+d.get(i, 0)
if num==0 or down >= total_size*th:
percent = float(down) / (total_size*th)
percent = round(percent*100, 2)
print_debug("Downloaded %d of %d bytes (%0.2f%%) in %d threads\r" %
(down, total_size*th, percent, th))
#if down >= total_size*th:
# print_debug('\n')
def ChunkRead(self, response, num, th, d, w=0, chunk_size=False, report_hook=None):
#print_debug("Thread num %d %d %d starting to report\n" % (th, num, d))
if not chunk_size:
chunk_size=self.chunksize
if(w==1):
return [0,0,0]
total_size = response.info().getheader('Content-Length').strip()
total_size = int(total_size)
bytes_so_far = 0
start=0
while 1:
chunk=0
if start == 0:
#print_debug("Started receiving data\n")
chunk = response.read(1)
start = time.time()
else:
chunk = response.read(chunk_size)
if not chunk:
break
bytes_so_far += len(chunk)
if report_hook:
report_hook(bytes_so_far, chunk_size, total_size, num, th, d, w)
end = time.time()
return [ bytes_so_far, start, end ]
def AsyncGet(self, conn, uri, num, th, d):
request=self.GetRequest(uri)
start=0
end=0
size=0
try:
response = urllib2.urlopen(request, timeout = 30);
size, start, end=self.ChunkRead(response, num, th, d, report_hook=self.ChunkReport)
#except urllib2.URLError, e:
# print_debug("Failed downloading.\n")
except:
print_debug(' \r')
print_debug("Failed downloading.\n")
conn.send([0, 0, False])
conn.close()
return
conn.send([size, start, end])
conn.close()
def AsyncPost(self, conn, uri, num, th, d):
postlen=len(self.postData)
stream = CallbackStringIO(num, th, d, self.postData)
request=self.PostRequest(uri, stream)
start=0
end=0
try:
response = urllib2.urlopen(request, timeout = 30);
size, start, end=self.ChunkRead(response, num, th, d, 1, report_hook=self.ChunkReport)
#except urllib2.URLError, e:
# print_debug("Failed uploading.\n")
except:
print_debug(' \r')
print_debug("Failed uploading.\n")
conn.send([0, 0, False])
conn.close()
return
conn.send([postlen, start, end])
conn.close()
def LoadConfig(self):
# Load the configuration file
print_debug("Loading speedtest configuration...\n")
uri = "http://speedtest.net/speedtest-config.php?x=" + str( time.time() )
request=self.GetRequest(uri)
response = None
try:
response = urllib2.urlopen(request, timeout=5)
except (urllib2.URLError, socket.timeout), e:
print_debug("Failed to get Speedtest.net config file.\n")
print_result("%0.2f,%0.2f,\"%s\",\"%s\"\n" % (self.down_speed, self.up_speed, self.units, self.servers))
sys.exit(1)
# Load etree from XML data
config = etree.fromstring(self.DecompressResponse(response))
ip=config.find("client").attrib['ip']
isp=config.find("client").attrib['isp']
lat=float(config.find("client").attrib['lat'])
lon=float(config.find("client").attrib['lon'])
print_debug("IP: %s; Lat: %f; Lon: %f; ISP: %s\n" % (ip, lat, lon, isp))
return { 'ip': ip, 'lat': lat, 'lon': lon, 'isp': isp }
def LoadServers(self):
# Load server list
print_debug("Loading server list...\n")
uri = "http://speedtest.net/speedtest-servers.php?x=" + str( time.time() )
request=self.GetRequest(uri)
response=None
try:
response = urllib2.urlopen(request);
except (urllib2.URLError, socket.timeout), e:
print_debug("Failed to get Speedtest.net server list.\n")
print_result("%0.2f,%0.2f,\"%s\",\"%s\"\n" % (self.down_speed, self.up_speed, self.units, self.servers))
sys.exit(1)
# Load etree from XML data
servers_xml = etree.fromstring(self.DecompressResponse(response))
servers=servers_xml.find("servers").findall("server")
server_list=[]
for server in servers:
server_list.append({
'lat': float(server.attrib['lat']),
'lon': float(server.attrib['lon']),
'url': server.attrib['url'].rsplit('/', 1)[0] + '/',
#'url2': server.attrib['url2'].rsplit('/', 1)[0] + '/',
'name': server.attrib['name'],
'country': server.attrib['country'],
'sponsor': server.attrib['sponsor'],
'id': server.attrib['id'],
})
return server_list
def DecompressResponse(sefl, response):
# Decompress gzipped response
data = StringIO(response.read())
gzipper = gzip.GzipFile(fileobj=data)
try:
return gzipper.read()
except IOError as e:
# Response isn't gzipped, therefore return the data.
return data.getvalue()
def FindBestServer(self):
print_debug("Looking for closest and best server...\n")
best=self.TestLatency(self.Closest([self.config['lat'], self.config['lon']], self.server_list, self.bestServers))
for server in best:
self.servers.append(server['url'])
def AsyncRequest(self, url, num, upload=0):
connections=[]
d=Manager().dict()
start=time.time()
for i in range(num):
full_url=self.servers[i % len(self.servers)]+url
#print full_url
connection={}
connection['parent'], connection['child']= Pipe()
if upload==1:
connection['connection'] = Process(target=self.AsyncPost, args=(connection['child'], full_url, i, num, d))
else:
connection['connection'] = Process(target=self.AsyncGet, args=(connection['child'], full_url, i, num, d))
connection['connection'].start()
connections.append(connection)
for c in range(num):
connections[c]['size'], connections[c]['start'], connections[c]['end']=connections[c]['parent'].recv()
connections[c]['connection'].join()
end=time.time()
print_debug(' \r')
sizes=0
#tspeed=0
for c in range(num):
if connections[c]['end'] is not False:
#tspeed=tspeed+(connections[c]['size']/(connections[c]['end']-connections[c]['start']))
sizes=sizes+connections[c]['size']
# Using more precise times for downloads
if upload==0:
if c==0:
start=connections[c]['start']
end=connections[c]['end']
else:
if connections[c]['start'] < start:
start=connections[c]['start']
if connections[c]['end'] > end:
end=connections[c]['end']
took=end-start
return [sizes, took]
def TestUpload(self):
# Testing upload speed
url="upload.php?x=" + str( time.time() )
sizes, took=[0,0]
counter=0
failures=0
data=""
for i in range(0, len(self.upSizes)):
if len(data) == 0 or self.upSizes[i] != self.upSizes[i-1]:
#print_debug("Generating new string to upload. Length: %d\n" % (self.upSizes[i]))
data=''.join("1" for x in xrange(self.upSizes[i]))
self.postData=urllib.urlencode({'upload6': data })
if i<2:
thrds=1
elif i<5:
thrds=2
elif i<7:
thrds=2
elif i<10:
thrds=3
elif i<25:
thrds=6
elif i<45:
thrds=4
elif i<65:
thrds=3
else:
thrds=2
sizes, took=self.AsyncRequest(url, thrds, 1)
#sizes, took=self.AsyncRequest(url, (i<4 and 1 or (i<6 and 2 or (i<6 and 4 or 8))), 1)
# Stop testing if too many failures
counter=counter+1
if sizes==0:
failures=failures+1
if failures>2:
break
continue
size=self.SpeedConversion(sizes)
speed=size/took
print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" %
(size, took))
print_debug("\033[92mUpload speed: %0.2f %s/s\033[0m\n" %
(speed, self.units))
if self.up_speed<speed:
self.up_speed=speed
if took>5 or counter>=self.uploadtests:
break
#print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % (self.SpeedConversion(sizes), took))
#print_debug("Upload speed: %0.2f MiB/s\n" % (self.SpeedConversion(sizes)/took))
def SpeedConversion(self, data):
if self.unit==1:
result=(float(data)/1024/1024)
else:
result=(float(data)/1024/1024)*1.048576*8
return result
def TestDownload(self):
# Testing download speed
sizes, took=[0,0]
counter=0
failures=0
for i in range(0, len(self.downList)):
url="random"+self.downList[i]+".jpg?x=" + str( time.time() ) + "&y=3"
if i<2:
thrds=1
elif i<5:
thrds=2
elif i<11:
thrds=2
elif i<13:
thrds=4
elif i<25:
thrds=2
elif i<45:
thrds=3
elif i<65:
thrds=2
else:
thrds=2
sizes, took=self.AsyncRequest(url, thrds )
#sizes, took=self.AsyncRequest(url, (i<1 and 2 or (i<6 and 4 or (i<10 and 6 or 8))) )
# Stop testing if too many failures
counter=counter+1
if sizes==0:
failures=failures+1
if failures>2:
break
continue
size=self.SpeedConversion(sizes)
speed=size/took
print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" %
(size, took))
print_debug("\033[91mDownload speed: %0.2f %s/s\033[0m\n" %
(speed, self.units))
if self.down_speed<speed:
self.down_speed=speed
if took>5 or counter>=self.downloadtests:
break
#print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % (self.SpeedConversion(sizes), took))
#print_debug("Download speed: %0.2f %s/s\n" % (self.SpeedConversion(sizes)/took, self.units))
def TestSpeed(self):
if self.server=='list-servers':
self.config=self.LoadConfig()
self.server_list=self.LoadServers()
self.ListServers(self.numTop)
return
if self.server == '':
self.config=self.LoadConfig()
self.server_list=self.LoadServers()
self.FindBestServer()
self.TestDownload()
self.TestUpload()
print_result("%0.2f,%0.2f,\"%s\",\"%s\"\n" % (self.down_speed, self.up_speed, self.units, self.servers))
def ListServers(self, num=0):
allSorted=self.Closest([self.config['lat'], self.config['lon']], self.server_list, num)
for i in range(0, len(allSorted)):
print_result("%s. %s (%s, %s, %s) [%0.2f km]\n" %
(i+1, allSorted[i]['url'], allSorted[i]['sponsor'], allSorted[i]['name'], allSorted[i]['country'], allSorted[i]['distance']))
def print_debug(string):
if args.suppress!=True:
sys.stderr.write(string.encode('utf8'))
#return
def print_result(string):
if args.store==True:
sys.stdout.write(string.encode('utf8'))
#return
# Thx to Ryan Sears for http://bit.ly/17HhSli
def set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host="127.0.0.1", port=9050):
socks.setdefaultproxy(typ, host, port)
socket.socket = socks.socksocket
def main(args):
if args.version:
print_debug("Tespeed v1.1\nNetwork speedtest using speedtest.net infrastructure - https://github.com/Janhouse/tespeed\n")
sys.exit(0)
if args.use_proxy:
if args.use_proxy==5:
set_proxy(typ=socks.PROXY_TYPE_SOCKS5, host=args.proxy_host, port=args.proxy_port)
else:
set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host=args.proxy_host, port=args.proxy_port)
if args.listservers:
args.store=True
if args.listservers!=True and args.server=='' and args.store!=True:
print_debug("Getting ready. Use parameter -h or --help to see available features.\n")
else:
print_debug("Getting ready\n")
try:
t=TeSpeed(
args.listservers and 'list-servers' or args.server,
args.listservers, args.servercount,
args.store and True or False,
args.suppress and True or False,
args.unit and True or False,
chunksize=args.chunksize,
downloadtests=args.downloadtests,
uploadtests=args.uploadtests,
)
except (KeyboardInterrupt, SystemExit):
print_debug("\nTesting stopped.\n")
#raise
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='TeSpeed, CLI SpeedTest.net')
parser.add_argument('server', nargs='?', type=str, default='', help='Use the specified server for testing (skip checking for location and closest server).')
parser.add_argument('-ls', '--list-servers', dest='listservers', nargs='?', default=0, const=10, help='List the servers sorted by distance, nearest first. Optionally specify number of servers to show.')
parser.add_argument('-w', '--csv', dest='store', action='store_const', const=True, help='Print CSV formated output to STDOUT.')
parser.add_argument('-s', '--suppress', dest='suppress', action='store_const', const=True, help='Suppress debugging (STDERR) output.')
parser.add_argument('-mib', '--mebibit', dest='unit', action='store_const', const=True, help='Show results in mebibits.')
parser.add_argument('-n', '--server-count', dest='servercount', nargs='?', default=1, const=1, help='Specify how many different servers should be used in paralel. (Default: 1) (Increase it for >100Mbit testing.)')
parser.add_argument('-p', '--proxy', dest='use_proxy', type=int, nargs='?', const=4, help='Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy.')
parser.add_argument('-ph', '--proxy-host', dest='proxy_host', type=str, nargs='?', default='127.0.0.1', help='Specify socks proxy host. (Default: 127.0.0.1)')
parser.add_argument('-pp', '--proxy-port', dest='proxy_port', type=int, nargs='?', default=9050, help='Specify socks proxy port. (Default: 9050)')
parser.add_argument('-cs', '--chunk-size', dest='chunksize', nargs='?', type=int, default=10240, help='Specify chunk size after wich tespeed calculates speed. Increase this number 4 or 5 times if you use weak hardware like RaspberryPi. (Default: 10240)')
parser.add_argument('-dt', '--max-download-tests', dest='downloadtests', nargs='?', type=int, default=15, help='Specify maximum number of download tests to be performed. (Default: 15)')
parser.add_argument('-ut', '--max-upload-tests', dest='uploadtests', nargs='?', type=int, default=10, help='Specify maximum number of upload tests to be performed. (Default: 10)')
#parser.add_argument('-i', '--interface', dest='interface', nargs='?', help='If specified, measures speed from data for the whole network interface.')
parser.add_argument('-v', '--version', dest='version', nargs='?', const=True, help='Show Tespeed version.')
args = parser.parse_args()
main(args)