forked from planetfederal/geoserver-performance-testing
-
Notifications
You must be signed in to change notification settings - Fork 0
/
generateNYC256.py
165 lines (135 loc) · 5.38 KB
/
generateNYC256.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
from __future__ import print_function
import csv
import datetime
import mmap
import multiprocessing
import threading
from Queue import Queue
from threading import Thread
import mercantile
import requests
from pyproj import *
""" Generates a csv file, so that each row contains a random bounding box, and width and height values equal to 256. These
parameters are used in the WMS GetMap request. The bounding boxes refer to areas in NYC.
Usage: python generateNYC256.py <csv file to be generated> <number of rows of csv file>
"""
def num2deg(xtile, ytile, zoom):
n = 2.0 ** zoom
lon_deg = xtile / n * 360.0 - 180.0
lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * ytile / n)))
lat_deg = math.degrees(lat_rad)
return (lat_deg, lon_deg)
def salehi():
with open(sys.argv[1], 'wb') as csvFile:
fileWriter = csv.writer(csvFile, delimiter=',')
numberOfRows = int(sys.argv[2])
sourceProj = Proj(init='epsg:4326')
targetProj = Proj(init='epsg:3857')
# x1, y1 = transform(sourceProj, targetProj, 43.0, 24.0)
# x2, y2 = transform(sourceProj, targetProj, 63.0, 40.0)
for i in mercantile.tiles(43.85, 25.25, 63.39, 39.87, [15], True):
bbox = mercantile.bounds(i)
assert isinstance(bbox, mercantile.LngLatBbox)
min_x, min_y = bbox.west, bbox.south
max_x, max_y = bbox.east, bbox.north
row = (min_x, min_y, max_x, max_y, 256, 256)
fileWriter.writerow(row)
numberOfRows = numberOfRows - 1
if numberOfRows <= 0:
break
def generate_all():
with open(sys.argv[1], 'a') as csvFile:
fileWriter = csv.writer(csvFile, delimiter=',')
numberOfRows = int(sys.argv[2])
iran_bbox = [4921321.6291, 2900938.0975, 7024868.6475, 4828374.2027]
iran_rightbottom_point = [6859229.6072, 2897727.7423]
base_tile = [4902976.742324346, 4775785.527257811, 4904199.7347769085, 4777008.519710374] # 15
base_tile = [4903588.238550628, 4776397.023484095, 4904199.7347769085, 4777008.519710374] # 16
base_tile = [4903893.986663768, 4776397.023484095, 4904199.7347769085, 4776702.7715972345] # 17
dx = base_tile[2] - base_tile[0]
dy = base_tile[3] - base_tile[1]
m = 1
tile = base_tile[:]
while tile[2] <= iran_rightbottom_point[0]:
tile[:] = [
tile[0] + dx * m,
base_tile[1],
tile[2] + dx * m,
base_tile[3]]
while tile[3] >= iran_rightbottom_point[1]:
fileWriter.writerow(tile)
numberOfRows -= 1
if numberOfRows <= 0:
exit(0)
tile[:] = [
tile[0],
tile[1] - dy * m,
tile[2],
tile[3] - dy * m]
base = 'http://geoserver/geoserver/gwc/service/wms?service=WMS&request=GetMap&layers=Shiveh:ShivehGSLD256&styles=&format=image/png&transparent=false&version=1.1.1&height=256&width=256&srs=EPSG:3857&bbox='
ans = {}
class ReqThread(threading.Thread):
def __init__(self, n, bbox, timeout):
threading.Thread.__init__()
self.n = n
self.bbox = bbox
self.timeout = timeout
self.req = None
def run(self):
self.req = requests.get('%s%s' % (base, self.bbox), timeout=self.timeout)
if self.req.status_code == 200:
ans[self.n] = 'y'
else:
ans[self.n] = 'n'
class Worker(Thread):
"""Thread executing tasks from a given tasks queue"""
def __init__(self, tasks):
Thread.__init__(self)
self.tasks = tasks
self.daemon = True
try:
self.start()
except:
pass
def run(self):
while True:
func, args, kargs = self.tasks.get()
try:
func(*args, **kargs)
except Exception as e:
print(e)
finally:
self.tasks.task_done()
class ThreadPool:
"""Pool of threads consuming tasks from a queue"""
def __init__(self, num_threads):
self.tasks = Queue(num_threads)
for _ in range(num_threads): Worker(self.tasks)
def add_task(self, func, *args, **kargs):
"""Add a task to the queue"""
self.tasks.put((func, args, kargs))
def wait_completion(self):
"""Wait for completion of all the tasks in the queue"""
self.tasks.join()
def bulk_req():
cpus = multiprocessing.cpu_count()
pool = ThreadPool(10000)
print('Number of threads: {}'.format(len(threading.enumerate())))
numOfLines = int(os.popen('wc -l %s' % sys.argv[1]).read().split(' ')[0])
with open(sys.argv[1], 'r') as csvFile:
mm = mmap.mmap(csvFile.fileno(), 0, prot=mmap.PROT_READ)
start = datetime.datetime.now()
for n, bbox in enumerate(iter(mm.readline, "")):
pool.add_task(requests.get, '%s%s' % (base, bbox))
if n % 1000 == 0:
print(n)
elapsed = datetime.datetime.now() - start
eta = (elapsed.seconds * numOfLines) / (n * 60)
print("%d seconds" % elapsed.seconds)
print("%d ETA(minutes)" % eta)
mm.close()
if __name__ == '__main__':
start = datetime.datetime.now()
generate_all()
elapsed = datetime.datetime.now() - start
print("Total: %d" % elapsed.seconds)