-
Notifications
You must be signed in to change notification settings - Fork 1
/
collect-data.js
111 lines (85 loc) · 2.63 KB
/
collect-data.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
var req = require('superagent')
, fs = require('fs')
, maxRequests = 1000
var BASE_URL= "https://maps.googleapis.com/maps/api/distancematrix/json?"
, API_KEY= process.env.API_KEY
, CENTER = JSON.parse(process.env.CENTER)
, OUTFILE = process.env.DATA_FILE
, BOUNDING_BOX = JSON.parse(process.env.BOUNDS)
, OUT = require(OUTFILE)
, misses = 0
console.log("Collecting data center:", CENTER, "bounds:", BOUNDING_BOX, " >", OUTFILE)
var getPoints = function(dests, cb){
var query = "origins=" + CENTER.join(',')
query += "&destinations=" + dests.map(function(c){return c.join(',')}).join('|')
query += "&mode=transit"
query += "&key=" + API_KEY
console.log("get", dests.length)
req.get(BASE_URL+query)
.end(function(err, resp){
if (err){
console.log("API ERROR:", err)
}
handleResponse(dests, resp, cb)
})
}
var handleResponse = function(req, resp, cb){
var json = JSON.parse(resp.text)
, out = {}
, i = 0
json.rows.forEach(function(r){
r.elements.forEach(function(el){
if (el.status && el.status == "ZERO_RESULTS"){
misses ++
i ++
return
}
var dest = req[i]
, dist = el.distance.value
, duration = el.duration.value
out[dest.join(',')] = {d: dist, t: duration}
i++
})
})
cb(null, out)
}
var randomPoint = function(bounding){
var minLat = Math.min(bounding[0][0], bounding[1][0])
, maxLat = Math.max(bounding[0][0], bounding[1][0])
, latScale = maxLat - minLat
, minLon = Math.min(bounding[0][1], bounding[1][1])
, maxLon = Math.max(bounding[0][1], bounding[1][1])
, lonScale = maxLon - minLon
return [ (minLat + Math.random() * latScale)
, (minLon + Math.random() * lonScale)]
}
var storeResults = function(err, res){
if (err)
throw err
Object.keys(res).forEach(function(k){
OUT[k] = res[k]
})
fs.writeFileSync(OUTFILE, JSON.stringify(OUT), 'utf8')
}
var sent = 0
var rateLimitGet = function(coordGen){
var ratePerReq = 30 //99
, timeout = 60*1000*10 // 10 min for now
, dests = []
if (sent > maxRequests){
console.log("Exceeded rate limit, sleep for Day :", sent)
maxRequests += sent
return setTimeout(rateLimitGet.bind(null, coordGen), 1000*60*60*24)
//process.exit(0)
}
for (var i=0; i<ratePerReq; i++){
dests.push(coordGen(BOUNDING_BOX))
}
getPoints(dests, function(err, res){
sent += Object.keys(res).length
storeResults.apply(this, arguments)
console.log("CB:", sent, Object.keys(OUT).length, misses)
setTimeout(rateLimitGet.bind(null, coordGen), timeout)
})
}
rateLimitGet(randomPoint)