-
Notifications
You must be signed in to change notification settings - Fork 1
/
coordinatescrape.py
59 lines (52 loc) · 17 KB
/
coordinatescrape.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
import urllib
import re
import codecs
import time
def scrap_coordinates(towns):
# for scrapping list of locations
coor_list = []
i = 1
j = 0
while i < len(towns):
url = "http://maps.googleapis.com/maps/api/geocode/json?address="+towns[i]+",+Singapore&sensor=false"
textdecode = urllib.urlopen(url).read().decode(encoding='UTF-8')
# 2 - define patterns able to match the part of the google map api output we are interested in
regexLat = '"location".+\\n.+"lat".+?([0-9]{1}\.[0-9]{,14}).?\\n'
regexLon = '"location".+?\\n.+?\\n.+?"lng".+([0-9]{3}\.[0-9]{,14}).?\\n'
patternLat = re.compile(regexLat)
patternLon = re.compile(regexLon)
# 3 - scrape those parts of the google map api output matching our patterns
lat = re.findall(patternLat,textdecode)
lon = re.findall(regexLon,textdecode)
# 4 - print the scraping output on the console window
print lat
print lon
print towns[i], "latitude", lat[0], "longitude", lon[0]
time.sleep(0.1) # this is a stratagem to get around the Google detection of intensive scraping
coor_list.append((towns[i], lat[0], lon[0]))
if j == 10:
url = "http://facebook.com"
textdecode = urllib.urlopen(url).read().decode(encoding='UTF-8')
j = 0
i += 1
j += 1
return coor_list
def scrap_coordinate(locality):
# for scrapping single locality
url = "http://maps.googleapis.com/maps/api/geocode/json?address="+locality+",+Singapore&sensor=false"
textdecode = urllib.urlopen(url).read().decode(encoding='UTF-8')
regexLat = '"location".+\\n.+"lat".+?([0-9]{1}\.[0-9]{,14}).?\\n'
regexLon = '"location".+?\\n.+?\\n.+?"lng".+([0-9]{3}\.[0-9]{,14}).?\\n'
patternLat = re.compile(regexLat)
patternLon = re.compile(regexLon)
lat = re.findall(patternLat,textdecode)
lon = re.findall(regexLon,textdecode)
return lon, lat
def get_list():
result = [('Central Water Catchment', u'1.3551526', u'103.7972022'), ('Sembawang North', u'1.4491107', u'103.8184954'), ('Pei Chun', u'1.3362578', u'103.8556225'), ('Aljunied', u'1.316441', u'103.88298'), ('Braddell', u'1.3341351', u'103.8573237'), ('Teck Whye', u'1.3788984', u'103.7526191'), ('Taman Jurong', u'1.3540993', u'103.7283971'), ('Sembawang Straits', u'1.4491107', u'103.8184954'), ('Boulevard', u'1.345871', u'103.7508974'), ('Northshore', u'1.4649329', u'103.8263604'), ('Pasir Ris Wafer Fab Park', u'1.352083', u'103.819836'), ('Sentosa', u'1.2494041', u'103.8303209'), ('Peng Siang', u'1.3939687', u'103.7344961'), ('Woodlands East', u'1.4381922', u'103.7889597'), ('Telok Blangah Rise', u'1.2722317', u'103.8216119'), ('Gul Circle', u'1.3198713', u'103.6694122'), ('Yio Chu Kang West', u'1.381905', u'103.844818'), ('Senoko North', u'1.4576121', u'103.8037006'), ('Clarke Quay', u'1.2908825', u'103.8444432'), ('Bidadari', u'1.3410855', u'103.8752552'), ('Tampines East', u'1.3495907', u'103.9567879'), ('Kampong Bugis', u'1.3072491', u'103.8661877'), ('Jurong Island and Bukom', u'1.352083', u'103.819836'), ('Lorong 8 Toa Payoh', u'1.339772', u'103.8570052'), ('Fort Canning', u'1.3', u'103.85'), ('Tiong Bahru', u'1.286081', u'103.826958'), ('Clementi West', u'1.3161811', u'103.7649377'), ('Trafalgar', u'1.352083', u'103.819836'), ('Siglap', u'1.3138672', u'103.923031'), ('Kovan', u'1.3600696', u'103.8853633'), ('Changi Bay', u'1.3219708', u'104.0290022'), ('Toh Tuck', u'1.3397003', u'103.7661705'), ('Paya Lebar East', u'1.3180585', u'103.8930085'), ('Simpang North', u'1.4372222', u'103.8352778'), ('Semakau', u'1.2064868', u'103.7709177'), ('One North', u'1.2991731', u'103.7870016'), ('Bukit Batok West', u'1.3590288', u'103.7636796'), ('Singapore Polytechnic', u'1.3078453', u'103.778416'), ('Shipyard', u'1.3036871', u'103.6903012'), ('Macpherson', u'1.326745', u'103.889867'), ('Woodlands South', u'1.4381922', u'103.7889597'), ('Anchorvale', u'1.3974992', u'103.8868523'), ('Phillip', u'1.2842184', u'103.8496499'), ('Faber', u'1.3188254', u'103.7587828'), ('Chinatown', u'1.2838955', u'103.8434637'), ('Hougang Central', u'1.3712274', u'103.8917513'), ('Gombak', u'1.3707004', u'103.761899'), ('North Coast', u'1.4564122', u'103.793187'), ('Reservoir View', u'1.3368341', u'103.9365515'), ('Clementi Woods', u'1.3018597', u'103.7661558'), ('Fajar', u'1.384729', u'103.77108'), ('Woodleigh', u'1.33935', u'103.87079'), ('Wenya', u'1.3562394', u'103.7071579'), ('Admiralty', u'1.44069', u'103.800934'), ('Matilda', u'1.352083', u'103.819836'), ('Boon Teck', u'1.3269668', u'103.8501196'), ('Bukit Batok East', u'1.3590288', u'103.7636796'), ('Turf Club', u'1.4177268', u'103.7608558'), ('Marina East', u'1.2912788', u'103.8709039'), ('Ulu Pandan', u'1.3205188', u'103.7750533'), ('Kranji', u'1.4252979', u'103.7621389'), ('Senoko South', u'1.4576121', u'103.8037006'), ('Robertson Quay', u'1.2908317', u'103.8380232'), ('Jurong Port', u'1.3132692', u'103.7193335'), ('Singapore General Hospital', u'1.352083', u'103.819836'), ('Northland', u'1.4364908', u'103.8421208'), ('Clementi North', u'1.3161811', u'103.7649377'), ('Springleaf', u'1.3996715', u'103.8213053'), ('Frankel', u'1.31656', u'103.9178494'), ('Yishun West', u'1.430368', u'103.8353628'), ('Gul Basin', u'1.352083', u'103.819836'), ('Kaki Bukit', u'1.3362352', u'103.9071892'), ('Pulau Seletar', u'1.4431127', u'103.8614817'), ('Kim Keat', u'1.3308209', u'103.8550567'), ('Katong', u'1.307786', u'103.8954833'), ('Shangri-La', u'1.3697866', u'103.8330503'), ('Midview', u'1.3398425', u'103.7568922'), ('Changi Airport', u'1.3644202', u'103.9915308'), ('Toa Payoh West', u'1.3352492', u'103.8413198'), ('Teban Gardens', u'1.3220618', u'103.7409427'), ('Bencoolen', u'1.3000848', u'103.8513221'), ('Fernvale', u'1.391935', u'103.876142'), ('Tagore', u'1.3868248', u'103.8263898'), ('Upper Paya Lebar', u'1.3433664', u'103.8827229'), ('Kent Ridge', u'1.293383', u'103.784394'), ('Changi Point', u'1.3183581', u'103.9088538'), ('Clifford Pier', u'1.352083', u'103.819836'), ('Selegie', u'1.302442', u'103.8500272'), ('Ridout', u'1.3039677', u'103.8036033'), ('Anson', u'1.2735406', u'103.8447021'), ('Ghim Moh', u'1.3118053', u'103.7876508'), ('Serangoon Garden', u'1.3636674', u'103.8670088'), ('Penjuru Crescent', u'1.3135244', u'103.7348443'), ('Bedok Reservoir', u'1.3413025', u'103.9245499'), ('Bishan East', u'1.350772', u'103.848183'), ('Serangoon North Ind Estate', u'1.352083', u'103.819836'), ('Pasir Ris Park', u'1.3736043', u'103.9522005'), ('Potong Pasir', u'1.3313554', u'103.8692779'), ('Serangoon North', u'1.3553567', u'103.8678708'), ('Mount Emily', u'1.3038889', u'103.8483333'), ('Toa Payoh Central', u'1.3341389', u'103.8491111'), ('Mackenzie', u'1.3058375', u'103.8481243'), ('Jurong River', u'1.3252038', u'103.709397'), ('Yuhua West', u'1.3473374', u'103.7231677'), ('Hougang East', u'1.3612182', u'103.8862529'), ('Choa Chu Kang North', u'1.3839803', u'103.7469611'), ('Seletar Hills', u'1.3852199', u'103.8749121'), ('Joo Koon', u'1.3277399', u'103.6784799'), ('Pasir Panjang 1', u'1.2717449', u'103.802537'), ('Tiong Bahru Station', u'1.286081', u'103.826958'), ('Mandai West', u'1.4260074', u'103.8241046'), ('Punggol Field', u'1.3998565', u'103.9050115'), ("Monk's Hill", u'1.3096113', u'103.8403681'), ('Seletar', u'1.405163', u'103.8663418'), ('Brickworks', u'1.352083', u'103.819836'), ('Hong Kah', u'1.3559341', u'103.7179451'), ('Paya Lebar North', u'1.3333333', u'103.8833333'), ('Sembawang Springs', u'1.4345248', u'103.8233396'), ('Tengah', u'1.3555189', u'103.7308145'), ('Tuas View', u'1.2999658', u'103.6282123'), ('Clementi Central', u'1.3303775', u'103.776821'), ('Mandai Estate', u'1.407043', u'103.7586567'), ("National University of S'pore", u'1.352083', u'103.819836'), ('Bugis', u'1.3010601', u'103.85599'), ('Bayfront Subzone', u'1.352083', u'103.819836'), ('Simpang South', u'1.4372222', u'103.8352778'), ('Institution Hill', u'1.2951611', u'103.8397034'), ('Choa Chu Kang Central', u'1.3812993', u'103.7463562'), ('Nee Soon', u'1.4007817', u'103.821278'), ('Kampong Java', u'1.312959', u'103.843557'), ('Redhill', u'1.289732', u'103.81675'), ('Mountbatten', u'1.306306', u'103.882531'), ('Cairnhill', u'1.3069018', u'103.8385608'), ('Commonwealth', u'1.302559', u'103.798225'), ('Farrer Park', u'1.3124127', u'103.8547'), ('Sennett', u'1.3188413', u'103.9386818'), ('Tuas View Extension', u'1.339866', u'103.6384243'), ('Goodwood Park', u'1.312144', u'103.8335345'), ('Kampong Ubi', u'1.352083', u'103.819836'), ('Geylang Bahru', u'1.320865', u'103.8712434'), ('Senja', u'1.3832057', u'103.762161'), ("Pearl's Hill", u'1.2838889', u'103.84'), ('Joo Seng', u'1.3360248', u'103.8816405'), ('Jelebu', u'1.3797961', u'103.7638324'), ('Cecil', u'1.2801967', u'103.8486153'), ('Pulau Punggol Timor', u'1.4188754', u'103.8897501'), ('Nature Reserve', u'1.3784372', u'103.7758416'), ('Gali Batu', u'1.3904942', u'103.7716123'), ('Lorong Halus North', u'1.3864441', u'103.9232692'), ('Coney Island', u'1.3553567', u'103.8678708'), ('West Coast', u'1.31047', u'103.756828'), ('Flora Drive', u'1.3571327', u'103.965079'), ('Pandan', u'1.3116886', u'103.7533131'), ('Saujana', u'1.3822138', u'103.7671487'), ('Coronation Road', u'1.321882', u'103.8053033'), ('Jurong West Central', u'1.3380754', u'103.7411914'), ('Bedok North', u'1.3236038', u'103.9273405'), ('Tai Seng', u'1.3376999', u'103.890659'), ('Tanjong Pagar', u'1.2766708', u'103.8465905'), ('Mount Pleasant', u'1.3328291', u'103.8324641'), ('Yishun South', u'1.430368', u'103.8353628'), ('Mandai East', u'1.4260074', u'103.8241046'), ('Pasir Panjang 2', u'1.2800529', u'103.7858126'), ('Pasir Ris Drive', u'1.3828573', u'103.9454061'), ('International Business Park', u'1.3274096', u'103.7477861'), ('Nassim', u'1.311655', u'103.821796'), ('Safti', u'1.3301791', u'103.6818154'), ('Kian Teck', u'1.3305916', u'103.6934318'), ('Victoria', u'1.3005032', u'103.8558723'), ('Loyang West', u'1.3737035', u'103.9745443'), ('Simei', u'1.3399194', u'103.949471'), ('Sengkang West', u'1.3868121', u'103.8914433'), ('Anak Bukit', u'1.3418387', u'103.7776653'), ('Marina East (MP)', u'1.352083', u'103.819836'), ('Pulau Punggol Barat', u'1.4271702', u'103.8767024'), ('Bangkit', u'1.380382', u'103.772636'), ('Kallang Way', u'1.3229591', u'103.878167'), ('Southern Group', u'1.352083', u'103.819836'), ('Marina Centre', u'1.2785587', u'103.8656807'), ('Loyang East', u'1.3737035', u'103.9745443'), ('Yunnan', u'1.338575', u'103.692206'), ('Tuas Bay', u'1.3169027', u'103.6312522'), ('Queensway', u'1.2979925', u'103.8005407'), ('Seletar Aerospace Park', u'1.4054053', u'103.8662685'), ('Kembangan', u'1.320982', u'103.912842'), ('Kebun Bahru', u'1.3743012', u'103.8402669'), ('Sudong', u'1.207916', u'103.7202285'), ('Plab', u'1.352083', u'103.819836'), ('Changi West', u'1.3450101', u'103.9832089'), ('Bedok South', u'1.3236038', u'103.9273405'), ('Sembawang Central', u'1.4212451', u'103.8248881'), ('Bras Basah', u'1.2975068', u'103.8504356'), ('Tampines North', u'1.3495907', u'103.9567879'), ('Chatsworth', u'1.2997921', u'103.8212674'), ('Tanglin Halt', u'1.302261', u'103.7972067'), ('Senoko West', u'1.4576121', u'103.8037006'), ('The Wharves', u'1.276161', u'103.7904907'), ('Bukit Merah', u'1.2819046', u'103.8239182'), ('North-Eastern Islands', u'1.4063775', u'104.0323021'), ('Benoi Sector', u'1.3195872', u'103.6867912'), ('Maritime Square', u'1.2642124', u'103.8191268'), ('Holland Road', u'1.3130045', u'103.797956'), ('Yio Chu Kang East', u'1.381905', u'103.844818'), ('Yishun Central', u'1.4263506', u'103.8404896'), ('Geylang East', u'1.3200544', u'103.8917746'), ('Lorong Ah Soo', u'1.3514242', u'103.8848967'), ('Margaret Drive', u'1.2968341', u'103.8084694'), ('Khatib', u'1.4173658', u'103.8329446'), ('Dhoby Ghaut', u'1.2985098', u'103.8459595'), ('Tanjong Rhu', u'1.2975', u'103.8663889'), ('Dairy Farm', u'1.364674', u'103.772895'), ('Kampong Glam', u'1.3029735', u'103.859875'), ('Serangoon Central', u'1.3510491', u'103.8730597'), ('Mei Chin', u'1.2933829', u'103.802453'), ('Leonie Hill', u'1.296791', u'103.833203'), ('Newton Circus', u'1.312903', u'103.8395668'), ('Pang Sua', u'1.3933358', u'103.7525869'), ('Farrer Court', u'1.3174935', u'103.8077246'), ('Swiss Club', u'1.3382923', u'103.790064'), ('Samulun', u'1.3047395', u'103.6970037'), ('Straits View', u'1.2728954', u'103.8627293'), ('Marymount', u'1.349078', u'103.839492'), ('Yishun East', u'1.430368', u'103.8353628'), ('Little India', u'1.3063483', u'103.8496785'), ('Telok Blangah Way', u'1.2755998', u'103.8195017'), ('Sunset Way', u'1.3279073', u'103.7668691'), ('Boon Keng', u'1.3193154', u'103.8620583'), ('Bayshore', u'1.3118799', u'103.9381523'), ('Oxley', u'1.2977986', u'103.842695'), ('Holland Drive', u'1.3100006', u'103.7928594'), ('Sungei Road', u'1.3036692', u'103.8541646'), ('Boat Quay', u'1.2878119', u'103.8494162'), ('Ang Mo Kio Town Centre', u'1.3691149', u'103.8454342'), ('Hong Kah North', u'1.3559341', u'103.7179451'), ('Tyersall', u'1.3096237', u'103.8129643'), ('Depot Road', u'1.2815449', u'103.8097816'), ('Townsville', u'1.3594214', u'103.8551458'), ('Woodgrove', u'1.4276741', u'103.7806913'), ('Defu Industrial Park', u'1.352083', u'103.819836'), ('Guilin', u'1.3560181', u'103.7533996'), ('Paya Lebar West', u'1.3180585', u'103.8930085'), ('Orange Grove', u'1.3106145', u'103.8259732'), ('Liu Fang', u'1.3141514', u'103.6981553'), ('China Square', u'1.2838407', u'103.8479038'), ('Lavender', u'1.3072265', u'103.862951'), ('Hougang West', u'1.3612182', u'103.8862529'), ('Waterway East', u'1.4110707', u'103.9045385'), ('Woodlands West', u'1.4381922', u'103.7889597'), ('Paterson', u'1.302229', u'103.831282'), ('City Hall', u'1.2930339', u'103.851963'), ('Punggol Town Centre', u'1.3984457', u'103.9072046'), ('Upper Thomson', u'1.3729333', u'103.8285086'), ('East Coast', u'1.3050191', u'103.9225352'), ('Crawford', u'1.3062872', u'103.8649646'), ('Keat Hong', u'1.378686', u'103.748947'), ('Tuas Promenade', u'1.316521', u'103.657727'), ('Yio Chu Kang North', u'1.381905', u'103.844818'), ('Western Water Catchment', u'1.3471977', u'103.682541'), ('City Terminals', u'1.352083', u'103.819836'), ('Tanglin', u'1.306932', u'103.8188845'), ('Lower Seletar', u'1.3989209', u'103.8336524'), ('Yio Chu Kang', u'1.381905', u'103.844818'), ('Leedon Park', u'1.3163348', u'103.7960521'), ("People's Park", u'1.2841977', u'103.8425488'), ('Everton Park', u'1.2770868', u'103.8399398'), ('Rochor Canal', u'1.3034293', u'103.8555718'), ('Sembawang Hills', u'1.3759482', u'103.8304284'), ('Sembawang East', u'1.4491107', u'103.8184954'), ('Lorong Halus', u'1.3864441', u'103.9232692'), ('Moulmein', u'1.3190874', u'103.8485005'), ('Punggol Canal', u'1.352083', u'103.819836'), ('Balestier', u'1.323004', u'103.8523054'), ('Tampines West', u'1.3495907', u'103.9567879'), ('Pasir Ris West', u'1.3730057', u'103.9492453'), ('Telok Blangah Drive', u'1.2728757', u'103.8086903'), ('One Tree Hill', u'1.3015574', u'103.8261541'), ('Pioneer Sector', u'1.2948882', u'103.6662816'), ('Toh Guan', u'1.3372143', u'103.7475242'), ('Istana Negara', u'1.352083', u'103.819836'), ('Bukit Batok South', u'1.3590288', u'103.7636796'), ('Jurong Gateway', u'1.3355346', u'103.7418606'), ('Sengkang Town Centre', u'1.3868121', u'103.8914433'), ('Bendemeer', u'1.3201284', u'103.8649128'), ('Lim Chu Kang', u'1.4304941', u'103.7173325'), ('Alexandra North', u'1.2882934', u'103.8054785'), ('Compassvale', u'1.394564', u'103.900156'), ('Chong Boon', u'1.3153584', u'103.788311'), ('Chin Bee', u'1.3361398', u'103.7141127'), ('Raffles Place', u'1.283933', u'103.8514545'), ('Somerset', u'1.3004708', u'103.8386926'), ('Marine Parade', u'1.3019687', u'103.8970821'), ('Tengeh', u'1.3555189', u'103.7308145'), ('Kangkar', u'1.3837389', u'103.9021944'), ('Cheng San', u'1.352083', u'103.819836'), ('Yuhua East', u'1.3401636', u'103.7363464'), ('Dover', u'1.311322', u'103.7786532'), ('Yew Tee', u'1.3975669', u'103.7473389'), ('Central Subzone', u'1.352083', u'103.819836'), ('Bukit Batok Central', u'1.3502895', u'103.749804'), ('Tanjong Irau', u'1.352083', u'103.819836'), ('Dunearn', u'1.3257813', u'103.8085672'), ('Rivervale', u'1.3876353', u'103.9049359'), ('Woodlands Regional Centre', u'1.352083', u'103.819836'), ('Xilin', u'1.3293908', u'103.9632964'), ('Lakeside', u'1.3442374', u'103.7207253'), ('Maxwell', u'1.2763215', u'103.8514872'), ('Malcolm', u'1.3236673', u'103.8311461'), ('Henderson Hill', u'1.2807296', u'103.81783'), ('Airport Road', u'1.343296', u'103.8989023'), ('Alexandra Hill', u'1.2882934', u'103.8054785'), ('Bukit Ho Swee', u'1.2874125', u'103.8297425'), ('Tukang', u'1.326699', u'103.7148166'), ('Hillcrest', u'1.3318642', u'103.8078398'), ('Greenwood Park', u'1.334482', u'103.8051526'), ('Marina South', u'1.279294', u'103.8701686'), ('Tuas North', u'1.2949472', u'103.6304833'), ('Boon Lay Place', u'1.3468707', u'103.7129422'), ('Lorong Chuan', u'1.3516608', u'103.8629139'), ('Pasir Ris Central', u'1.3721378', u'103.9493096'), ('Hillview', u'1.362472', u'103.767389'), ('Kampong Tiong Bahru', u'1.352083', u'103.819836'), ('Port', u'1.2670651', u'103.8038239'), ('Kallang Bahru', u'1.3179742', u'103.8665875')]
return result
'''
townlist = ["punggol", "sengkang east", "geylang bahru"]
newlist = scrap_coordinates(townlist)
print newlist
'''