-
Notifications
You must be signed in to change notification settings - Fork 2
/
geonames_api_application.py
106 lines (94 loc) · 3.74 KB
/
geonames_api_application.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
import requests
import time
from database.database import Database
def make_search_request(query):
base_payload = {'username': 'Svenskjefel', 'order_by': 'relevance'}
base_url = "http://api.geonames.org/searchJSON"
base_payload['q'] = query
r = requests.get(base_url, base_payload)
return r.json()
#Not used
def split_on_common_characters(query):
result = []
if '/' in query:
result = query.split('/')
elif '|' in query:
result = query.split('|')
elif ';' in query:
result = query.split(';')
elif ':' in query:
result = query.split(':')
elif '-' in query:
result = query.split('-')
elif '.' in query:
result = query.split('.')
return result
def split_to_words(query):
if " " in query:
return query.split(" ")
listify = []
listify.append(query)
return listify
def lookup(db, location, throttle_count=0, no_result_count=0):
throttle_count += 1
if throttle_count == 1950:
time.sleep(3650)
user_id = location[0]
user_location = location[1]
preprocessed_data = db.select_preprocessed_data_from_user_id(user_id)
preprocessed_location = preprocessed_data[0]
preprocessed_rest = None
if len(preprocessed_data) > 1:
preprocessed_rest = preprocessed_data[1]
#Check given user location
query = user_location
result = make_search_request(query)
total_results = result['totalResultsCount']
if total_results > 0:
id = result['geonames'][0]['geonameId']
country_name = result['geonames'][0].get('countryName', None)
db.insert_into_identified_via_geonames(user_id, id, country_name)
else:
# Check our preprocessed locations
query = preprocessed_location
result = make_search_request(query)
if result['totalResultsCount'] > 0:
id = result['geonames'][0]['geonameId']
country_name = result['geonames'][0].get('countryName', None)
db.insert_into_identified_via_geonames(user_id, id, country_name)
else:
# Check our preprocessed preprocessed rest
if (preprocessed_rest):
query = preprocessed_rest
result = make_search_request(query)
if (result['totalResultsCount'] > 0):
id = result['geonames'][0]['geonameId']
country_name = result['geonames'][0].get('countryName', None)
db.insert_into_identified_via_geonames(user_id, id, country_name)
else:
#CHECKING EACH INDIVIDUAL WORD
list_words = split_to_words(preprocessed_location)
if (preprocessed_rest):
list_words += split_to_words(preprocessed_rest)
for word in list_words:
query = word
result = make_search_request(query)
if result['totalResultsCount'] > 0:
id = result['geonames'][0]['geonameId']
country_name = result['geonames'][0].get('countryName', None)
db.insert_into_identified_via_geonames(user_id, id, country_name)
break
else:
no_result_count += 1
def run_location_lookup_on_all_users():
db = Database("twitter-geo")
some_user_locations_with_ids = db.select_user_locations()
no_result_count = 0
for location in some_user_locations_with_ids:
throttle_count = 0
lookup(db, location, throttle_count, no_result_count)
def run_location_lookup_on_a_user(user_id):
db = Database("twitter-geo")
user_location = db.select_locations_based_on_user_id(user_id)
if len(user_location) > 0:
lookup(db, user_location[0])