-
Notifications
You must be signed in to change notification settings - Fork 0
/
first_step.py
139 lines (114 loc) · 7.94 KB
/
first_step.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
from __future__ import division
import threading
import os.path
import json
from classes import Game, Player, Misc, Database, API_KEY, GAMES_FOLDERS_PATH, MAX_THREAD_NUM, DATABASE_DETAILS, STATIC_DATA_PATH, Static
from second_step_class import Second_step
from third_step import Third_step
from select import select
import sys
import time
class Data_collector (threading.Thread):
def __init__(self, threadID, lock, region):
threading.Thread.__init__(self)
self.threadID = threadID
self.player_region = region
self.lock = lock
def run(self):
database=Database(DATABASE_DETAILS)
aram_files_path = GAMES_FOLDERS_PATH + self.player_region
num_games_to_get = 99
time_check = time.time()
checking_period = 3600
keep_running = True
m = Misc()
player_count = 0
checked_players_games = dict()
second_step = Second_step(self.player_region)
player_population = database.get_row_count(self.player_region + "_players_checked")
processing_unit = player_population/MAX_THREAD_NUM
start_position = int(round(processing_unit * (self.threadID%MAX_THREAD_NUM)))
end_position = int(round(start_position + processing_unit - 1))
if self.threadID%MAX_THREAD_NUM == MAX_THREAD_NUM-1:
end_position = player_population
while True:
for one_player in list(database.get_all_items(self.player_region + "_players_checked", "id"))[start_position: end_position]:
self.lock[self.threadID].acquire()
if keep_running == True:
patch_date = Static(API_KEY).get_patch_date()
player_id = str(one_player)
#print player_id
player = Player( self.player_region,API_KEY,account_id=player_id )
# Check if player already exists in players table. If not, add them.
## m.logging(self.player_region, "Checking player: " + str(player_id), "log")
database.insert_items(self.player_region + "_summoners", "id", player_id)
# Get the timestamp of the last checked game
current_last_game_epoch = database.get_database_item(self.player_region + "_summoners", "id", player_id, "last_game_epoch")
if current_last_game_epoch < patch_date:
current_last_game_epoch = patch_date
recent_games = player.get_games(current_last_game_epoch, count=num_games_to_get)
if player.total_games == 0:
self.lock[self.threadID].release()
continue
if recent_games == -2:
m.logging(self.player_region, "Removing " + str(player_id) + " from " + self.player_region + "_summoners and " + self.player_region + "_players_checked", "log")
database.delete_line(self.player_region + "_summoners", "id", player_id)
database.delete_line(self.player_region + "_players_checked", "id", player_id)
self.lock[self.threadID].release()
continue
database.update_numberof_games(self.player_region + "_summoners", "id", player_id, "total_games", player.total_games)
# If the player has played new games since the last check, update the overall number of games and the time of the last played game.
if not current_last_game_epoch == player.date_last_game:
database.update_fields(self.player_region + "_summoners", "id", player_id, {"last_game_epoch": player.date_last_game})
## m.logging(self.player_region, "Player " + str(player_id) + " played new games since last check", "log" )
## else:
## m.logging(self.player_region, "Player " + str(player_id) + " Didn't play any new games since last check", "log")
# If the games are ARAM games, update the table with new number of aram games and the new percentage of played aram games.
if player.aram_games is not 0:
database.update_numberof_games(self.player_region + "_summoners", "id", player_id, "aram_games", player.aram_games)
#if not database.get_database_item(self.player_region + "_summoners", "id", player_id, "total_games") == 0:
aram_percentage = (float(database.get_database_item(self.player_region + "_summoners", "id", player_id,"aram_games")) / float(database.get_database_item(self.player_region + "_summoners", "id", player_id, "total_games"))) * 100
database.update_fields(self.player_region + "_summoners", "id", player_id, {"aram_games_percentage": aram_percentage})
# If new ARAM games were played since last check, update the games file for this player with the new games data
games = []
if not player.aram_games == 0:
## m.logging(self.player_region, "Updating " + player_id + "'s file with recent ARAM games", "log" )
player_file = os.path.join(aram_files_path, player_id)
with open(player_file, 'a') as player_history:
for game_id in recent_games:
if database.insert_items(self.player_region + "_games", "id, summoner, date", game_id + " , " + player_id + " , " + recent_games[game_id]) == 1:
games.append(game_id)
game = Game(game_id, self.player_region, API_KEY)
try:
json.dump(game.game_details(), player_history)
player_history.write("\n")
except ValueError:
m.logging(self.player_region, "Error while saving games " + game_id + " for " + player_id, "error")
self.lock[self.threadID].release()
continue
## else:
## m.logging(self.player_region, player_id + " did not play any ARAM games recently", "log")
m.logging(self.player_region, self.player_region + "- Thread: " + str(self.threadID) + ", Player: " + str(player_id) +
", All: " + str(player.total_games) + ", ARAM: " + str(player.aram_games), "log")
if games:
checked_players_games[player_id] = games
player_count += 1
self.lock[self.threadID].release()
if player_count % 2 == 0:
if checked_players_games:
self.lock[self.threadID].acquire()
second_step.update_tables(checked_players_games)
self.lock[self.threadID].release()
checked_players_games = dict()
if time.time() - time_check >= checking_period:
time_check = time.time()
with open(STATIC_DATA_PATH+"End_Exec", "r") as end_check:
for line in end_check.readlines():
if line.strip() == "True":
m.logging(self.player_region, str(self.threadID) + ": End of execution was requested. This thread will exit now", "log")
print str(self.threadID) + ": End of execution was requested. This thread will now exit."
keep_running = False
else:
database.close_db()
return
m.logging(self.player_region, "End of loop reached. Restarting..", "log")