/
update_tenhou_yakuman.py
134 lines (105 loc) · 4.79 KB
/
update_tenhou_yakuman.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
import datetime
import json
import os
import pytz
import requests
from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils import timezone
from player.tenhou.models import CollectedYakuman, TenhouNickname
def get_date_string():
return timezone.now().strftime("%H:%M:%S")
class Command(BaseCommand):
def handle(self, *args, **options):
print("{0}: Start".format(get_date_string()))
tenhou_objects = TenhouNickname.objects.all().prefetch_related("player")
player_profiles = {}
for tenhou_object in tenhou_objects:
player_profiles[tenhou_object.tenhou_username] = tenhou_object
with transaction.atomic():
self.old_records(player_profiles)
self.current_month_data(player_profiles)
print("{0}: End".format(get_date_string()))
def current_month_data(self, player_profiles):
url = "http://tenhou.net/sc/ykm.js"
print(url)
current_date = datetime.datetime.now().replace(tzinfo=pytz.timezone("Asia/Tokyo"))
current_year = current_date.year
data = requests.get(url).content.decode("utf-8")
self.parse_and_create_records(player_profiles, data, current_year)
def old_records(self, player_profiles):
"""
Download historical data
"""
current_date = datetime.datetime.now().replace(tzinfo=pytz.timezone("Asia/Tokyo"))
current_year = current_date.year
# 2006 - 2009 years have old format
# we need to add additional support for them
start_year = 2009
stop_year = current_year
months = ["{:02}".format(x) for x in range(1, 13)]
folder = os.path.join("/tmp", "yakuman")
if not os.path.exists(folder):
os.mkdir(folder)
for year in range(start_year, stop_year + 1):
# we don't need to load data from the future
if current_year == year:
months = ["{:02}".format(x) for x in range(1, current_date.month)]
for month in months:
print(year, month)
file_path = os.path.join(folder, "{}-{}.json".format(year, month))
data = None
# load from cache
if os.path.exists(file_path):
with open(file_path, "r") as f:
data = f.read()
else:
url = "http://tenhou.net/sc/{}/{}/ykm.js".format(year, month)
response = requests.get(url)
if response.status_code == 200:
data = response.content.decode("utf-8")
# store to cache
with open(file_path, "w") as f:
f.write(data)
if not data:
print("Missed data")
continue
self.parse_and_create_records(player_profiles, data, year)
def parse_and_create_records(self, player_profiles, yakuman_data, year):
# tenhou returns it not in really handy format
# new format
if "\r\n" in yakuman_data:
yakuman_data = yakuman_data.split("\r\n")[2]
yakuman_data = json.loads(yakuman_data[4:-1].replace('"', '\\"').replace("'", '"'))
# old format
else:
yakuman_data = yakuman_data.split(";\n")[2].strip()
yakuman_data = yakuman_data[4:].replace('"', '\\"').replace("'", '"').replace("\n", "")
yakuman_data = json.loads(yakuman_data)
filtered_results = []
for x in range(0, len(yakuman_data), 5):
yakuman_date = yakuman_data[x]
name = yakuman_data[x + 1]
yakuman_list = yakuman_data[x + 3]
log = yakuman_data[x + 4]
if name in player_profiles:
date = "{} {}".format(year, yakuman_date)
date = datetime.datetime.strptime(date, "%Y %m/%d %H:%M")
date = date.replace(tzinfo=pytz.timezone("Asia/Tokyo"))
tenhou_object = player_profiles[name]
# let's add only yakumans related to the current profile
if date.date() >= tenhou_object.username_created_at:
filtered_results.append(
{
"tenhou_object": tenhou_object,
"date": date,
"yakuman_list": ",".join([str(x) for x in yakuman_list]),
"log_id": log,
}
)
for item in filtered_results:
exists = (
CollectedYakuman.objects.filter(tenhou_object=item["tenhou_object"]).filter(date=item["date"])
).exists()
if not exists:
CollectedYakuman.objects.create(**item)