-
Notifications
You must be signed in to change notification settings - Fork 5
/
bananas_server.py
165 lines (129 loc) · 6.12 KB
/
bananas_server.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
import asyncio
import logging
from collections import defaultdict
from concurrent import futures
from ..helpers.safe_filename import safe_filename
from ..openttd.protocol.enums import ContentType
log = logging.getLogger(__name__)
class Application:
def __init__(self, storage, index):
super().__init__()
self.storage = storage
self.index = index
self.protocol = None
self._by_content_id = None
self._by_content_type = None
self._by_unique_id = None
self._by_unique_id_and_md5sum = None
self._reload_busy = asyncio.Event()
self._reload_busy.set()
loop = asyncio.get_event_loop()
loop.run_until_complete(self.reload())
async def _send_content_entry(self, source, content_entry):
await source.protocol.send_PACKET_CONTENT_SERVER_INFO(
content_type=content_entry.content_type,
content_id=content_entry.content_id,
filesize=content_entry.filesize,
name=content_entry.name,
version=content_entry.version,
url=content_entry.url,
description=content_entry.description,
unique_id=content_entry.unique_id,
md5sum=content_entry.md5sum,
dependencies=content_entry.dependencies,
tags=content_entry.tags,
)
def get_by_content_id(self, content_id):
return self._by_content_id.get(content_id)
def get_by_unique_id(self, content_type, unique_id):
return self._by_unique_id[content_type].get(unique_id)
def get_by_unique_id_and_md5sum(self, content_type, unique_id, md5sum):
return self._by_unique_id_and_md5sum[content_type].get(unique_id, {}).get(md5sum)
async def receive_PACKET_CONTENT_CLIENT_INFO_LIST(self, source, content_type, openttd_version):
version_major = (openttd_version >> 28) & 0xF
version_minor = (openttd_version >> 24) & 0xF
version_patch = (openttd_version >> 20) & 0xF
version = [version_major, version_minor, version_patch]
for content_entry in self._by_content_type[content_type]:
if content_entry.min_version and version < content_entry.min_version:
continue
if content_entry.max_version and version >= content_entry.max_version:
continue
await self._send_content_entry(source, content_entry)
async def receive_PACKET_CONTENT_CLIENT_INFO_EXTID(self, source, content_infos):
for content_info in content_infos:
content_entry = self.get_by_unique_id(content_info.content_type, content_info.unique_id)
if content_entry:
await self._send_content_entry(source, content_entry)
async def receive_PACKET_CONTENT_CLIENT_INFO_EXTID_MD5(self, source, content_infos):
for content_info in content_infos:
content_entry = self.get_by_unique_id_and_md5sum(
content_info.content_type, content_info.unique_id, content_info.md5sum
)
if content_entry:
await self._send_content_entry(source, content_entry)
async def receive_PACKET_CONTENT_CLIENT_INFO_ID(self, source, content_infos):
for content_info in content_infos:
content_entry = self.get_by_content_id(content_info.content_id)
if content_entry:
await self._send_content_entry(source, content_entry)
async def receive_PACKET_CONTENT_CLIENT_CONTENT(self, source, content_infos):
for content_info in content_infos:
content_entry = self.get_by_content_id(content_info.content_id)
if not content_entry:
continue
try:
stream = self.storage.get_stream(content_entry)
except Exception:
log.exception("Error with storage, aborting for this client ...")
return
await source.protocol.send_PACKET_CONTENT_SERVER_CONTENT(
content_type=content_entry.content_type,
content_id=content_entry.content_id,
filesize=content_entry.filesize,
filename=safe_filename(content_entry),
stream=stream,
)
async def reload(self):
await self._reload_busy.wait()
self._reload_busy.clear()
try:
reload_helper = ReloadHelper(self.storage, self.index)
reload_helper.prepare()
# Run the reload in a new process, so we don't block the rest of the
# server while doing this job.
loop = asyncio.get_event_loop()
with futures.ProcessPoolExecutor(max_workers=1) as executor:
task = loop.run_in_executor(executor, reload_helper.reload)
(
self._by_content_id,
self._by_content_type,
self._by_unique_id,
self._by_unique_id_and_md5sum,
) = await task
finally:
self._reload_busy.set()
class ReloadHelper:
def __init__(self, storage, index):
self.storage = storage
self.index = index
def _get_md5sum_mapping(self):
log.info("Building md5sum mapping")
md5sum_mapping = defaultdict(lambda: defaultdict(dict))
for content_type in ContentType:
if content_type == ContentType.CONTENT_TYPE_END:
continue
for unique_id_str in self.storage.list_folder(content_type):
unique_id = bytes.fromhex(unique_id_str)
for filename in self.storage.list_folder(content_type, unique_id_str):
md5sum, _, _ = filename.partition(".")
md5sum_partial = bytes.fromhex(md5sum[0:8])
md5sum = bytes.fromhex(md5sum)
md5sum_mapping[content_type][unique_id][md5sum_partial] = md5sum
# defaultdict() cannot be pickled, so convert to a normal dict.
return {key: dict(value) for key, value in md5sum_mapping.items()}
def prepare(self):
self.storage.clear_cache()
def reload(self):
md5sum_mapping = self._get_md5sum_mapping()
return self.index.reload(md5sum_mapping)