forked from mohd-akram/item.tf
-
Notifications
You must be signed in to change notification settings - Fork 0
/
updatestore.py
executable file
·140 lines (104 loc) · 4.36 KB
/
updatestore.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
#!/usr/bin/env python3
import sys
import time
import asyncio
from xml.dom.minidom import getDOMImplementation
from aioredis import create_redis
from slugify import slugify
import config
import tf2api
import tf2search
from store import Redis
from main import getitemkey, getclasskey, gettagkey
class Sitemap:
"""A class that is used to create XML sitemaps"""
def __init__(self):
impl = getDOMImplementation()
self.doc = impl.createDocument(None, 'urlset', None)
self.urlset = self.doc.documentElement
self.urlset.setAttribute('xmlns',
'http://www.sitemaps.org/schemas/sitemap/0.9')
def add(self, path):
"""Add a url to the sitemap"""
url = self.doc.createElement('url')
loc = url.appendChild(self.doc.createElement('loc'))
loc.appendChild(self.doc.createTextNode(path))
return self.urlset.appendChild(url)
def toxml(self):
"""Return a pretty XML version of the sitemap"""
return self.doc.toprettyxml()
async def main(flush):
store = await create_redis(('localhost', 6379), commands_factory=Redis)
tf2info = tf2search.gettf2info(config.apikey,
config.backpackkey, config.tradekey,
config.blueprintsfile)
if flush:
await store.delete('items')
await store.delete_all('items:*')
await store.delete_all('item:*')
suggestions = [[], [], []]
sitemap = Sitemap()
sitemap.add(config.homepage)
all_classes = [class_.lower() for class_ in tf2api.getallclasses()]
all_tags = list(tf2api.getalltags())
all_qualities = [quality.replace("'", '').lower() for quality in
tf2api.getallqualities().values()]
keywords = all_classes + all_tags + all_qualities
for keyword in keywords:
sitemap.add(f'{config.homepage}/search/{keyword}')
for class_tag in all_classes + all_tags:
for quality in all_qualities:
sitemap.add(f'{config.homepage}/search/{quality}-{class_tag}')
for class_ in all_classes:
for tag in all_tags:
sitemap.add(f'{config.homepage}/search/{class_}-{tag}')
for quality in all_qualities:
sitemap.add(
f'{config.homepage}/search/{quality}-{class_}-{tag}'
)
for index in tf2info.items:
pipe = store.pipeline()
itemdict = tf2search.createitemdict(index, tf2info)
name = itemdict['name']
pipe.hmset_dict(getitemkey(index), itemdict)
pipe.sadd('items', index)
classes = itemdict['classes']
tags = itemdict['tags']
if index == tf2info.itemsbyname[name]['defindex']:
slug = slugify(name)
pipe.hmset_dict('items:slugs', {slug: index})
if tf2search.isvalidresult(itemdict, False):
if not classes:
pipe.sadd(getclasskey(), index)
if len(classes) > 1:
pipe.sadd(getclasskey(multi=True), index)
if not tags:
pipe.sadd(gettagkey(), index)
for class_ in classes:
pipe.sadd(getclasskey(class_), index)
for tag in tags:
pipe.sadd(gettagkey(tag), index)
if tf2search.isvalidresult(itemdict):
pipe.sadd('items:indexes', index)
pipe.hmset_dict('items:names', {name: index})
path = f'{config.homepage}/{slug}'
suggestions[0].append(name)
suggestions[1].append('{} - {}'.format(
', '.join(itemdict['classes']),
', '.join(itemdict['tags'])))
suggestions[2].append(path)
sitemap.add(path)
await pipe.execute()
await store.delete('items:new')
for index in tf2info.newstoreprices:
await store.sadd('items:new', index)
bundles = {str(k): v for k, v in tf2info.bundles.items()}
data = {'items:sets': tf2info.itemsets,
'items:bundles': bundles,
'items:suggestions': suggestions,
'items:lastupdated': time.time(),
'sitemap': sitemap.toxml()}
await store.mset_dict(data)
if __name__ == '__main__':
flush = len(sys.argv) == 2 and sys.argv[1] == '-f'
asyncio.get_event_loop().run_until_complete(main(flush))