-
-
Notifications
You must be signed in to change notification settings - Fork 0
/
reindex.py
122 lines (111 loc) · 3.63 KB
/
reindex.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
from steem import Steem
from pymongo import MongoClient
from pprint import pprint
import json
import sys
import os
# load config from json file
print('Reading config.json file')
with open('../config.json') as json_config_file:
config = json.load(json_config_file)
print(config)
ns = os.environ['namespace'] if 'namespace' in os.environ else 'btstalk'
s = Steem(config['steemd_nodes'])
mongo = MongoClient(config['mongo_url'])
db = mongo[ns]
# db = mongo.forums
data = json.loads(sys.argv[1])
def update_forum(data):
update = {
'$set': data,
'$unset': {
'children': 1
}
}
if 'parent' in data:
parent = db.forums.find_one({'_id': data['parent']})
update['$set'].update({
'parent_name': parent['name']
})
else:
data.pop('parent', None)
data.pop('parent_name', None)
update['$unset'].update({
'parent': True,
'parent_name': True,
})
query = {
'_id': data['_id']
}
results = db.forums.update(query, update, upsert=True)
print(results)
if results['n'] == 1 and results['updatedExisting'] == False:
pprint("[FORUM][REINDEXER] - Inserting new forum [" + data['_id'] + "]")
if results['n'] == 1 and results['updatedExisting'] == True:
pprint("[FORUM][REINDEXER] - Updating forum [" + data['_id'] + "]")
def update_posts(data):
query = {}
if 'tags' in data and len(data['tags']) > 0:
query.update({'category': {'$in': data['tags']}})
if 'accounts' in data and len(data['accounts']) > 0:
query.update({'author': {'$in': data['accounts']}})
sort = [("last_reply",-1),("created",-1)]
results = db.posts.find(query).sort(sort).limit(1)
for comment in results:
query = {
'_id': data['_id'],
}
updates = {
'updated': comment['created'],
'last_post': {
'created': comment['created'],
'author': comment['author'],
'title': comment['title'],
'url': comment['url']
}
}
pprint("[FORUM][REINDEXER] - Updating latest post to [" + str(comment['_id']) + "]...")
response = db.forums.update(query, {'$set': updates}, upsert=True)
def update_replies(data):
query = {}
if 'tags' in data and len(data['tags']) > 0:
query.update({'category': {'$in': data['tags']}})
if 'accounts' in data and len(data['accounts']) > 0:
query.update({'author': {'$in': data['accounts']}})
sort = [("last_reply",-1),("created",-1)]
results = db.replies.find(query).sort(sort).limit(1)
for comment in results:
query = {
'_id': data['_id'],
}
updates = {
'updated': comment['created'],
'last_reply': {
'created': comment['created'],
'author': comment['author'],
'title': comment['root_title'],
'url': comment['url']
}
}
pprint("[FORUM][REINDEXER] - Updating latest reply to [" + str(comment['_id']) + "]...")
db.forums.update(query, {'$set': updates}, upsert=True)
def update_parent(data):
db.forums.update({
'_id': data['parent'],
'children._id': {'$ne': data['_id']}
}, {
'$addToSet': {
'children': {
'_id': data['_id'],
'name': data['name']
}
}
})
if __name__ == '__main__':
pprint("[FORUM][REINDEXER] - Starting script...")
#sys.stdout.flush()
update_forum(data)
update_posts(data)
update_replies(data)
if 'parent' in data:
update_parent(data)