-
Notifications
You must be signed in to change notification settings - Fork 1
/
onetime_feeds_datetime.py
53 lines (46 loc) · 1.63 KB
/
onetime_feeds_datetime.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
"""Run this script to convert all existing feeds to use datetime objects instead of seconds since 1970."""
import db
from datetime import datetime
def fixArticleIntStamps():
query = db.qdoc.find({'$or': [
{'timestamp': {'$type': 'int'}},
{'timestamp': {'$type': 'double'}},
]})
for article in query:
stamp = article.get('timestamp', 0)
date = datetime.utcfromtimestamp(stamp)
db.qdoc.update({'_id': article['_id']}, {'$set': {'timestamp': date}})
def fixIntStamps():
bulkUpdate = db.feed.initialize_unordered_bulk_op()
query = db.feed.find({'$or': [
{'stamp': {'$type': 'int'}},
{'stamp': {'$type': 'double'}},
]})
shouldExecute = False
for feed in query:
shouldExecute = True
stamp = feed.get('stamp', 0)
date = datetime.utcfromtimestamp(stamp)
bulkUpdate.find({'stamp': stamp}).update({'$set': {'stamp': date}})
if shouldExecute:
bulkUpdate.execute()
def fixIntLastCrawl():
bulkUpdate = db.feed.initialize_unordered_bulk_op()
query = db.feed.find({'$or': [
{'lastCrawl': {'$type': 'int'}},
{'lastCrawl': {'$type': 'double'}},
]})
shouldExecute = False
for feed in query:
shouldExecute = True
crawl = feed.get('lastCrawl', 0)
date = datetime.utcfromtimestamp(crawl)
bulkUpdate.find({'lastCrawl': crawl}).update({'$set': {'lastCrawl': date}})
if shouldExecute:
bulkUpdate.execute()
print 'Fixing feed timestamps.'
fixIntStamps()
print 'Fixing feed lastcrawl timestamps.'
fixIntLastCrawl()
print 'Fixing article timestamps.'
fixArticleIntStamps()