Skip to content


Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
tree: 70d940cdad
Fetching contributors…

Cannot retrieve contributors at this time

99 lines (78 sloc) 2.709 kb
import datetime
import argparse
import os
import time
import ConfigParser
import primitive_util
import pymongo
def get_mongo_connection():
mongo_connection = 'localhost'
config = ConfigParser.ConfigParser()'conf.ini')
mongo_connection = config.get('mongo', 'connection')
#FIXME: this is too broad
return pymongo.Connection(mongo_connection)
# Should I read once somewhere and cache? I guess when
# we have more config things.
def get_mongo_database():
connection = get_mongo_connection()
db = None
config = ConfigParser.ConfigParser()'conf.ini')
db = connection[config.get('mongo', 'database')]
db = connection['test']
return db
def read_object_from_db(obj, collection, _id):
prim = collection.find_one({'_id': _id})
if prim:
def write_object_to_db(obj, collection, _id):
prim = primitive_util.to_primitive(obj)
prim['_id'] = _id
def ensure_exists(path):
if not os.path.exists(path):
def at_least_as_big_as(path, min_file_size):
if not os.path.exists(path):
return False
return os.stat(path).st_size >= min_file_size
def daterange(start_date, end_date):
for n in range((end_date - start_date).days):
yield start_date + datetime.timedelta(n)
def incremental_parser():
parser = argparse.ArgumentParser()
parser.add_argument('--noincremental', action='store_false',
return parser
def incremental_max_parser():
parser = incremental_parser()
parser.add_argument('--max_games', default=-1, type=int)
return parser
def incremental_date_range_cmd_line_parser():
parser = incremental_parser()
# 20101015 is the first day with standard turn labels
parser.add_argument('--startdate', default='20101015')
parser.add_argument('--enddate', default='99999999')
return parser
def includes_day(args, str_yyyymmdd):
assert len(str_yyyymmdd) == 8, '%s not 8 chars' % str_yyyymmdd
return args.startdate <= str_yyyymmdd <= args.enddate
def progress_meter(iterable, chunksize=1000):
""" Prints progress through iterable at chunksize intervals."""
scan_start = time.time()
since_last = time.time()
for idx, val in enumerate(iterable):
if idx % chunksize == 0 and idx > 0:
print idx
print 'avg rate', idx / (time.time() - scan_start)
print 'inst rate', chunksize / (time.time() - since_last)
since_last = time.time()
yield val
Jump to Line
Something went wrong with that request. Please try again.