Permalink
Browse files

refactored the openings code so that it can be run incrementally

  • Loading branch information...
1 parent c806c6c commit e45a8aba1f46c999e94d0c5a552af99e815f2f12 @rrenaud rrenaud committed Jun 13, 2011
Showing with 285 additions and 185 deletions.
  1. +8 −15 frontend.py
  2. +5 −0 incremental_scanner.py
  3. +9 −1 primitive_util.py
  4. +120 −91 run_trueskill.py
  5. +82 −45 trueskill/trueskill.py
  6. +61 −33 trueskill/trueskill_test.py
View
@@ -82,7 +82,6 @@ def make_level_key(floor, ceil):
def skill_str(mu, sigma):
return u'%3.3f ± %3.3f' % (mu, sigma*3)
-
class OpeningPage(object):
def GET(self):
web.header("Content-Type", "text/html; charset=utf-8")
@@ -93,25 +92,19 @@ def GET(self):
if 'card' in query_dict:
selected_card = query_dict['card']
+ results = db.trueskill_openings.find({'_id': {'$regex': '^open:'}})
+ openings = list(results)
+ card_list = card_info.OPENING_CARDS
if selected_card not in ('All cards', ''):
- query = db.trueskill_openings.find({'cards': selected_card})
- else:
- query = db.trueskill_openings.find({})
+ openings = [o for o in openings if selected_card in o['_id']]
- #offset = db.trueskill_openings.find_one({'name':
- # 'open:Silver+Silver'})['mu']
- offset = 0
- openings = list(query)
- card_list = card_info.OPENING_CARDS
for opening in openings:
- for stat in ('mu', 'floor', 'ceil'):
- opening[stat] -= offset
-
- floor = opening['floor']
- ceil = opening['ceil']
+ floor = opening['mu'] - opening['sigma'] * 3
+ ceil = opening['mu'] + opening['sigma'] * 3
opening['level_key'] = make_level_key(floor, ceil)
opening['level_str'] = make_level_str(floor, ceil)
opening['skill_str'] = skill_str(opening['mu'], opening['sigma'])
+ opening['cards'] = opening['_id'][len('open:'):].split('+')
opening['cards'].sort()
opening['cards'].sort(key=lambda card: (card_info.cost(card)),
reverse=True)
@@ -125,7 +118,7 @@ def GET(self):
if selected_card == '':
openings = [op for op in openings
if op['level_key'][0] != 0
- or op['cards'] == ['Silver', 'Silver']]
+ or op['_id'] == ['Silver', 'Silver']]
render = web.template.render('')
return render.openings_template(openings, card_list, selected_card)
@@ -1,3 +1,8 @@
+""" Scan over a collection and remember which documents were seen.
+
+This is useful for implementing daily updates, so that we only scan where we
+left off."""
+
class IncrementalScanner(object):
def __init__(self, scan_name, db):
self.num_games = 0
View
@@ -1,13 +1,21 @@
#!/usr/bin/python
+""" This a mixin to a generically serialize objects to primative types.
+
+This is serializing the internal variables of classes, and hence is a
+big abstraction leak. By mixing with this class, you can give yourself
+headaches if you change the implementation of a class and want to work
+with previously serialized versions of data.
+"""
+
import collections
PRIMITIVES = [dict, str, int, list, float, unicode]
def to_primitive(val):
if hasattr(val, 'to_primitive_object'):
return val.to_primitive_object()
- assert type(val) in PRIMITIVES, val
+ assert type(val) in PRIMITIVES, (val, type(val))
return val
class PrimitiveConversion(object):
View
@@ -1,101 +1,130 @@
-from utils import get_mongo_connection
-con = get_mongo_connection()
-DB = con.test
-games = DB.games
+""" Update trueskill ratings for openings."""
-from trueskill.trueskill import db_update_trueskill, get_skill, get_stdev
+from utils import get_mongo_connection, progress_meter
+
+import trueskill.trueskill as ts
+import incremental_scanner
+import primitive_util
+import utils
def results_to_ranks(results):
sorted_results = sorted(results)
return [sorted_results.index(r) for r in results]
-def run_trueskill_players():
- collection = DB.trueskill_players
- collection.remove()
- collection.ensure_index('name')
- for game in games.find():
- if len(game['decks']) >= 2:
- players = []
- results = []
- for deck in game['decks']:
- nturns = len(deck['turns'])
- if deck['resigned']:
- vp = -1000
- else:
- vp = deck['points']
- results.append((-vp, nturns))
- players.append(deck['name'])
- ranks = results_to_ranks(results)
- team_results = [
- ([player], [1.0], rank)
- for player, rank in zip(players, ranks)
- ]
- db_update_trueskill(team_results, collection)
+class PrimitiveSkillInfo(primitive_util.PrimitiveConversion):
+ def to_primitive_object(self):
+ return {'mu': float(self.mu),
+ 'sigma': float(self.sigma),
+ 'gamma': float(self.gamma)}
+
+class DbBackedSkillTable(ts.SkillTable):
+ def __init__(self, coll):
+ ts.SkillTable.__init__(self, self._db_backed_missing_func)
+ self.coll = coll
+ self.skill_infos = {}
+
+ def _db_backed_missing_func(self, name):
+ if name in self.skill_infos:
+ return self.skill_infos[name]
+ db_data = self.coll.find_one({'_id': name})
+ skill_info = PrimitiveSkillInfo()
+ if db_data:
+ skill_info.mu = db_data['mu']
+ skill_info.sigma = db_data['sigma']
+ skill_info.gamma = db_data['gamma']
+ else:
+ skill_info.mu = 25
+ skill_info.sigma = 25.0/3
+ if name.startswith('open:'):
+ skill_info.gamma = 0.0001
+ else:
+ skill_info.gamma = 25.0/300
+
+ self.skill_infos[name] = skill_info
+ return self.skill_infos[name]
+
+ def save(self):
+ for key, val in self.skill_infos.iteritems():
+ utils.write_object_to_db(val, self.coll, key)
+
+def setup_openings_collection(coll):
+ coll.ensure_index('_id')
+
+def update_skills_for_game(game, opening_skill_table,
+ #player_skill_table
+ ):
+ teams = []
+ results = []
+ openings = []
+ dups = False
+ for deck in game['decks']:
+ opening = deck['turns'][0].get('buys', []) + \
+ deck['turns'][1].get('buys', [])
+
+ opening.sort()
+ open_name = 'open:' + '+'.join(opening)
+ if open_name in openings:
+ dups = True
+ openings.append(open_name)
+ nturns = len(deck['turns'])
+ if deck['resigned']:
+ vp = -1000
+ else:
+ vp = deck['points']
+ results.append((-vp, nturns))
+ player_name = deck['name']
+
+ teams.append([open_name, player_name])
+ ranks = results_to_ranks(results)
+ if not dups:
+ team_results = [
+ (team, [0.5, 0.5], rank)
+ for team, rank in zip(teams, ranks)
+ ]
+ ts.update_trueskill_team(team_results, opening_skill_table)
+ # player_results = [
+ # ([team[1]], [1.0], rank)
+ # for team, rank in zip(teams, ranks)
+ # ]
+ # ts.update_trueskill_team(player_results, player_skill_table)
+
def run_trueskill_openings():
- collection = DB.trueskill_openings
- player_collection = DB.trueskill_players
- player_collection.remove()
- player_collection.ensure_index('name')
- player_collection.ensure_index('mu')
- player_collection.ensure_index('floor')
- player_collection.ensure_index('ceil')
- collection.remove()
- collection.ensure_index('name')
- collection.ensure_index('mu')
- collection.ensure_index('floor')
- collection.ensure_index('ceil')
- for game in games.find():
+ con = get_mongo_connection()
+ db = con.test
+ games = db.games
+
+ collection = db.trueskill_openings
+ player_collection = db.trueskill_players
+ # player_collection.remove()
+ # collection.remove()
+ setup_openings_collection(collection)
+ # setup_openings_collection(player_collection)
+
+ opening_skill_table = DbBackedSkillTable(collection)
+ # player_skill_table = DbBackedSkillTable(player_collection)
+
+ args = utils.incremental_max_parser().parse_args()
+ scanner = incremental_scanner.IncrementalScanner('trueskill', db)
+ if not args.incremental:
+ scanner.reset()
+ collection.drop()
+
+ for ind, game in enumerate(
+ progress_meter(scanner.scan(db.games, {}), 100)):
if len(game['decks']) >= 2 and len(game['decks'][1]['turns']) >= 5:
- teams = []
- results = []
- openings = []
- dups = False
- for deck in game['decks']:
- opening = deck['turns'][0].get('buys', []) +\
- deck['turns'][1].get('buys', [])
- opening.sort()
- open_name = 'open:'+ ('+'.join(opening))
- if open_name in openings:
- dups = True
- openings.append(open_name)
- nturns = len(deck['turns'])
- if deck['resigned']:
- vp = -1000
- else:
- vp = deck['points']
- results.append((-vp, nturns))
- player_name = deck['name']
- player_info = {
- 'name': player_name,
- 'mu': get_skill(player_name, player_collection),
- 'sigma': get_stdev(player_name, player_collection)
- }
-
- teams.append([open_name, player_info])
- ranks = results_to_ranks(results)
- if not dups:
- team_results = [
- (team, [0.5, 0.5], rank)
- for team, rank in zip(teams, ranks)
- ]
- db_update_trueskill(team_results, collection)
- player_results = [
- ([team[1]], [1.0], rank)
- for team, rank in zip(teams, ranks)
- ]
- db_update_trueskill(player_results, player_collection)
-
-def update_plays():
- DB.trueskill_openings.ensure_index('cards')
- for opening in DB.trueskill_openings.find():
- key = opening['name']
- cards = key[5:].split('+')
- if cards == ['']:
- cards = []
- print cards
- DB.trueskill_openings.update(
- {'name': key},
- {'$set': {'cards': cards}}
- )
+ update_skills_for_game(game, opening_skill_table)
+
+ if ind == args.max_games:
+ break
+
+ #player_skill_table.save()
+ opening_skill_table.save()
+ scanner.save()
+ print scanner.status_msg()
+
+def main():
+ run_trueskill_openings()
+if __name__ == '__main__':
+ main()
Oops, something went wrong.

0 comments on commit e45a8ab

Please sign in to comment.