From d6d6453ee3b07463377e49090106f0c8e3e39a84 Mon Sep 17 00:00:00 2001 From: James Clarke Date: Sun, 8 Jul 2018 21:49:34 +1000 Subject: [PATCH] get training to pass runtime checks --- basic-queue-manager.py | 16 +-- conf/ConfigWrapper.py | 4 - deep-queue-manager.py | 22 ++-- default_imports.py | 3 - modules/__init__.py | 0 modules/auth/Auth.py | 44 +++---- modules/auth/Env.py | 7 +- modules/auth/Priv.py | 10 +- modules/auth/Token.py | 17 +-- modules/auth/User.py | 30 ++--- modules/client/Api.py | 2 - modules/client/Env.py | 1 - modules/client/Job.py | 28 +++- modules/db/DBManager.py | 8 +- modules/game/AnalysedGame.py | 58 +++----- modules/game/AnalysedMove.py | 28 +--- modules/game/AnalysedPosition.py | 11 +- modules/game/Api.py | 4 +- modules/game/Blurs.py | 6 +- modules/game/EngineEval.py | 11 +- modules/game/EngineTools.py | 3 - modules/game/Env.py | 11 +- modules/game/Game.py | 28 +--- modules/game/GameStore.py | 13 -- modules/game/Player.py | 25 +--- modules/irwin/AnalysedGameActivation.py | 61 +++++---- modules/irwin/AnalysedGameModel.py | 124 +++--------------- modules/irwin/AnalysisReport.py | 4 +- modules/irwin/BasicGameModel.py | 105 ++------------- modules/irwin/GameBasicActivation.py | 62 +++++---- modules/irwin/Irwin.py | 6 +- .../irwin/training/AnalysedModelTraining.py | 99 ++++++++++++++ modules/irwin/training/BasicModelTraining.py | 95 ++++++++++++++ modules/irwin/training/Env.py | 19 +++ modules/irwin/{ => training}/Evaluation.py | 2 +- modules/irwin/training/Training.py | 16 +++ modules/queue/EngineQueue.py | 93 +++++++------ modules/queue/Env.py | 13 +- modules/queue/IrwinQueue.py | 28 ++-- modules/queue/Origin.py | 6 + modules/queue/Queue.py | 17 ++- scan-update.py | 54 ++++---- tools.py | 32 ++--- utils/updatePlayerDatabase.py | 2 +- webapp/DefaultResponse.py | 17 +++ webapp/controllers/api/blueprint.py | 64 ++++----- 46 files changed, 666 insertions(+), 643 deletions(-) delete mode 100644 modules/__init__.py create mode 100644 modules/irwin/training/AnalysedModelTraining.py create mode 100644 modules/irwin/training/BasicModelTraining.py create mode 100644 modules/irwin/training/Env.py rename modules/irwin/{ => training}/Evaluation.py (96%) create mode 100644 modules/irwin/training/Training.py create mode 100644 modules/queue/Origin.py create mode 100644 webapp/DefaultResponse.py diff --git a/basic-queue-manager.py b/basic-queue-manager.py index adb3e9a..13a80a4 100644 --- a/basic-queue-manager.py +++ b/basic-queue-manager.py @@ -16,7 +16,7 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--quiet", dest="loglevel", - default=logging.DEBUG, action="store_const", const=logging.INFO, + default=logging.DEBUG, action="store_const", const=logging.INFO, help="reduce the number of logged messages") config = parser.parse_args() @@ -39,7 +39,7 @@ basicPlayerQueue = env.basicPlayerQueueDB.nextUnprocessed() if basicPlayerQueue is not None: logging.info("Basic Queue: " + str(basicPlayerQueue)) - userId = basicPlayerQueue.id + playerId = basicPlayerQueue.id origin = basicPlayerQueue.origin else: logging.info("Basic Queue empty. Pausing") @@ -49,21 +49,21 @@ # if there is already a deep queue item open # don't update. This will push the request # down the queue - if env.deepPlayerQueueDB.exists(userId): + if env.deepPlayerQueueDB.exists(playerId): continue # get analysed (by fishnet/lichess) games from the db gameStore = GameStore.new() - gameStore.addGames(env.gameDB.byUserIdAnalysed(userId)) - gameTensors = gameStore.gameTensors(userId) + gameStore.addGames(env.gameDB.byPlayerIdAnalysed(playerId)) + gameTensors = gameStore.gameTensors(playerId) if len(gameTensors) > 0: gamePredictions = env.irwin.predictBasicGames(gameTensors) - gameActivations = [GameBasicActivation.fromPrediction(gameId, userId, prediction, False) + gameActivations = [GameBasicActivation.fromPrediction(gameId, playerId, prediction, False) for gameId, prediction in gamePredictions] env.gameBasicActivationDB.lazyWriteMany(gameActivations) deepPlayerQueue = DeepPlayerQueue.new( - userId=userId, + playerId=playerId, origin=origin, gamePredictions=gamePredictions) logging.info("Writing DeepPlayerQueue: " + str(deepPlayerQueue)) @@ -71,7 +71,7 @@ else: logging.info("No gameTensors") deepPlayerQueue = DeepPlayerQueue.new( - userId=userId, + playerId=playerId, origin=origin, gamePredictions=[]) logging.info("Writing DeepPlayerQueue: " + str(deepPlayerQueue)) diff --git a/conf/ConfigWrapper.py b/conf/ConfigWrapper.py index f5f52f2..88c1dc6 100644 --- a/conf/ConfigWrapper.py +++ b/conf/ConfigWrapper.py @@ -6,17 +6,14 @@ class ConfigWrapper: """ Used for loading and accessing values from a json config file. """ - @validated def __init__(self, d: Dict): self.d = d @staticmethod - @validated def new(filename: str): with open(filename) as confFile: return ConfigWrapper(json.load(confFile)) - @validated def __getitem__(self, key: str): """ allows for accessing like, conf["index items like this"] @@ -27,7 +24,6 @@ def __getitem__(self, key: str): except ValueError: return self.__getattr__(key) - @validated def __getattr__(self, key: str): """ allows for accessing like, conf.index.like.this diff --git a/deep-queue-manager.py b/deep-queue-manager.py index a99da8c..ff57f20 100644 --- a/deep-queue-manager.py +++ b/deep-queue-manager.py @@ -15,9 +15,9 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--name", dest="name", - default=None, type=str, help="name of the thread") + default=None, type=str, help="name of the thread") parser.add_argument("--quiet", dest="loglevel", - default=logging.DEBUG, action="store_const", const=logging.INFO, + default=logging.DEBUG, action="store_const", const=logging.INFO, help="reduce the number of logged messages") config = parser.parse_args() @@ -45,8 +45,8 @@ sleep(30) continue logging.info("Deep Queue: " + str(deepPlayerQueue)) - userId = deepPlayerQueue.id - playerData = env.api.getPlayerData(userId) + playerId = deepPlayerQueue.id + playerData = env.api.getPlayerData(playerId) if playerData is None: logging.warning("getPlayerData returned None") @@ -55,12 +55,12 @@ # get player data, this is really only # useful for knowing which games must be analysed - player = env.playerDB.byId(userId) + player = env.playerDB.byId(playerId) # pull what we already have on the player gameStore = GameStore.new() - gameStore.addGames(env.gameDB.byUserId(userId)) - gameStore.addAnalysedGames(env.analysedGameDB.byUserId(userId)) + gameStore.addGames(env.gameDB.byPlayerId(playerId)) + gameStore.addAnalysedGames(env.analysedGameDB.byPlayerId(playerId)) # Filter games and assessments for relevant info try: @@ -70,12 +70,12 @@ env.deepPlayerQueueDB.complete(deepPlayerQueue) continue # if this doesn't gather any useful data, skip - env.gameDB.lazyWriteGames(gameStore.games) + env.gameDB.lazyWriteMany(gameStore.games) logging.info("Already Analysed: " + str(len(gameStore.analysedGames))) # decide which games should be analysed - gameTensors = gameStore.gameTensorsWithoutAnalysis(userId) + gameTensors = gameStore.gameTensorsWithoutAnalysis(playerId) if len(gameTensors) > 0: gamePredictions = env.irwin.predictBasicGames(gameTensors) # [(gameId, prediction)] @@ -100,7 +100,7 @@ game=game, engine=env.engine, infoHandler=env.infoHandler, - white=game.white == userId, + white=game.white == playerId, nodes=env.config['stockfish']['nodes'], analysedPositionDB=env.analysedPositionDB )) @@ -109,7 +109,7 @@ env.analysedGameDB.lazyWriteAnalysedGames(gameStore.analysedGames) - logging.info('Posting report for ' + userId) + logging.info('Posting report for ' + playerId) env.api.postReport(env.irwin.report( player=player, gameStore=gameStore, diff --git a/default_imports.py b/default_imports.py index de81c50..516997a 100644 --- a/default_imports.py +++ b/default_imports.py @@ -5,9 +5,6 @@ from numpy import float16, float32, float64 from numpy import float as npfloat -from enforce import runtime_validation as validated -from enforce.exceptions import RuntimeTypeError - Number = TypeVar('Number', int, float, npfloat, float16, float32, float64) ## Logging diff --git a/modules/__init__.py b/modules/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/modules/auth/Auth.py b/modules/auth/Auth.py index 760574c..fdefe7b 100644 --- a/modules/auth/Auth.py +++ b/modules/auth/Auth.py @@ -1,18 +1,18 @@ from default_imports import * from modules.auth.Env import Env -from modules.auth.User import User, Username, Password -from modules.auth.Token import Token +from modules.auth.User import User, UserID, Username, Password +from modules.auth.Token import Token, TokeniD from modules.auth.Priv import Priv -TokenID = NewType('TokenID', str) - Authable = TypeVar('Authable', User, Token) -@validated +Authorised = NewType('Authorised', bool) + +AuthID = TypeVar('AuthID', UserID, TokenID) + class Auth(NamedTuple('Auth', [('env', Env)])): - @validated - def loginUser(self, username: Username, password: Password) -> Tuple[Opt[User], bool]: + def loginUser(self, username: Username, password: Password) -> Tuple[Opt[User], Authorised]: """ Attempts to log in a user. Returns True is successful. @@ -24,8 +24,7 @@ def loginUser(self, username: Username, password: Password) -> Tuple[Opt[User], return (user, user.checkPassword(password)) return (None, False) - @validated - def registerUser(self, name: str, password: Password, privs: List[Priv] = []) -> Opt[User]: + def registerUser(self, name: Username, password: Password, privs: List[Priv] = []) -> Opt[User]: """ Will attempt to register a user. Returns User object if successful, otherwise None. @@ -36,8 +35,7 @@ def registerUser(self, name: str, password: Password, privs: List[Priv] = []) -> return user return None - @validated - def authoriseTokenId(self, tokenId: TokenID, priv: Priv) -> Tuple[Opt[Token], bool]: + def authoriseTokenId(self, tokenId: TokenID, priv: Priv) -> Tuple[Opt[Token], Authorised]: """ Given a tokenId, will check if the tokenId has priv. """ @@ -46,8 +44,7 @@ def authoriseTokenId(self, tokenId: TokenID, priv: Priv) -> Tuple[Opt[Token], bo return (token, token.hasPriv(priv)) return (None, False) - @validated - def authoriseUser(self, username: Username, password: Password, priv: Priv) -> Tuple[Opt[User], bool]: + def authoriseUser(self, username: Username, password: Password, priv: Priv) -> Tuple[Opt[User], Authorised]: """ Checks if user has priv in list of privs. """ @@ -56,20 +53,23 @@ def authoriseUser(self, username: Username, password: Password, priv: Priv) -> T return (user, loggedIn and user.hasPriv(priv)) return (None, False) - @validated - def authoriseRequest(self, req: Dict, priv: Priv) -> Tuple[Opt[Authable], bool]: + def authoriseRequest(self, req: Opt[Dict], priv: Priv) -> Tuple[Opt[Authable], Authorised]: """ Checks if a request is verified with priv. """ if req is not None: - tokenId = reg.get('auth', {}).get('token') - if tokenId is not None: - return self.authoriseTokenId(tokenId, priv) + # Attempt to authorise token first + authReq = req.get('auth') + if authReq is not None: + tokenId = authReq.get('token') + if tokenId is not None: + return self.authoriseTokenId(tokenId, priv) - username = req.get('auth', {}).get('username') - password = req.get('auth', {}).get('password') + # Then attempt to authorise user/password + username = authReq.get('username') + password = authReq.get('password') - if None not in [username, password]: - return self.authoriseUser(username, password, priv) + if None not in [username, password]: + return self.authoriseUser(username, password, priv) return (None, False) \ No newline at end of file diff --git a/modules/auth/Env.py b/modules/auth/Env.py index 52a39c3..071a496 100644 --- a/modules/auth/Env.py +++ b/modules/auth/Env.py @@ -8,9 +8,8 @@ from pymongo.database import Database class Env: - @validated - def __init__(self, config: ConfigWrapper, db: Database): + def __init__(self, config: ConfigWrapper, db: Database): self.db = db self.config = config - self.userDB = UserDB(self.db[self.config.coll.user]) - self.tokenDB = TokenDB(self.db[self.config.coll.token]) \ No newline at end of file + self.userDB = UserDB(self.db[self.config["auth coll user"]]) + self.tokenDB = TokenDB(self.db[self.config["auth coll token"]]) \ No newline at end of file diff --git a/modules/auth/Priv.py b/modules/auth/Priv.py index 706453c..19f16f0 100644 --- a/modules/auth/Priv.py +++ b/modules/auth/Priv.py @@ -2,9 +2,9 @@ Permission = NewType('Permission', str) -Priv = validated(NamedTuple('Priv' [ +Priv = NamedTuple('Priv' [ ('permission', Permission) - ])) + ]) RequestJob = Priv('request_job') # client can request work CompleteJob = Priv('complete_job') # client can post results of work @@ -12,13 +12,11 @@ class PrivBSONHandler: @staticmethod - @validated - def reads(bson: Dict) -> Priv: + def reads(bson: Dict) -> Priv: return Priv( permission=bson['_id']) @staticmethod - @validated - def writes(priv: Priv) -> Dict: + def writes(priv: Priv) -> Dict: return { '_id': priv.permission} \ No newline at end of file diff --git a/modules/auth/Token.py b/modules/auth/Token.py index d605285..f28d1c0 100644 --- a/modules/auth/Token.py +++ b/modules/auth/Token.py @@ -6,39 +6,32 @@ TokenID = NewType('TokenID', str) -@validated class Token(NamedTuple('Token', [ ('id', TokenID), ('privs', List[Priv]) ])): - @validated - def hasPriv(self, priv: Priv) -> bool: + def hasPriv(self, priv: Priv) -> bool: return priv in self.privs class TokenBSONHandler: @staticmethod - @validated - def reads(bson: Dict) -> Token: + def reads(bson: Dict) -> Token: return Token( id = bson['_id'], privs = [PrivBSONHandler.reads(p) for p in bson['privs']]) @staticmethod - @validated - def writes(token: Token) -> Dict: + def writes(token: Token) -> Dict: return { '_id': token.id, 'privs': [PrivBSONHandler.writes(p) for p in token.privs]} -@validated class TokenDB(NamedTuple('TokenDB', [ ('coll', Collection) ])): - @validated - def write(self, token: Token): + def write(self, token: Token): self.coll.update_one({'_id': token.id}, {'$set': TokenBSONHandler.writes(token)}, upsert=True) - @validated - def byId(self, _id: TokenId) -> Opt[Token]: + def byId(self, _id: TokenId) -> Opt[Token]: doc = self.coll.find_one({'_id': _id}) return None if doc is None else TokenBSONHandler.reads(doc) \ No newline at end of file diff --git a/modules/auth/User.py b/modules/auth/User.py index 49f4bd2..e5a450e 100644 --- a/modules/auth/User.py +++ b/modules/auth/User.py @@ -6,21 +6,18 @@ import hashlib, uuid Username = NewType('Username', str) -UserID = Username -Name = NewType('Name', str) +UserID = NewType('UserID', str) Password = NewType('Password', str) Salt = NewType('Salt', str) -@validated class User(NamedTuple('User', [ - ('id', Username), - ('name', Name), + ('id', UserID), + ('name', Username), ('password', Password), ('salt', Salt), ('privs', List[Priv])])): @staticmethod - @validated - def new(name: Name, password: Password, privs: List[Priv] = []) -> User: + def new(name: Username, password: Password, privs: List[Priv] = []) -> User: """ Creates a new User object. """ @@ -34,8 +31,7 @@ def new(name: Name, password: Password, privs: List[Priv] = []) -> User: ) @staticmethod - @validated - def hashPassword(password: Password, salt: Opt[Salt] = None) -> Tuple[Password, Salt]: + def hashPassword(password: Password, salt: Opt[Salt] = None) -> Tuple[Password, Salt]: """ Given a string and a salt this function will generate a hash of the password. If salt is not provided a new random salt is created. @@ -45,8 +41,7 @@ def hashPassword(password: Password, salt: Opt[Salt] = None) -> Tuple[Password, hashedPassword = hashlib.sha512(password + salt).hexdigest() return hashedPassword, salt - @validated - def checkPassword(self, password: Password) -> bool: + def checkPassword(self, password: Password) -> bool: """ Checks if a raw password matches that hashed password of the user. """ @@ -54,8 +49,7 @@ def checkPassword(self, password: Password) -> bool: class UserBSONHandler: @staticmethod - @validated - def reads(bson: Dict) -> User: + def reads(bson: Dict) -> User: return User( id = bson['_id'], name = bson['name'], @@ -64,8 +58,7 @@ def reads(bson: Dict) -> User: privs = [PrivBSONHandler.reads(p) for p in bson['privs']]) @staticmethod - @validated - def writes(user: User) -> Dict: + def writes(user: User) -> Dict: return { '_id': user.id, 'name': user.name, @@ -74,15 +67,12 @@ def writes(user: User) -> Dict: 'privs': [PrivBSONHandler.writes(p) for p in user.privs] } -@validated class UserDB(NamedTuple('UserDB', [ ('coll', Collection) ])): - @validated - def write(self, user: User): + def write(self, user: User): self.coll.update_one({'_id': user.id}, {'$set': UserBSONHandler.writes(user)}, upsert=True) - @validated - def byId(self, _id: UserID) -> Opt[User]: + def byId(self, _id: UserID) -> Opt[User]: doc = self.coll.find_one({'_id': _id}) return None if doc is None else UserBSONHandler.reads(doc) \ No newline at end of file diff --git a/modules/client/Api.py b/modules/client/Api.py index bb64a22..4131184 100644 --- a/modules/client/Api.py +++ b/modules/client/Api.py @@ -4,11 +4,9 @@ from modules.game.Game import GameBSONHandler -@validated class Api(NamedTuple('Api', [ ('config', ConfigWrapper) ])): - @validated def request_job(self) -> Opt[Dict]: for i in range(5): try: diff --git a/modules/client/Env.py b/modules/client/Env.py index a71e243..94988ce 100644 --- a/modules/client/Env.py +++ b/modules/client/Env.py @@ -3,6 +3,5 @@ from conf.ConfigWrapper import ConfigWrapper class Env: - @validated def __init__(self, config: ConfigWrapper): self.config = config \ No newline at end of file diff --git a/modules/client/Job.py b/modules/client/Job.py index 558d9fb..19fb3a5 100644 --- a/modules/client/Job.py +++ b/modules/client/Job.py @@ -1,7 +1,27 @@ from default_imports import * -from modules.game.Player import Player +from modules.game.Player import PlayerID +from modules.game.Game import Game, GameBSONHandler +from modules.game.AnalysedPosition import AnalysedPosition, AnalysedPositionBSONHandler -class Job(NamedTuple('Job', [ - ('userId') - ])) \ No newline at end of file +Job = NamedTuple('Job', [ + ('playerId', PlayerID), + ('games', List[Game]), + ('analysedPositions', List[AnalysedPosition]) + ]) + +class JobBSONHandler: + @staticmethod + def reads(bson: Dict) -> Job: + return Job( + playerId = bson['playerId'], + games = [GameBSONHandler.reads(g) for g in bson['games']], + analysedPositions = [AnalysedPositionBSONHandler.reads(ap) for ap in bson['analysedPositions']]) + + @staticmethod + def writes(job: Job) -> Dict: + return { + 'playerId': job.playerId, + 'games': [GameBSONHandler.writes(g) for g in job.games], + 'analysedPositions': [AnalysedPositionBSONHandler.writes(ap) for ap in job.analysedPositions] + } \ No newline at end of file diff --git a/modules/db/DBManager.py b/modules/db/DBManager.py index 74e3711..e63f482 100644 --- a/modules/db/DBManager.py +++ b/modules/db/DBManager.py @@ -5,18 +5,16 @@ from pymongo import MongoClient from pymongo.database import Database -@validated class DBManager(NamedTuple('DBManager', [ - ('config', ConfigWrapper) + ('config', 'ConfigWrapper') ])): - @validated def client(self) -> MongoClient: return MongoClient(self.config.db.host) def db(self) -> Database: client = self.client() - db = client[self.config.database] - if self.config.authenticate: + db = client[self.config['db database']] + if self.config['db authenticate']: db.authenticate( self.config.authentication.username, self.config.authentication.password, mechanism='MONGODB-CR') diff --git a/modules/game/AnalysedGame.py b/modules/game/AnalysedGame.py index 335ba2a..0d80707 100644 --- a/modules/game/AnalysedGame.py +++ b/modules/game/AnalysedGame.py @@ -8,7 +8,7 @@ from modules.game.Game import GameID, Blur, Emt from modules.game.Colour import Colour from modules.game.Player import PlayerID -from modules.game.AnalysedMove import AnalysedMove, AnalysedMoveBSONHandler, EngineEval, Analysis +from modules.game.AnalysedMove import AnalysedMove, AnalysedMoveBSONHandler, EngineEval, Analysis, Rank, TrueRank from modules.game.AnalysedPosition import AnalysedPosition from pymongo.collection import Collection @@ -17,7 +17,6 @@ AnalysedGameTensor = NewType('AnalysedGameTensor', np.ndarray) -@validated class AnalysedGame(NamedTuple('AnalysedGame', [ ('id', AnalysedGameID), ('playerId', PlayerID), @@ -29,8 +28,7 @@ class AnalysedGame(NamedTuple('AnalysedGame', [ player's perspective. """ @staticmethod - @validated - def new(gameId: GameID, colour: Colour, playerId: PlayerID, analysedMoves: List[AnalysedMove]) -> AnalysedGame: + def new(gameId: GameID, colour: Colour, playerId: PlayerID, analysedMoves: List[AnalysedMove]): return AnalysedGame( id=AnalysedGame.makeId(gameId, colour), playerId=playerId, @@ -38,11 +36,9 @@ def new(gameId: GameID, colour: Colour, playerId: PlayerID, analysedMoves: List[ analysedMoves=analysedMoves) @staticmethod - @validated def makeId(gameId: GameID, colour: Colour) -> AnalysedGameID: return gameId + '/' + ('white' if colour else 'black') - @validated def tensor(self, length: int = 60) -> AnalysedGameTensor: emtAvg = self.emtAverage() wclAvg = self.wclAverage() @@ -51,39 +47,30 @@ def tensor(self, length: int = 60) -> AnalysedGameTensor: ts = ts + (length-len(ts))*[AnalysedMove.nullTensor()] return np.array(ts) - @validated def emtAverage(self) -> Number: return np.average([m.emt for m in self.analysedMoves]) - @validated def wclAverage(self) -> Number: return np.average([m.winningChancesLoss() for m in self.analysedMoves]) - @validated def gameLength(self) -> int: return len(self.analysedMoves) - @validated def blurs(self) -> List[Blur]: return [move.blur for move in self.analysedMoves] - @validated def emts(self) -> List[Emt]: return [m.emt for m in self.analysedMoves] - @validated def emtSeconds(self) -> List[Number]: return [emt/100 for emt in self.emts()] - @validated def winningChances(self) -> List[Number]: return [m.advantage() for m in self.analysedMoves] - @validated def winningChancesPercent(self) -> List[Number]: return [100*m.advantage() for m in self.analysedMoves] - @validated def winningChancesLossPercent(self, usePV: bool = True) -> List[Number]: return [100*m.winningChancesLoss(usePV=usePV) for m in self.analysedMoves] @@ -102,24 +89,21 @@ def winningChancesLossByPV(self): pvs[i][2].append('null') return pvs - @validated - def ranks(self) -> List[Rank]: + def ranks(self) -> List[TrueRank]: """ for generating graphs """ return [move.trueRank() for move in self.analysedMoves] - @validated def ambiguities(self) -> List[int]: """ for generating graphs """ return [move.ambiguity() for move in self.analysedMoves] - @validated def length(self) -> int: return len(self.analysedMoves) - def ranksJSON(self): + def ranksJSON(self) -> str: return json.dumps(self.ranks()) - def binnedSeconds(self, bins=10): + def binnedSeconds(self, bins: int = 10) -> Dict: # JSON format for graphing emts = self.emts() minSec = min(emts) @@ -132,7 +116,7 @@ def binnedSeconds(self, bins=10): labels[min(bins-1, i)] = str(round_sig(stepStart/100)) + '-' + str(round_sig((stepStart+step)/100)) + 's' return {'data': json.dumps(data), 'labels': json.dumps(labels)} - def binnedLosses(self, bins=10): + def binnedLosses(self, bins: int = 10) -> Dict: # JSON format for graphing losses = self.winningChancesLossPercent() data = [[] for i in range(bins+1)] @@ -143,7 +127,7 @@ def binnedLosses(self, bins=10): labels.append('Other') return {'data': json.dumps(data), 'labels': json.dumps(labels)} - def binnedPVs(self, bins=6): + def binnedPVs(self, bins: int = 6) -> Dict: # JSON format for graphing pvs = self.ranks() data = [[] for i in range(bins)] @@ -152,24 +136,24 @@ def binnedPVs(self, bins=6): labels = ['PV 1', 'PV 2', 'PV 3', 'PV 4', 'PV 5', 'Other'] return {'data': json.dumps(data), 'labels': json.dumps(labels)} - def moveRankByTime(self): + def moveRankByTime(self) -> List[Dict]: return [{'x': time, 'y': rank} for rank, time in zip(self.ranks(), self.emtSeconds())] - def moveRankByTimeJSON(self): + def moveRankByTimeJSON(self) -> str: # json format for graphing return json.dumps(self.moveRankByTime()) - def lossByTime(self): + def lossByTime(self) -> List[Dict]: return [{'x': time, 'y': loss} for loss, time in zip(self.winningChancesLossPercent(), self.emtSeconds())] - def lossByTimeJSON(self): + def lossByTimeJSON(self) -> str: # json format for graphing return json.dumps(self.lossByTime()) - def lossByRank(self): + def lossByRank(self) -> List[Dict]: return [{'x': rank, 'y': loss} for loss, rank in zip(self.winningChancesLossPercent(), self.ranks())] - def lossByRankJSON(self): + def lossByRankJSON(self) -> str: # json format for graphing return json.dumps(self.lossByRank()) @@ -180,7 +164,6 @@ def round_sig(x, sig=2): class AnalysedGameBSONHandler: @staticmethod - @validated def reads(bson: Dict) -> AnalysedGame: return AnalysedGame( id = bson['_id'], @@ -189,7 +172,6 @@ def reads(bson: Dict) -> AnalysedGame: analysedMoves = [AnalysedMoveBSONHandler.reads(am) for am in bson['analysis']]) @staticmethod - @validated def writes(analysedGame: AnalysedGame) -> Dict: return { '_id': analysedGame.id, @@ -198,34 +180,27 @@ def writes(analysedGame: AnalysedGame) -> Dict: 'analysis': [AnalysedMoveBSONHandler.writes(am) for am in analysedGame.analysedMoves] } -@validated class AnalysedGameDB(NamedTuple('AnalysedGameDB', [ ('analysedGameColl', Collection) ])): - @validated def write(self, analysedGame: AnalysedGame): self.analysedGameColl.update_one( {'_id': analysedGame.id}, {'$set': AnalysedGameBSONHandler.writes(analysedGame)}, upsert=True) - @validated def lazyWriteAnalysedGames(self, analysedGames: List[AnalysedGame]): [self.write(ga) for ga in analysedGames] - @validated - def byUserId(self, playerId: PlayerID) -> List[AnalysedGame]: + def byPlayerId(self, playerId: PlayerID) -> List[AnalysedGame]: return [AnalysedGameBSONHandler.reads(ga) for ga in self.analysedGameColl.find({'playerId': playerId})] - @validated - def byUserIds(self, playerIds: List[PlayerID]) -> List[AnalysedGame]: - return [self.byUserId(playerId) for playerId in playerIds] + def byPlayerIds(self, playerIds: List[PlayerID]) -> List[AnalysedGame]: + return [self.byPlayerId(playerId) for playerId in playerIds] - @validated def byIds(self, ids: List[AnalysedGameID]) -> List[AnalysedGame]: return [AnalysedGameBSONHandler.reads(ga) for ga in self.analysedGameColl.find({"_id": {"$in": ids}})] - @validated def allBatch(self, batch: int, batchSize: int = 500): """ Gets all analysed games in a paged format @@ -234,7 +209,6 @@ def allBatch(self, batch: int, batchSize: int = 500): """ return [AnalysedGameBSONHandler.reads(ga) for ga in self.analysedGameColl.find(skip=batch*batchSize, limit=batchSize)] - @validated def byGameIdAndUserId(self, gameId: GameID, playerId: PlayerID) -> Opt[AnalysedGame]: bson = self.analysedGameColl.find_one({'gameId': gameId, 'playerId': playerId}) return None if bson is None else AnalysedGameBSONHandler.reads(bson) \ No newline at end of file diff --git a/modules/game/AnalysedMove.py b/modules/game/AnalysedMove.py index 570fe36..e5a03e5 100644 --- a/modules/game/AnalysedMove.py +++ b/modules/game/AnalysedMove.py @@ -13,15 +13,14 @@ MoveNumber = NewType('MoveNumber', int) -Analysis = validated(NamedTuple('Analysis', [ - ('uci', UCI), - ('engineEval', EngineEval) -])) +Analysis = NamedTuple('Analysis', [ + ('uci', 'UCI'), + ('engineEval', 'EngineEval') +]) Rank = NewType('Rank', int) TrueRank = NewType('TrueRank', Opt[Rank]) -@validated class AnalysedMove(NamedTuple('AnalysedMove', [ ('uci', UCI), ('move', MoveNumber), @@ -30,7 +29,6 @@ class AnalysedMove(NamedTuple('AnalysedMove', [ ('engineEval', EngineEval), ('analyses', List[Analysis]) ])): - @validated def tensor(self, timeAvg: Number, wclAvg: Number) -> List[Number]: return [ self.rank() + 1, @@ -46,15 +44,12 @@ def tensor(self, timeAvg: Number, wclAvg: Number) -> List[Number]: ] @staticmethod - @validated def nullTensor() -> List[int]: return 10*[0] - @validated def top(self) -> Opt[Analysis]: return next(iter(self.analyses or []), None) - @validated def difToNextBest(self) -> Number: tr = self.trueRank() if tr is not None and tr != 1: @@ -64,22 +59,18 @@ def difToNextBest(self) -> Number: else: return winningChances(self.analyses[-1].engineEval) - self.advantage() - @validated def difToNextWorst(self) -> Number: tr = self.trueRank() if tr is not None and tr <= len(self.analyses)-1: return winningChances(self.analyses[tr].engineEval) - self.advantage() return 0 - @validated def PVsWinningChancesLoss(self) -> Number: return [winningChances(self.top().engineEval) - winningChances(a.engineEval) for a in self.analyses] - @validated def averageWinningChancesLoss(self) -> Number: return np.average(self.PVsWinningChancesLoss()) - @validated def winningChancesLoss(self, usePV: bool = False) -> Number: adv = self.advantage() if usePV: @@ -89,23 +80,18 @@ def winningChancesLoss(self, usePV: bool = False) -> Number: return max(0, winningChances(self.top().engineEval) - adv) - @validated def advantage(self) -> Number: return winningChances(self.engineEval) - @validated def ambiguity(self) -> int: # 1 = only one top move, 5 = all moves good return sum(int(similarChances(winningChances(self.top().engineEval), winningChances(analysis.engineEval))) for analysis in self.analyses) - @validated def trueRank(self) -> TrueRank: return next((x+1 for x, am in enumerate(self.analyses) if am.uci == self.uci), None) - @validated def rank(self) -> Rank: return min(15, next((x for x, am in enumerate(self.analyses) if am.uci == self.uci), self.projectedRank()) + 1) - @validated def projectedRank(self) -> Number: if len(self.analyses) == 1: return 10 @@ -116,25 +102,21 @@ def projectedRank(self) -> Number: return 10 @lru_cache(maxsize=64) -@validated def winningChances(engineEval: EngineEval) -> Number: if engineEval.mate is not None: return 1 if engineEval.mate > 0 else 0 else: return 1 / (1 + exp(-0.004 * engineEval.cp)) -@validated def similarChances(c1: Number, c2: Number) -> bool: return abs(c1 - c2) < 0.05 class AnalysisBSONHandler: @staticmethod - @validated def reads(bson: Dict) -> Analysis: return Analysis(bson['uci'], EngineEvalBSONHandler.reads(bson['engineEval'])) @staticmethod - @validated def writes(analysis: Analysis) -> Dict: return { 'uci': analysis.uci, @@ -144,7 +126,6 @@ def writes(analysis: Analysis) -> Dict: class AnalysedMoveBSONHandler: @staticmethod - @validated def reads(bson: Dict) -> AnalysedMove: return AnalysedMove( uci = bson['uci'], @@ -156,7 +137,6 @@ def reads(bson: Dict) -> AnalysedMove: ) @staticmethod - @validated def writes(analysedMove: AnalysedMove) -> Dict: return { 'uci': analysedMove.uci, diff --git a/modules/game/AnalysedPosition.py b/modules/game/AnalysedPosition.py index f81c6aa..a563e46 100644 --- a/modules/game/AnalysedPosition.py +++ b/modules/game/AnalysedPosition.py @@ -8,7 +8,6 @@ AnalysedPositionID = NewType('AnalysedPositionID', str) -@validated class AnalysedPosition(NamedTuple('AnalysedPosition', [ ('id', AnalysedPositionID), ('analyses', List[Analysis]) @@ -18,37 +17,31 @@ class AnalysedPosition(NamedTuple('AnalysedPosition', [ This is used for accelerating stockfish analysis. """ @staticmethod - @validated - def fromBoardAndAnalyses(board: Board, analyses: List[Analysis]) -> AnalysedPosition: + def fromBoardAndAnalyses(board: Board, analyses: List[Analysis]): return AnalysedPosition( id=AnalysedPosition.idFromBoard(board), analyses=analyses) @staticmethod - @validated def idFromBoard(board: Board) -> AnalysedPositionID: return str(polyglot.zobrist_hash(board)) class AnalysedPositionBSONHandler: @staticmethod - @validated def reads(bson: Dict) -> AnalysedPosition: return AnalysedPosition( id=bson['_id'], analyses=[AnalysisBSONHandler.reads(b) for b in bson['analyses']]) - @validated def writes(analysedPosition: AnalysedPosition) -> Dict: return { '_id': analysedPosition.id, 'analyses': [AnalysisBSONHandler.writes(a) for a in analysedPosition.analyses] } -@validated class AnalysedPositionDB(NamedTuple('AnalysedPositionDB', [ ('analysedPositionColl', Collection) ])): - @validated def write(self, analysedPosition: AnalysedPosition): try: self.analysedPositionColl.update_one( @@ -58,11 +51,9 @@ def write(self, analysedPosition: AnalysedPosition): except pymongo.errors.DuplicateKeyError: logging.warning("DuplicateKeyError when attempting to write position: " + str(analysedPosition.id)) - @validated def lazyWriteMany(self, analysedPositions: List[AnalysedPosition]): [self.write(analysedPosition) for analysedPosition in analysedPositions] - @validated def byBoard(self, board: Board) -> Opt[AnalysedPosition]: analysedPositionBSON = self.analysedPositionColl.find_one({'_id': AnalysedPosition.idFromBoard(board)}) return None if analysedPositionBSON is None else AnalysedPositionBSONHandler.reads(analysedPositionBSON) \ No newline at end of file diff --git a/modules/game/Api.py b/modules/game/Api.py index 71f7fb6..0b95ae8 100644 --- a/modules/game/Api.py +++ b/modules/game/Api.py @@ -4,12 +4,10 @@ from modules.game.Env import Env -@validated class Api(NamedTuple('Api', [ ('env', Env) ])): - @validated - def insertAnalysedGames(self, analysedGamesBSON: Dict): + def insertAnalysedGames(self, analysedGamesBSON: Dict): try: analysedGames = [AnalysedGameBSONHandler.reads(g) for g in analysedGamesBSON] env.analysedGameDB.lazyWriteMany(analysedGames) diff --git a/modules/game/Blurs.py b/modules/game/Blurs.py index 5a7c76d..c7c4701 100644 --- a/modules/game/Blurs.py +++ b/modules/game/Blurs.py @@ -1,6 +1,5 @@ from default_imports import * -@validated class Blurs(NamedTuple('Blurs', [ ('nb', int), ('moves', List[bool]) @@ -9,8 +8,7 @@ class Blurs(NamedTuple('Blurs', [ The blurs for a single side in a Game """ @staticmethod - @validated - def fromDict(d: Dict, l: int) -> Blurs: + def fromDict(d: Dict, l: int): """ d being from game data l being the amount of plys played by the player @@ -24,7 +22,6 @@ def fromDict(d: Dict, l: int) -> Blurs: class BlursBSONHandler: @staticmethod - @validated def reads(bson: Dict) -> Blurs: return Blurs( nb = sum(list(bson['bits'])), @@ -32,7 +29,6 @@ def reads(bson: Dict) -> Blurs: ) @staticmethod - @validated def writes(blurs: Blurs) -> Dict: return { 'bits': ''.join(['1' if i else '0' for i in blurs.moves]) diff --git a/modules/game/EngineEval.py b/modules/game/EngineEval.py index 7fc03cb..37e0232 100644 --- a/modules/game/EngineEval.py +++ b/modules/game/EngineEval.py @@ -2,26 +2,21 @@ from modules.game.Colour import Colour -@validated class EngineEval(NamedTuple('EngineEval', [ ('cp', Opt[Number]), ('mate', Opt[int]) ])): @staticmethod - @validated - def fromDict(d: Dict) -> EngineEval: + def fromDict(d: Dict): return EngineEval(d.get('cp', None), d.get('mate', None)) - @validated def asdict(self) -> Dict: return {'cp': self.cp} if self.cp is not None else {'mate': self.mate} - @validated - def inverse(self) -> EngineEval: + def inverse(self): return EngineEval(-self.cp if self.cp is not None else None, -self.mate if self.mate is not None else None) - @validated def winningChances(self, colour: Colour) -> Number: if self.mate is not None: base = (1 if self.mate > 0 else 0) @@ -31,10 +26,8 @@ def winningChances(self, colour: Colour) -> Number: class EngineEvalBSONHandler: @staticmethod - @validated def reads(bson: Dict) -> List[EngineEval]: return EngineEval.fromDict(bson) - @validated def writes(engineEval: EngineEval) -> Dict: return engineEval.asdict() \ No newline at end of file diff --git a/modules/game/EngineTools.py b/modules/game/EngineTools.py index 435e28a..d6ff4a1 100644 --- a/modules/game/EngineTools.py +++ b/modules/game/EngineTools.py @@ -16,14 +16,12 @@ except ImportError: from io import StringIO -@validated class EngineTools(NamedTuple('EngineTools', [ ('engine', Engine), ('infoHandler', InfoHandler), ('analysedPositionDB', AnalysedPositionDB) ])): @staticmethod - @validated def analyseGame(game: Game, colour: Colour, nodes: int) -> Opt[AnalysedGame]: logging.info("analysing: " + game.id) if len(game.pgn) < 40 or len(game.pgn) > 120: @@ -86,6 +84,5 @@ def analyseGame(game: Game, colour: Colour, nodes: int) -> Opt[AnalysedGame]: return AnalysedGame.new(game.id, colour, userId, analysedMoves) @staticmethod - @validated def ply(moveNumber, colour: Colour) -> int: return (2*(moveNumber-1)) + (0 if colour else 1) \ No newline at end of file diff --git a/modules/game/Env.py b/modules/game/Env.py index 4cfc5e1..1493cf3 100644 --- a/modules/game/Env.py +++ b/modules/game/Env.py @@ -10,12 +10,11 @@ from pymongo.database import Database class Env: - @validated - def __init__(self, config: ConfigWrapper, db: Database): + def __init__(self, config: ConfigWrapper, db: Database): self.config = config self.db = db - self.gameDB = GameDB(self.db[self.config.coll.game]) - self.analysedGameDB = AnalysedGameDB(self.db[self.config.coll.game_analysis]) - self.playerDB = PlayerDB(self.db[self.config.coll.player]) - self.analysedPositionDB = AnalysedPositionDB(self.db[self.config.coll.analysed_position]) \ No newline at end of file + self.gameDB = GameDB(self.db[self.config["game coll game"]]) + self.analysedGameDB = AnalysedGameDB(self.db[self.config["game coll analysed_game"]]) + self.playerDB = PlayerDB(self.db[self.config["game coll player"]]) + self.analysedPositionDB = AnalysedPositionDB(self.db[self.config["game coll analysed_position"]]) \ No newline at end of file diff --git a/modules/game/Game.py b/modules/game/Game.py index 4532a29..824b886 100644 --- a/modules/game/Game.py +++ b/modules/game/Game.py @@ -18,7 +18,6 @@ MoveTensor = NewType('MoveTensor', List[Number]) GameTensor = NewType('GameTensor', List[MoveTensor]) -@validated class Game(NamedTuple('Game', [ ('id', GameID), ('white', PlayerID), @@ -30,8 +29,7 @@ class Game(NamedTuple('Game', [ ('analysis', Analysis) ])): @staticmethod - @validated - def fromDict(gid: GameID, playerId: PlayerID, d: Dict) -> Game: + def fromDict(gid: GameID, playerId: PlayerID, d: Dict): pgn = d['pgn'].split(" ") white, black = None, None if d['color'] == 'white': @@ -50,13 +48,11 @@ def fromDict(gid: GameID, playerId: PlayerID, d: Dict) -> Game: ) @staticmethod - @validated - def fromPlayerData(playerData: Dict) -> List[Game]: - """Returns a list of Game items from playerData json object from lichess api""" - return [Game.fromDict(gid, playerData['user']['id'], g) for gid, g in playerData['games'].items() \ + def fromJson(json: Dict): + """Returns a list of Game items from json json object from lichess api""" + return [Game.fromDict(gid, json['user']['id'], g) for gid, g in json['games'].items() \ if g.get('initialFen') is None and g.get('variant') is None] - @validated def tensor(self, playerId: PlayerID) -> GameTensor: if self.analysis == [] or self.emts is None and (self.white == playerId or self.black == playerId): return None @@ -73,12 +69,10 @@ def tensor(self, playerId: PlayerID) -> GameTensor: tensors = (max(0, 100-len(tensors)))*[Game.nullMoveTensor()] + tensors return tensors[:100] - @validated def emtsByColour(self, colour: Colour) -> List[Emt]: return self.emts[(0 if colour else 1)::2] @staticmethod - @validated def moveTensor(analysis: Analysis, blur: Blur, emt: Emt, avgEmt: Number, colour: Colour) -> MoveTensor: return [ analysis[1].winningChances(colour), @@ -90,16 +84,13 @@ def moveTensor(analysis: Analysis, blur: Blur, emt: Emt, avgEmt: Number, colour: ] @staticmethod - @validated def nullMoveTensor() -> MoveTensor: return [0, 0, 0, 0, 0, 0] @staticmethod - @validated def ply(moveNumber: int, colour: Colour) -> int: return (2*(moveNumber-1)) + (0 if colour else 1) - @validated def getBlur(self, colour: Colour, moveNumber: int) -> Blur: if colour: return self.whiteBlurs.moves[moveNumber-1] @@ -107,7 +98,6 @@ def getBlur(self, colour: Colour, moveNumber: int) -> Blur: class GameBSONHandler: @staticmethod - @validated def reads(bson: Dict) -> Game: return Game( id = bson['_id'], @@ -120,7 +110,6 @@ def reads(bson: Dict) -> Game: analysis = EngineEvalBSONHandler.reads(bson.get('analysis', []))) @staticmethod - @validate def writes(game: Game) -> Dict: return { '_id': game.id, @@ -134,31 +123,24 @@ def writes(game: Game) -> Dict: 'analysed': len(game.analysis) > 0 } -@validated class GameDB(NamedTuple('GameDB', [ ('gameColl', Collection) ])): - @validated def byId(self, _id: GameID) -> Opt[Game]: bson = self.gameColl.find_one({'_id': _id}) return None if bson is None else GameBSONHandler.reads(bson) - @validated def byIds(self, ids: List[GameID]) -> List[Game]: return [GameBSONHandler.reads(g) for g in self.gameColl.find({'_id': {'$in': [i for i in ids]}})] - @validated def byPlayerId(self, playerId: PlayerID) -> List[Game]: return [GameBSONHandler.reads(g) for g in self.gameColl.find({"$or": [{"white": playerId}, {"black": playerId}]})] - @validated def byPlayerIdAndAnalysed(self, playerId: PlayerID, analysed: bool = True) -> List[Game]: return [GameBSONHandler.reads(g) for g in self.gameColl.find({"analysed": analysed, "$or": [{"white": playerId}, {"black": playerId}]})] - @validated def write(self, game: Game): self.gameColl.update_one({'_id': game.id}, {'$set': GameBSONHandler.writes(game)}, upsert=True) - @validated - def lazyWriteGames(self, games: List[Game]): + def lazyWriteMany(self, games: List[Game]): [self.write(g) for g in games] \ No newline at end of file diff --git a/modules/game/GameStore.py b/modules/game/GameStore.py index 67fc8ea..56ff07b 100644 --- a/modules/game/GameStore.py +++ b/modules/game/GameStore.py @@ -8,63 +8,50 @@ import math import json -@validated class GameStore(NamedTuple('GameStore', [ ('playerId', PlayerID) ('games', List[Game]), ('analysedGames', List[AnalysedGame]) ])): @staticmethod - @validated def new(playerId: PlayerID) -> GameStore: return GameStore(playerId, [], []) - @validated def gamesWithoutAnalysis(self, excludeIds: List[GameID] = []) -> List[Game]: return [game for game in self.games if not self.gameIdHasAnalysis(game.id) if (game.id not in excludeIds)] - @validated def gameIdHasAnalysis(self, gid: GameID) -> bool: return any([ga for ga in self.analysedGames if ga.gameId == gid]) - @validated def hasGameId(self, gid: GameID) -> bool: return any([g for g in self.games if gid == g.id]) - @validated def gameById(self, gid: GameID) -> Opt[Game]: return next(iter([g for g in self.games if gid == g.id]), None) - @validated def addGames(self, games: List[Game]) -> None: [self.games.append(g) for g in games if (not self.hasGameId(g.id) and g.emts is not None and len(g.pgn) < 120 and len(g.pgn) > 40)] - @validated def addAnalysedGame(self, ga: AnalysedGame) -> None: if not self.gameIdHasAnalysis(ga.gameId) and ga is not None and len(ga.analysedMoves) < 60 and len(ga.analysedMoves) > 20: self.analysedGames.append(ga) - @validated def addAnalysedGames(self, analysedGames: List[AnalysedGame]) -> None: [self.addAnalysedGame(ga) for ga in analysedGames] - @validated def randomGamesWithoutAnalysis(self, size: int = 10, excludeIds: List[GameID] = []) -> List[Game]: gWithout = self.gamesWithoutAnalysis(excludeIds) if len(gWithout) > 0: return [gWithout[x] for x in np.random.choice(list(range(len(gWithout))), min(len(gWithout), size), replace=False)] return [] - @validated def gameTensors(self) -> List[GameTensor]: tensors = [(g.id, g.tensor(self.playerId)) for g in self.games] return [t for t in tensors if t[1] is not None] - @validated def gameTensorsWithoutAnalysis(self) -> List[GameTensor]: return [(gid, t) for gid, t in self.gameTensors(self.playerId) if not self.gameIdHasAnalysis(gid)] - @validated def analysedGameTensors(self) -> List[AnalysedGameTensor]: return [(analysedGame.tensor(), analysedGame.length()) for analysedGame in self.analysedGames if len(analysedGame.analysedMoves) < 60 and len(analysedGame.analysedMoves) > 20 and analysedGame.emtAverage() < 2000] diff --git a/modules/game/Player.py b/modules/game/Player.py index 3b438f5..224262a 100644 --- a/modules/game/Player.py +++ b/modules/game/Player.py @@ -4,18 +4,18 @@ import pymongo from pymongo.collection import Collection +from typing import NewType + PlayerID = NewType('PlayerID', str) -@validated class Player(NamedTuple('Player', [ - ('id', PlayerID), + ('id', 'PlayerID'), ('titled', bool), ('engine', bool), ('gamesPlayed', int), - ('relatedCheaters', List[PlayerID])])): + ('relatedCheaters', List['PlayerID'])])): @staticmethod - @validated - def fromPlayerData(data: Dict) -> Opt[Player]: + def fromJson(data: Dict): user = data.get('user') assessment = data.get('assessment', {}) try: @@ -31,7 +31,6 @@ def fromPlayerData(data: Dict) -> Opt[Player]: class PlayerBSONHandler: @staticmethod - @validated def reads(bson: Dict) -> Player: return Player( id = bson['_id'], @@ -41,7 +40,6 @@ def reads(bson: Dict) -> Player: relatedCheaters = bson.get('relatedCheaters', []) ) - @validated def writes(player: Player) -> Dict: return { '_id': player.id, @@ -52,25 +50,20 @@ def writes(player: Player) -> Dict: 'date': datetime.now() } -@validated class PlayerDB(NamedTuple('PlayerDB', [ - ('playerColl', Collection) + ('playerColl', 'Collection') ])): - @validated def byId(self, playerId: PlayerID) -> Opt[Player]: playerBSON = self.playerColl.find_one({'_id': playerId}) return None if playerBSON is None else PlayerBSONHandler.reads(playerBSON) - @validated - def byUserId(self, playerId: PlayerID) -> Opt[Player]: + def byPlayerId(self, playerId: PlayerID) -> Opt[Player]: return self.byId(playerId) - @validated def unmarkedByUserIds(self, playerIds: List[PlayerID]) -> List[Player]: return [(None if bson is None else PlayerBSONHandler.reads(bson)) for bson in [self.playerColl.find_one({'_id': playerId, 'engine': False}) for playerId in playerIds]] - @validated def balancedSample(self, size: int) -> List[Player]: pipelines = [[ {"$match": {"engine": True}}, @@ -83,7 +76,6 @@ def balancedSample(self, size: int) -> List[Player]: legits = [PlayerBSONHandler.reads(p) for p in self.playerColl.aggregate(pipelines[1])] return engines + legits - @validated def oldestNonEngine(self) -> Opt[Player]: playerBSON = self.playerColl.find_one_and_update( filter={'$or': [{'engine': False}, {'engine': None}], 'date': {'$lt': datetime.now() - timedelta(days=30)}}, @@ -91,14 +83,11 @@ def oldestNonEngine(self) -> Opt[Player]: sort=[('date', pymongo.ASCENDING)]) return None if playerBSON is None else PlayerBSONHandler.reads(playerBSON) - @validated def byEngine(self, engine: bool = True) -> List[Player]: return [PlayerBSONHandler.reads(p) for p in self.playerColl.find({'engine': engine})] - @validated def all(self) -> List[Player]: return [PlayerBSONHandler.reads(p) for p in self.playerColl.find({})] - @validated def write(self, player: Player): self.playerColl.update_one({'_id': player.id}, {'$set': PlayerBSONHandler.writes(player)}, upsert=True) \ No newline at end of file diff --git a/modules/irwin/AnalysedGameActivation.py b/modules/irwin/AnalysedGameActivation.py index 55e9c1d..c282323 100644 --- a/modules/irwin/AnalysedGameActivation.py +++ b/modules/irwin/AnalysedGameActivation.py @@ -1,54 +1,65 @@ -from collections import namedtuple +from default_imports import * -class AnalysedGameActivation(namedtuple('AnalysedGameActivation', ['id', 'userId', 'engine', 'length', 'prediction'])): +from modules.game.Game import GameID +from modules.game.Player import PlayerID +from modules.game.AnalysedGame import AnalysedGame + +from pymongo.collection import Collection + +Prediction = NewType('Prediction', int) + +class AnalysedGameActivation(NamedTuple('AnalysedGameActivation', [ + ('id', GameID), + ('playerId', PlayerID), + ('engine', bool), + ('length', int), + ('prediction', Prediction)] + )): """ - id: String (gameId) - userId: String (userId) - engine: Bool (is the user and engine?) - length: Int (length of the game in plys) - prediction: Int (Irwin's last reported confidence that this user is a cheater) Used as a pivot coll for training. """ @staticmethod - def fromGamesAnalysisandPrediction(analysedGame, prediction, engine): + def fromGamesAnalysisandPrediction(analysedGame: AnalysedGame, prediction: Prediction, engine: bool): return AnalysedGameActivation( id = analysedGame.id, - userId = analysedGame.userId, + playerId = analysedGame.playerId, engine = engine, length = len(analysedGame.analysedMoves), prediction = prediction) class AnalysedGameActivationBSONHandler: @staticmethod - def reads(bson): + def reads(bson: Dict) -> AnalysedGameActivation: return AnalysedGameActivation( id = bson['_id'], - userId = bson['userId'], + playerId = bson['playerId'], engine = bson['engine'], length = bson['length'], prediction = bson['prediction']) @staticmethod - def writes(cga): + def writes(analysedGameActivation: AnalysedGameActivation) -> Dict: return { - '_id': cga.id, - 'userId': cga.userId, - 'engine': cga.engine, - 'length': cga.length, - 'prediction': cga.prediction + '_id': analysedGameActivation.id, + 'playerId': analysedGameActivation.playerId, + 'engine': analysedGameActivation.engine, + 'length': analysedGameActivation.length, + 'prediction': analysedGameActivation.prediction } -class AnalysedGameActivationDB(namedtuple('AnalysedGameActivationDB', ['confidentAnalysedGamePivotColl'])): - def byUserId(self, userId): - return [AnalysedGameActivationBSONHandler.reads(bson) for bson in self.confidentAnalysedGamePivotColl.find({'userId': userId})] +class AnalysedGameActivationDB(NamedTuple('AnalysedGameActivationDB', [ + ('confidentAnalysedGamePivotColl', Collection) + ])): + def byPlayerId(self, playerId: PlayerID) -> List[AnalysedGameActivation]: + return [AnalysedGameActivationBSONHandler.reads(bson) for bson in self.confidentAnalysedGamePivotColl.find({'userId': playerId})] - def byEngineAndPrediction(self, engine, prediction): + def byEngineAndPrediction(self, engine: bool, prediction: Prediction) -> List[AnalysedGameActivation]: if engine: return [AnalysedGameActivationBSONHandler.reads(bson) for bson in self.confidentAnalysedGamePivotColl.find({'engine': engine, 'prediction': {'$gte': prediction}})] return [AnalysedGameActivationBSONHandler.reads(bson) for bson in self.confidentAnalysedGamePivotColl.find({'engine': engine, 'prediction': {'$lte': prediction}})] - def write(self, cga): # Game - self.confidentAnalysedGamePivotColl.update_one({'_id': cga.id}, {'$set': AnalysedGameActivationBSONHandler.writes(cga)}, upsert=True) + def write(self, analysedGameActivation: AnalysedGameActivation): + self.confidentAnalysedGamePivotColl.update_one({'_id': analysedGameActivation.id}, {'$set': AnalysedGameActivationBSONHandler.writes(analysedGameActivation)}, upsert=True) - def lazyWriteMany(self, cgas): - [self.write(cga) for cga in cgas] \ No newline at end of file + def lazyWriteMany(self, analysedGameActivations: List[AnalysedGameActivation]): + [self.write(analysedGameActivation) for analysedGameActivation in analysedGameActivations] \ No newline at end of file diff --git a/modules/irwin/AnalysedGameModel.py b/modules/irwin/AnalysedGameModel.py index d1f5056..b720e04 100644 --- a/modules/irwin/AnalysedGameModel.py +++ b/modules/irwin/AnalysedGameModel.py @@ -1,24 +1,34 @@ +from default_imports import * + +from conf.ConfigWrapper import ConfigWrapper + import numpy as np import logging import os from random import shuffle -from collections import namedtuple +from modules.game.AnalysedGame import AnalysedGame from keras.models import load_model, Model from keras.layers import Dropout, Dense, LSTM, Input, concatenate, Conv1D, Flatten from keras.optimizers import Adam -from keras.callbacks import TensorBoard + +from keras.engine.training import Model + +from numpy import ndarray from functools import lru_cache -class AnalysedGameModel(namedtuple('AnalysedGameModel', ['env'])): - @lru_cache(maxsize=2) - def model(self, newmodel=False): - if os.path.isfile('modules/irwin/models/analysedGame.h5') and not newmodel: +class AnalysedGameModel: + def __init__(self, config: ConfigWrapper, newmodel: bool = False): + self.config = config + self.model = self.createModel(newmodel) + + def createModel(self, newmodel: bool = False) -> Model: + if os.path.isfile(self.config["irwin model analysed file"]) and not newmodel: logging.debug("model already exists, opening from file") - return load_model('modules/irwin/models/analysedGame.h5') + return load_model(self.config["irwin model analysed file"]) logging.debug('model does not exist, building from scratch') inputGame = Input(shape=(60, 10), dtype='float32', name='game_input') @@ -76,100 +86,8 @@ def model(self, newmodel=False): metrics=['accuracy']) return model - def predict(self, analysedGames): - return [self.model().predict(np.array([ag.tensor()])) for ag in analysedGames] - - def train(self, epochs, filtered=True, newmodel=False): - # get player sample - logging.debug("getting model") - model = self.model(newmodel) - logging.debug("getting dataset") - batch = self.getTrainingDataset(filtered) - - logging.debug("training") - logging.debug("Batch Info: Games: " + str(len(batch['data']))) - - logging.debug("Game Len: " + str(len(batch['data'][0]))) - - tensorBoard = TensorBoard( - log_dir='./logs/analysedGameModel', - histogram_freq=10, - batch_size=32, write_graph=True) - - model.fit( - batch['data'], batch['labels'], - epochs=epochs, batch_size=32, validation_split=0.2, - callbacks=[tensorBoard]) - - self.saveModel(model) - logging.debug("complete") - - def saveModel(self, model): - logging.debug("saving model") - model.save('modules/irwin/models/analysedGame.h5') - - def getTrainingDataset(self, filtered): - if filtered: - logging.debug("gettings game IDs from DB") - cheatPivotEntries = self.env.analysedGameActivationDB.byEngineAndPrediction(True, 80) - legits = self.env.playerDB.byEngine(False) - - shuffle(cheatPivotEntries) - shuffle(legits) - - legits = legits[:10000] - - logging.debug("Getting game analyses from DB") - - legitAnalysedGames = [] - - cheatAnalysedGames = self.env.analysedGameDB.byIds([cpe.id for cpe in cheatPivotEntries]) - [legitAnalysedGames.extend(ga) for ga in self.env.analysedGameDB.byUserIds([u.id for u in legits])] - else: - logging.debug("getting players by engine") - cheats = self.env.playerDB.byEngine(True) - legits = self.env.playerDB.byEngine(False) - - shuffle(cheats) - shuffle(legits) - - cheats = cheats[:10000] - legits = legits[:10000] - - cheatAnalysedGames = [] - legitAnalysedGames = [] - - logging.debug("getting game analyses from DB") - [cheatAnalysedGames.extend(ga) for ga in self.env.analysedGameDB.byUserIds([u.id for u in cheats])] - [legitAnalysedGames.extend(ga) for ga in self.env.analysedGameDB.byUserIds([u.id for u in legits])] - - logging.debug("building tensor") - cheatGameTensors = [tga.tensor() for tga in cheatAnalysedGames if tga.gameLength() <= 60] - legitGameTensors = [tga.tensor() for tga in legitAnalysedGames if tga.gameLength() <= 60] - - logging.debug("batching tensors") - return self.createBatchAndLabels(cheatGameTensors, legitGameTensors) - - @staticmethod - def createBatchAndLabels(cheatBatch, legitBatch): - # group the dataset into batches by the length of the dataset, because numpy needs it that way - mlen = min(len(cheatBatch), len(legitBatch)) - - cheats = cheatBatch[:mlen] - legits = legitBatch[:mlen] - - logging.debug("batch size " + str(len(cheats + legits))) - - labels = [1.0]*len(cheats) + [0.0]*len(legits) - - blz = list(zip(cheats+legits, labels)) - shuffle(blz) + def predict(self, analysedGames: List[AnalysedGame]) -> List[ndarray]: + return [self.model.predict(np.array([ag.tensor()])) for ag in analysedGames] - return { - 'data': np.array([t for t, l in blz]), - 'labels': [ - np.array([l for t, l in blz]), - np.array([[[l]]*(len(t)-13) for t, l in blz]), - np.array([[[l]]*(len(t)-4) for t, l in blz]) - ] - } \ No newline at end of file + def saveModel(self): + self.model.save(self.config["irwin model analysed file"]) \ No newline at end of file diff --git a/modules/irwin/AnalysisReport.py b/modules/irwin/AnalysisReport.py index 3c6ccf5..7046558 100644 --- a/modules/irwin/AnalysisReport.py +++ b/modules/irwin/AnalysisReport.py @@ -263,7 +263,7 @@ def writes(moveReport): } class PlayerReportDB(namedtuple('PlayerReportDB', ['playerReportColl'])): - def byUserId(self, userId): + def byPlayerId(self, userId): return [PlayerReportBSONHandler.reads(bson) for bson in self.playerReportColl.find( @@ -276,7 +276,7 @@ def newestByUserId(self, userId): sort=[('date', pymongo.DESCENDING)]) return None if bson is None else PlayerReportBSONHandler.reads(bson) - def byUserIds(self, userIds): + def byPlayerIds(self, userIds): return [self.newestByUserId(userId) for userId in userIds] def newest(self, amount=50): diff --git a/modules/irwin/BasicGameModel.py b/modules/irwin/BasicGameModel.py index fecd0a8..85295b9 100644 --- a/modules/irwin/BasicGameModel.py +++ b/modules/irwin/BasicGameModel.py @@ -1,3 +1,7 @@ +from default_imports import * + +from conf.ConfigWrapper import ConfigWrapper + import numpy as np import logging import os @@ -13,12 +17,15 @@ from functools import lru_cache -class BasicGameModel(namedtuple('BasicGameModel', ['env'])): - @lru_cache(maxsize=2) - def model(self, newmodel=False): - if os.path.isfile('modules/irwin/models/basicGame.h5') and not newmodel: +class BasicGameModel: + def __init__(self, config: ConfigWrapper, newmodel: bool = False): + self.config = config + self.model = self.createModel(newmodel) + + def createModel(self, newmodel: bool = False): + if os.path.isfile(self.config["irwin model basic file"]) and not newmodel: logging.debug("model already exists, opening from file") - return load_model('modules/irwin/models/basicGame.h5') + return load_model(self.config["irwin model basic file"]) logging.debug('model does not exist, building from scratch') moveStatsInput = Input(shape=(100, 6), dtype='float32', name='move_input') @@ -64,90 +71,6 @@ def model(self, newmodel=False): metrics=['accuracy']) return model - def train(self, epochs, filtered=True, newmodel=False): - # get player sample - logging.debug("getting model") - model = self.model(newmodel) - logging.debug("getting dataset") - batch = self.getTrainingDataset(filtered) - - logging.debug("training") - logging.debug("Batch Info: Games: " + str(len(batch['data']))) - - tensorBoard = TensorBoard( - log_dir='./logs/basicGameModel', - histogram_freq=10, - batch_size=32, write_graph=True) - - model.fit( - batch['data'], batch['labels'], - epochs=epochs, batch_size=32, validation_split=0.2, - callbacks=[tensorBoard]) - - self.saveModel(model) - logging.debug("complete") - - def saveModel(self, model): + def saveModel(self): logging.debug("saving model") - model.save('modules/irwin/models/basicGame.h5') - - def getTrainingDataset(self, filtered): - logging.debug("Getting players from DB") - - cheatTensors = [] - legitTensors = [] - - logging.debug("Getting games from DB") - if filtered: - legits = self.env.playerDB.byEngine(False) - shuffle(legits) - legits = legits[:10000] - for p in legits: - legitTensors.extend([g.tensor(p.id) for g in self.env.gameDB.byUserIdAnalysed(p.id)]) - cheatGameActivations = self.env.gameBasicActivationDB.byEngineAndPrediction(True, 70) - cheatGames = self.env.gameDB.byIds([ga.gameId for ga in cheatGameActivations]) - cheatTensors.extend([g.tensor(ga.userId) for g, ga in zip(cheatGames, cheatGameActivations)]) - else: - cheats = self.env.playerDB.byEngine(True) - legits = self.env.playerDB.byEngine(False) - - shuffle(cheats) - shuffle(legits) - - cheats = cheats[:10000] - legits = legits[:10000] - - for p in legits + cheats: - if p.engine: - cheatTensors.extend([g.tensor(p.id) for g in self.env.gameDB.byUserIdAnalysed(p.id)]) - else: - legitTensors.extend([g.tensor(p.id) for g in self.env.gameDB.byUserIdAnalysed(p.id)]) - - cheatTensors = [t for t in cheatTensors if t is not None] - legitTensors = [t for t in legitTensors if t is not None] - - shuffle(cheatTensors) - shuffle(legitTensors) - - logging.debug("batching tensors") - return self.createBatchAndLabels(cheatTensors, legitTensors) - - @staticmethod - def createBatchAndLabels(cheatBatch, legitBatch): - # group the dataset into batches by the length of the dataset, because numpy needs it that way - mlen = min(len(cheatBatch), len(legitBatch)) - - cheats = cheatBatch[:mlen] - legits = legitBatch[:mlen] - - logging.debug("batch size " + str(len(cheats + legits))) - - labels = [1]*len(cheats) + [0]*len(legits) - - blz = list(zip(cheats+legits, labels)) - shuffle(blz) - - return { - 'data': np.array([t for t, l in blz]), - 'labels': np.array([l for t, l in blz]) - } \ No newline at end of file + self.model.save(config["irwin model basic file"]) \ No newline at end of file diff --git a/modules/irwin/GameBasicActivation.py b/modules/irwin/GameBasicActivation.py index 5604857..a6b7854 100644 --- a/modules/irwin/GameBasicActivation.py +++ b/modules/irwin/GameBasicActivation.py @@ -1,63 +1,67 @@ """Type used for pivot coll for basic game model training""" -from collections import namedtuple +from default_imports import * -class GameBasicActivation(namedtuple('GameBasicActivation', ['id', 'gameId', 'userId', 'engine', 'prediction'])): +from modules.game.Game import GameID, PlayerID + +from pymongo.collection import Collection + +GameBasicActivationID = NewType('GameBasicActivationID', str) +Prediction = NewType('Prediction', int) + +class GameBasicActivation(NamedTuple('GameBasicActivation', [ + ('id', GameBasicActivationID), + ('gameId', GameID), + ('playerId', PlayerID), + ('engine', bool), + ('prediction', int) + ])): @staticmethod - def fromPrediction(gameId, userId, prediction, engine): + def fromPrediction(gameId: GameID, playerId: PlayerID, prediction: Prediction, engine: bool) -> GameBasicActivation: return GameBasicActivation( - id = gameId + '/' + userId, + id = gameId + '/' + playerId, gameId = gameId, - userId = userId, + playerId = playerId, engine = engine, prediction = prediction ) - def color(self): - darkColours = [ - 'rgba(84, 231, 96, 0.8)', - 'rgba(109, 231, 84, 0.8)', - 'rgba(131, 231, 84, 0.8)', - 'rgba(163, 231, 84, 0.8)', - 'rgba(197, 231, 84, 0.8)', - 'rgba(231, 229, 84, 0.8)', - 'rgba(231, 194, 84, 0.8)', - 'rgba(231, 158, 84, 0.8)', - 'rgba(231, 118, 84, 0.8)', - 'rgba(231, 84, 84, 0.8)'] - - return darkColours[int(self.prediction/10)] + @staticmethod + def makeId(gameId: GameID, playerId: PlayerID) -> GameBasicActivationID: + return gameId + '/' + playerId class GameBasicActivationBSONHandler: @staticmethod - def reads(bson): + def reads(bson: Dict) -> GameBasicActivation: return GameBasicActivation( id = bson['_id'], gameId = bson['gameId'], - userId = bson['userId'], + playerId = bson['playerId'], engine = bson['engine'], prediction = bson['prediction']) @staticmethod - def writes(gba): + def writes(gba: GameBasicActivation) -> Dict: return { '_id': gba.id, 'gameId': gba.gameId, - 'userId': gba.userId, + 'playerId': gba.playerId, 'engine': gba.engine, 'prediction': gba.prediction } -class GameBasicActivationDB(namedtuple('GameBasicActivationDB', ['gameBasicActivationColl'])): - def byUserId(self, userId): - return [GameBasicActivationBSONHandler.reads(bson) for bson in self.gameBasicActivationColl.find({'userId': userId})] +class GameBasicActivationDB(NamedTuple('GameBasicActivationDB', [ + ('gameBasicActivationColl', Collection) + ])): + def byPlayerId(self, playerId: PlayerID) -> List[GameBasicActivation]: + return [GameBasicActivationBSONHandler.reads(bson) for bson in self.gameBasicActivationColl.find({'playerId': playerId})] - def byEngineAndPrediction(self, engine, prediction): + def byEngineAndPrediction(self, engine: bool, prediction: Prediction) -> List[GameBasicActivation]: if engine: return [GameBasicActivationBSONHandler.reads(bson) for bson in self.gameBasicActivationColl.find({'engine': engine, 'prediction': {'$gte': prediction}})] return [GameBasicActivationBSONHandler.reads(bson) for bson in self.gameBasicActivationColl.find({'engine': engine, 'prediction': {'$lte': prediction}})] - def write(self, gba): # Game + def write(self, gba: GameBasicActivation): self.gameBasicActivationColl.update_one({'_id': gba.id}, {'$set': GameBasicActivationBSONHandler.writes(gba)}, upsert=True) - def lazyWriteMany(self, gbas): + def lazyWriteMany(self, gbas: List[GameBasicActivation]): [self.write(gba) for gba in gbas] \ No newline at end of file diff --git a/modules/irwin/Irwin.py b/modules/irwin/Irwin.py index 4f7865d..9b03363 100644 --- a/modules/irwin/Irwin.py +++ b/modules/irwin/Irwin.py @@ -137,7 +137,7 @@ def discover(self): sus = [] for i, player in enumerate(players): logging.debug("investigating "+player.id + " " + str(i) + "/" + totalPlayers) - gameStore = GameStore([], [ga for ga in self.env.analysedGameDB.byUserId(player.id)]) + gameStore = GameStore([], [ga for ga in self.env.analysedGameDB.byPlayerId(player.id)]) predictions = self.predictAnalysed(gameStore.analysedGameTensors()) activation = self.activation(player, predictions) logging.debug(str(activation)) @@ -158,7 +158,7 @@ def buildAnalysedTable(self): for i, p in enumerate(cheats): logging.info("predicting: " + p.id + " - " + str(i) + '/' + lenPlayers) - analysedGames = self.env.analysedGameDB.byUserId(p.id) + analysedGames = self.env.analysedGameDB.byPlayerId(p.id) tensors = [(ga.tensor(), ga.length()) for ga in analysedGames] predictions = self.predictAnalysed(tensors) analysedGameActivations = [AnalysedGameActivation.fromGamesAnalysisandPrediction( @@ -178,7 +178,7 @@ def buildBasicTable(self): logging.info("getting games and predicting") for i, p in enumerate(cheats): logging.info("predicting: " + p.id + " - " + str(i) + "/" + lenPlayers) - gameStore = GameStore(self.env.gameDB.byUserIdAnalysed(p.id), []) + gameStore = GameStore(self.env.gameDB.byPlayerIdAnalysed(p.id), []) gameTensors = gameStore.gameTensors(p.id) if len(gameTensors) > 0: gamePredictions = self.predictBasicGames(gameTensors) diff --git a/modules/irwin/training/AnalysedModelTraining.py b/modules/irwin/training/AnalysedModelTraining.py new file mode 100644 index 0000000..f3435c3 --- /dev/null +++ b/modules/irwin/training/AnalysedModelTraining.py @@ -0,0 +1,99 @@ +from default_imports import * + +from modules.irwin.training.Env import Env +from modules.game.AnalysedGame import AnalysedGameTensor +from modules.irwin.AnalysedGameModel import AnalysedGameModel + +from numpy import ndarray + +from random import shuffe + +Batch = NamedTuple('Batch', [ + ('data', ndarray), + ('labels', List[ndarray]) + ]) + +class AnalysedModelTraining(NamedTuple('AnalysedModelTraining', [ + ('env', Env), + ('analysedGameModel', AnalysedGameModel) + ])): + def train(self, epochs: int, filtered: bool = True) -> None: + logging.debug("getting dataset") + batch = self.getTrainingDataset(filtered) + + logging.debug("training") + logging.debug("Batch Info: Games: {}".format(len(batch.data))) + + logging.debug("Game Len: {}".format(len(batch.data[0]))) + + self.analysedGameModel.model.fit( + batch.data, batch.labels, + epochs=epochs, batch_size=32, validation_split=0.2) + + self.analysedGameModel.saveModel() + logging.debug("complete") + + def getTrainingDataset(self, filtered: bool): + if filtered: + logging.debug("gettings game IDs from DB") + cheatPivotEntries = self.env.analysedGameActivationDB.byEngineAndPrediction(True, 80) + legits = self.env.playerDB.byEngine(False) + + shuffle(cheatPivotEntries) + shuffle(legits) + + legits = legits[:10000] + + logging.debug("Getting game analyses from DB") + + legitAnalysedGames = [] + + cheatAnalysedGames = self.env.analysedGameDB.byIds([cpe.id for cpe in cheatPivotEntries]) + [legitAnalysedGames.extend(ga) for ga in self.env.analysedGameDB.byPlayerIds([u.id for u in legits])] + else: + logging.debug("getting players by engine") + cheats = self.env.playerDB.byEngine(True) + legits = self.env.playerDB.byEngine(False) + + shuffle(cheats) + shuffle(legits) + + cheats = cheats[:10000] + legits = legits[:10000] + + cheatAnalysedGames = [] + legitAnalysedGames = [] + + logging.debug("getting game analyses from DB") + [cheatAnalysedGames.extend(ga) for ga in self.env.analysedGameDB.byPlayerIds([u.id for u in cheats])] + [legitAnalysedGames.extend(ga) for ga in self.env.analysedGameDB.byPlayerIds([u.id for u in legits])] + + logging.debug("building tensor") + cheatGameTensors = [tga.tensor() for tga in cheatAnalysedGames if tga.gameLength() <= 60] + legitGameTensors = [tga.tensor() for tga in legitAnalysedGames if tga.gameLength() <= 60] + + logging.debug("batching tensors") + return self.createBatchAndLabels(cheatGameTensors, legitGameTensors) + + @staticmethod + def createBatchAndLabels(cheatTensors: List[AnalysedGameTensor], legitTensors: List[AnalysedGameTensor]) -> Batch: + # group the dataset into batches by the length of the dataset, because numpy needs it that way + mlen = min(len(cheatTensors), len(legitTensors)) + + cheats = cheatTensors[:mlen] + legits = legitTensors[:mlen] + + logging.debug("batch size " + str(len(cheats + legits))) + + labels = [1.0]*len(cheats) + [0.0]*len(legits) + + blz = list(zip(cheats+legits, labels)) + shuffle(blz) + + return Batch( + data=np.array([t for t, l in blz]), + labels=[ + np.array([l for t, l in blz]), + np.array([[[l]]*(len(t)-13) for t, l in blz]), + np.array([[[l]]*(len(t)-4) for t, l in blz]) + ]) \ No newline at end of file diff --git a/modules/irwin/training/BasicModelTraining.py b/modules/irwin/training/BasicModelTraining.py new file mode 100644 index 0000000..eae0720 --- /dev/null +++ b/modules/irwin/training/BasicModelTraining.py @@ -0,0 +1,95 @@ +from default_imports import * + +from modules.game.Game import GameTensor + +from modules.irwin.training.Env import Env +from modules.irwin.BasicGameModel import BasicGameModel + +from numpy import ndarray + +from random import shuffle + +Batch = NamedTuple('Batch', [ + ('data', ndarray), + ('labels', ndarray) + ]) + +class BasicModelTraining(NamedTuple('BasicModelTraining', [ + ('env', Env), + ('basicGameModel', BasicGameModel) + ])): + def train(self, epochs: int, filtered: bool = True, newmodel: bool = False): + logging.debug("getting dataset") + batch = self.getTrainingDataset(filtered) + + logging.debug("training") + logging.debug("Batch Info: Games: {}".format(len(batch.data))) + + self.basicGameModel.model.fit( + batch.data, batch.labels, + epochs=epochs, batch_size=32, validation_split=0.2) + + self.basicGameModel.saveModel() + logging.debug("complete") + + def getTrainingDataset(self, filtered: bool): + logging.debug("Getting players from DB") + + cheatTensors = [] + legitTensors = [] + + logging.debug("Getting games from DB") + if filtered: + legits = self.env.playerDB.byEngine(False) + shuffle(legits) + legits = legits[:10000] + for p in legits: + legitTensors.extend([g.tensor(p.id) for g in self.env.gameDB.byPlayerIdAnalysed(p.id)]) + cheatGameActivations = self.env.gameBasicActivationDB.byEngineAndPrediction(True, 70) + cheatGames = self.env.gameDB.byIds([ga.gameId for ga in cheatGameActivations]) + cheatTensors.extend([g.tensor(ga.userId) for g, ga in zip(cheatGames, cheatGameActivations)]) + + else: + cheats = self.env.playerDB.byEngine(True) + legits = self.env.playerDB.byEngine(False) + + shuffle(cheats) + shuffle(legits) + + cheats = cheats[:10000] + legits = legits[:10000] + + for p in legits + cheats: + if p.engine: + cheatTensors.extend([g.tensor(p.id) for g in self.env.gameDB.byPlayerIdAnalysed(p.id)]) + else: + legitTensors.extend([g.tensor(p.id) for g in self.env.gameDB.byPlayerIdAnalysed(p.id)]) + + cheatTensors = [t for t in cheatTensors if t is not None] + legitTensors = [t for t in legitTensors if t is not None] + + shuffle(cheatTensors) + shuffle(legitTensors) + + logging.debug("batching tensors") + return self.createBatchAndLabels(cheatTensors, legitTensors) + + @staticmethod + def createBatchAndLabels(cheatTensors: List[GameTensor], legitTensors: List[GameTensor]) -> Batch: + # group the dataset into batches by the length of the dataset, because numpy needs it that way + mlen = min(len(cheatTensors), len(legitTensors)) + + cheats = cheatTensors[:mlen] + legits = legitTensors[:mlen] + + logging.debug("batch size " + str(len(cheats + legits))) + + labels = [1]*len(cheats) + [0]*len(legits) + + blz = list(zip(cheats+legits, labels)) + shuffle(blz) + + return Batch( + data = np.array([t for t, l in blz]), + labels = np.array([l for t, l in blz]) + ) \ No newline at end of file diff --git a/modules/irwin/training/Env.py b/modules/irwin/training/Env.py new file mode 100644 index 0000000..5250210 --- /dev/null +++ b/modules/irwin/training/Env.py @@ -0,0 +1,19 @@ +from default_imports import * + +from conf.ConfigWrapper import ConfigWrapper + +from pymongo.database import Database + +from modules.game.Player import PlayerDB +from modules.game.AnalysedGame import AnalysedGameDB + +from modules.irwin.AnalysedGameActivation import AnalysedGameActivationDB + +class Env: + def __init__(self, config: ConfigWrapper, db: Database): + self.config = config + self.db = db + + self.playerDB = PlayerDB(db[self.config["game coll player"]]) + self.analysedGameDB = AnalysedGameDB(db[self.config["game coll analysed_game"]]) + self.analysedGameActivationDB = AnalysedGameActivationDB(db[self.config["irwin coll analysed_game_activation"]]) \ No newline at end of file diff --git a/modules/irwin/Evaluation.py b/modules/irwin/training/Evaluation.py similarity index 96% rename from modules/irwin/Evaluation.py rename to modules/irwin/training/Evaluation.py index 8953598..b8f5167 100644 --- a/modules/irwin/Evaluation.py +++ b/modules/irwin/training/Evaluation.py @@ -9,7 +9,7 @@ def getEvaluationDataset(self, batchSize): players = self.env.playerDB.balancedSample(batchSize) print(" %d" % len(players)) print("getting game analyses") - analysesByPlayer = [(player, GameStore([], [ga for ga in self.env.analysedGameDB.byUserId(player.id)])) for player in players] + analysesByPlayer = [(player, GameStore([], [ga for ga in self.env.analysedGameDB.byPlayerId(player.id)])) for player in players] return analysesByPlayer def evaluate(self): diff --git a/modules/irwin/training/Training.py b/modules/irwin/training/Training.py new file mode 100644 index 0000000..cf4c2e1 --- /dev/null +++ b/modules/irwin/training/Training.py @@ -0,0 +1,16 @@ +from default_imports import * + +from conf.ConfigWrapper import ConfigWrapper + +from modules.irwin.training.Env import Env + +from modules.irwin.training.AnalysedModelTraining import AnalysedModelTraining +from modules.irwin.training.BasicModelTraining import BasicModelTraining + +from modules.irwin.AnalysedGameModel import AnalysedGameModel +from modules.irwin.BasicGameModel import BasicGameModel + +class Training: + def __init__(self, env: Env): + self.analysedModelTraining = AnalysedModelTraining(env = env, analysedGameModel = AnalysedGameModel(env.config)) + self.basicModelTraining = BasicModelTraining(env = env, basicGameModel = BasicGameModel(env.config)) \ No newline at end of file diff --git a/modules/queue/EngineQueue.py b/modules/queue/EngineQueue.py index f62877b..3772612 100644 --- a/modules/queue/EngineQueue.py +++ b/modules/queue/EngineQueue.py @@ -1,33 +1,51 @@ """Queue item for basic analysis by irwin""" -from collections import namedtuple +from default_imports import * + +from modules.auth.Auth import AuthID +from modules.game.Game import PlayerID, GameID +from modules.queue.Origin import Origin, OriginReport, OriginModerator, OriginRandom + from datetime import datetime, timedelta -from math import ceil + import pymongo +from pymongo.collection import Collection + import numpy as np +from math import ceil -class EngineQueue(namedtuple('EngineQueue', ['id', 'origin', 'precedence', 'progress', 'complete', 'owner', 'date'])): +EngineQueueID = NewType('EngineQueueID', str) +Precedence = NewType('Precedence', int) + +class EngineQueue(NamedTuple('EngineQueue', [ + ('id', EngineQueueID), + ('origin', Origin), + ('gameIds', List[GameID]) + ('precedence', Precedence), + ('complete', bool), + ('owner', AuthID), + ('date', datetime) + ])): @staticmethod - def new(userId, origin, gamePredictions): + def new(playerId: PlayerID, origin: Origin, gamePredictions) -> EngineQueue: if len(gamePredictions) > 0: activations = sorted([(a[1]*a[1]) for a in gamePredictions], reverse=True) top30avg = ceil(np.average(activations[:ceil(0.3*len(activations))])) else: top30avg = 0 precedence = top30avg - if origin == 'report': + if origin == OriginReport: precedence += 5000 - elif origin == 'moderator': + elif origin == OriginModerator: precedence = 100000 return EngineQueue( - id=userId, + id=playerId, origin=origin, precedence=precedence, owner=None, - progress=0, complete=False, date=datetime.now()) - def json(self): + def json(self) -> Dict: return { 'id': self.id, 'origin': self.origin, @@ -38,90 +56,87 @@ def json(self): 'date': "{:%d %b %Y at %H:%M}".format(self.date) } - def __complete__(self): + def complete(self) -> EngineQueue: return EngineQueue( id=self.id, origin=self.origin, precedence=self.precedence, - progress=100, complete=True, owner=self.owner, date=self.date) class EngineQueueBSONHandler: @staticmethod - def reads(bson): + def reads(bson: Dict) -> EngineQueue: return EngineQueue( id=bson['_id'], origin=bson['origin'], precedence=bson['precedence'], - progress=bson.get('progress', 0), + gameIds=bson.get('gameIds'), complete=bson.get('complete', False), owner=bson.get('owner'), date=bson.get('date')) @staticmethod - def writes(engineQueue): + @validate + def writes(engineQueue: EngineQueue) -> Dict: return { '_id': engineQueue.id, 'origin': engineQueue.origin, 'precedence': engineQueue.precedence, - 'progress': engineQueue.progress, + 'gameIds': engineQueue.gameIds, 'complete': engineQueue.complete, 'owner': engineQueue.owner, 'date': datetime.now() } -class EngineQueueDB(namedtuple('EngineQueueDB', ['engineQueueColl'])): - def write(self, engineQueue): +class EngineQueueDB(NamedTuple('EngineQueueDB', [ + ('engineQueueColl', Collection) + ])): + def write(self, engineQueue: EngineQueue): self.engineQueueColl.update_one( {'_id': engineQueue.id}, {'$set': EngineQueueBSONHandler.writes(engineQueue)}, upsert=True) - def updateProgress(self, _id, progress): - self.engineQueueColl.update_one( - {'_id': _id}, - {'$set': {'progress': progress}}) - - def inProgress(self): + def inProgress(self) -> List[EngineQueue]: return [EngineQueueBSONHandler.reads(bson) for bson in self.engineQueueColl.find({'owner': {'$ne': None}, 'complete': False})] - def byId(self, _id): + def byId(self, _id: EngineQueueID) -> Opt[EngineQueue]: bson = self.engineQueueColl.find_one({'_id': _id}) return None if bson is None else EngineQueueBSONHandler.reads(bson) - def complete(self, engineQueue): + def complete(self, engineQueue: EngineQueue): """remove a complete job from the queue""" - self.write(engineQueue.__complete__()) + self.write(engineQueue.complete()) - def updateComplete(self, _id, complete): + def updateComplete(self, _id: EngineQueueID, complete: bool): self.engineQueueColl.update_one( {'_id': _id}, {'$set': {'complete': complete}}) - def removeUserId(self, userId): - """remove all jobs related to userId""" - self.engineQueueColl.remove({'_id': userId}) + def removePlayerId(self, playerId: PlayerID): + """remove all jobs related to playerId""" + self.engineQueueColl.remove({'_id': playerId}) - def exists(self, userId): - """userId has a engineQueue object against their name""" - return self.engineQueueColl.find_one({'_id': userId}) is not None + def exists(self, playerId: PlayerID) -> bool: + """playerId has a engineQueue object against their name""" + return self.engineQueueColl.find_one({'_id': playerId}) is not None - def owned(self, userId): - """Does any deep player queue for userId have an owner""" - bson = self.engineQueueColl.find_one({'_id': userId, 'owner': None}) + def owned(self, playerId: PlayerID) -> bool: + """Does any deep player queue for playerId have an owner""" + bson = self.engineQueueColl.find_one({'_id': playerId, 'owner': None}) hasOwner = False if bson is not None: hasOwner = bson['owner'] is not None return hasOwner - def oldest(self): + def oldest(self) -> Opt[EngineQueue]: bson = self.engineQueueColl.find_one( filter={'date': {'$lt': datetime.now() - timedelta(days=2)}}, sort=[('date', pymongo.ASCENDING)]) return None if bson is None else EngineQueueBSONHandler.reads(bson) - def nextUnprocessed(self, name): + def nextUnprocessed(self, name: AuthID) -> Opt[EngineQueue]: """find the next job to process against owner's name""" incompleteBSON = self.engineQueueColl.find_one({'owner': name, '$or': [{'complete': {'$exists': False}}, {'complete': False}]}) if incompleteBSON is not None: # owner has unfinished business @@ -134,7 +149,7 @@ def nextUnprocessed(self, name): ("date", pymongo.ASCENDING)]) return None if engineQueueBSON is None else EngineQueueBSONHandler.reads(engineQueueBSON) - def top(self, amount=20): + def top(self, amount: int = 20) -> List[EngineQueue]: """Return the top `amount` of players, ranked by precedence""" bsons = self.engineQueueColl.find( filter={'complete': False}, diff --git a/modules/queue/Env.py b/modules/queue/Env.py index 0a7318a..1ba143c 100644 --- a/modules/queue/Env.py +++ b/modules/queue/Env.py @@ -1,8 +1,15 @@ +from default_imports import * + +from conf.ConfigWrapper import ConfigWrapper + +from pymongo.collection import Collection + from modules.queue.EngineQueue import EngineQueueDB from modules.queue.IrwinQueue import IrwinQueueDB class Env: - def __init__(self, config, db): + def __init__(self, config: ConfigWrapper, db: Collection): self.db = db - self.engineQueueDB = EngineQueueDB(db[config['queue']['coll']['engine_queue']]) - self.irwinQueueDB = IrwinQueueDB(db[config['queue']['coll']['irwin_queue']]) \ No newline at end of file + + self.engineQueueDB = EngineQueueDB(db[config['queue coll engine_queue']]) + self.irwinQueueDB = IrwinQueueDB(db[config['queue coll irwin_queue']]) \ No newline at end of file diff --git a/modules/queue/IrwinQueue.py b/modules/queue/IrwinQueue.py index 2e7c189..6de9b2e 100644 --- a/modules/queue/IrwinQueue.py +++ b/modules/queue/IrwinQueue.py @@ -1,36 +1,46 @@ """Queue item for deep analysis by irwin""" -from collections import namedtuple +from default_imports import * + +from modules.queue.Origin import Origin +from modules.game.Game import PlayerID + from datetime import datetime import pymongo +from pymongo.collection import Collection -IrwinQueue = namedtuple('IrwinQueue', ['id', 'origin']) +IrwinQueue = NamedTuple('IrwinQueue', [ + ('id', PlayerID), + ('origin', Origin) + ]) class IrwinQueueBSONHandler: @staticmethod - def reads(bson): + def reads(bson: Dict) -> IrwinQueue: return IrwinQueue( id=bson['_id'], origin=bson['origin']) @staticmethod - def writes(irwinQueue): + def writes(irwinQueue: IrwinQueue) -> Dict: return { '_id': irwinQueue.id, 'origin': irwinQueue.origin, 'date': datetime.now() } -class IrwinQueueDB(namedtuple('IrwinQueueDB', ['irwinQueueColl'])): - def write(self, irwinQueue): +class IrwinQueueDB(NamedTuple('IrwinQueueDB', [ + ('irwinQueueColl', Collection) + ])): + def write(self, irwinQueue: IrwinQueue): self.irwinQueueColl.update_one( {'_id': irwinQueue.id}, {'$set': IrwinQueueBSONHandler.writes(irwinQueue)}, upsert=True) - def removeUserId(self, userId): - self.irwinQueueColl.remove({'_id': userId}) + def removePlayerId(self, playerId: PlayerID): + self.irwinQueueColl.remove({'_id': playerId}) - def nextUnprocessed(self): + def nextUnprocessed(self) -> Opt[IrwinQueue]: irwinQueueBSON = self.irwinQueueColl.find_one_and_delete( filter={}, sort=[("date", pymongo.ASCENDING)]) diff --git a/modules/queue/Origin.py b/modules/queue/Origin.py new file mode 100644 index 0000000..4dc4f4c --- /dev/null +++ b/modules/queue/Origin.py @@ -0,0 +1,6 @@ +from default_imports import * + +Origin = NewType('Origin', str) +OriginReport = Origin('report') +OriginModerator = Origin('moderator') +OriginRandom = Origin('random') \ No newline at end of file diff --git a/modules/queue/Queue.py b/modules/queue/Queue.py index ed89624..f985c0e 100644 --- a/modules/queue/Queue.py +++ b/modules/queue/Queue.py @@ -1,11 +1,16 @@ -from collections import namedtuple +from default_imports import * -class Queue(namedtuple('Queue', ['env'])): - def nextEngineAnalysis(self, token): - return self.env.engineAnalysisQueueDB.nextUnprocessed(token.id) +from modules.queue.Env import Env +from modules.queue.EngineQueue import EngineQueueID - def completeEngineAnalysis(self, _id): +from modules.auth.Auth import Authable + +class Queue(NamedTuple('Queue', [('env', Env)])): + def nextEngineAnalysis(self, authable: Authable): + return self.env.engineAnalysisQueueDB.nextUnprocessed(authable.id) + + def completeEngineAnalysis(self, _id: EngineQueueID): return self.env.engineAnalysisQueueDB.updateComplete(_id, complete=True) - def nextIrwinAnalysis(self): + def nextIrwinAnalysis(self): return self.env.irwinAnalysisQueueDB. \ No newline at end of file diff --git a/scan-update.py b/scan-update.py index 4d8a71b..c29b967 100644 --- a/scan-update.py +++ b/scan-update.py @@ -16,7 +16,7 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--quiet", dest="loglevel", - default=logging.DEBUG, action="store_const", const=logging.INFO, + default=logging.DEBUG, action="store_const", const=logging.INFO, help="reduce the number of logged messages") config = parser.parse_args() @@ -33,22 +33,22 @@ env = Env(config, engine=False) -def updatePlayerData(userId): - playerData = env.api.getPlayerData(userId) +def updatePlayerData(playerId): + playerData = env.api.getPlayerData(playerId) if playerData is None: - logging.warning("getPlayerData returned None for " + userId) + logging.warning("getPlayerData returned None for " + playerId) return None player = Player.fromPlayerData(playerData) env.playerDB.write(player) - env.gameDB.lazyWriteGames(Game.fromPlayerData(playerData)) + env.gameDB.lazyWriteMany(Game.fromPlayerData(playerData)) return player -def predictPlayer(userId): +def predictPlayer(playerId): gameStore = GameStore.new() - gameStore.addGames(env.gameDB.byUserIdAnalysed(userId)) - gameTensors = gameStore.gameTensors(userId) + gameStore.addGames(env.gameDB.byPlayerIdAnalysed(playerId)) + gameTensors = gameStore.gameTensors(playerId) if len(gameTensors) > 0: return env.irwin.predictBasicGames(gameTensors) @@ -61,8 +61,8 @@ def updateOldestPlayerQueue(): if deepPlayerQueue is not None: if deepPlayerQueue.owner is None: - userId = deepPlayerQueue.id - player = updatePlayerData(userId) + playerId = deepPlayerQueue.id + player = updatePlayerData(playerId) if player is None: logging.info("No player data") @@ -70,20 +70,20 @@ def updateOldestPlayerQueue(): if player.engine: logging.info("Player is now engine. Closing all reports.") - env.modReportDB.close(userId) + env.modReportDB.close(playerId) if deepPlayerQueue.origin != 'moderator': logging.info("And not requested by a moderator. Closing analysis") - env.deepPlayerQueueDB.removeUserId(userId) + env.deepPlayerQueueDB.removePlayerId(playerId) return - predictions = predictPlayer(userId) + predictions = predictPlayer(playerId) if predictions is None: logging.info("No predictions to process. Removing Queue Item") - env.deepPlayerQueueDB.removeUserId(player.id) + env.deepPlayerQueueDB.removePlayerId(player.id) return deepPlayerQueue = DeepPlayerQueue.new( - userId=userId, + playerId=playerId, origin=deepPlayerQueue.origin, gamePredictions=predictions) @@ -95,15 +95,15 @@ def updateOldestPlayerQueue(): env.deepPlayerQueueDB.write(deepPlayerQueue) else: logging.info("DeepPlayerQueue is insignificant. Removing") - env.deepPlayerQueueDB.removeUserId(userId) + env.deepPlayerQueueDB.removePlayerId(playerId) def spotCheck(): logging.info("--Spot check on player DB--") randomPlayer = env.playerDB.oldestNonEngine() logging.info("Player: " + str(randomPlayer)) if randomPlayer is not None: - userId = randomPlayer.id - player = updatePlayerData(userId) + playerId = randomPlayer.id + player = updatePlayerData(playerId) if player is None: logging.info("No player data") @@ -113,14 +113,14 @@ def spotCheck(): logging.info("Diced didn't roll lucky. Player is engine. Not proceeding") return - predictions = predictPlayer(userId) + predictions = predictPlayer(playerId) if predictions is None: logging.info("No predictions to process") return deepPlayerQueue = DeepPlayerQueue.new( - userId=userId, + playerId=playerId, origin='random', gamePredictions=predictions) @@ -135,13 +135,13 @@ def updateOldestReport(): report = env.modReportDB.oldestUnprocessed() logging.info("Report: " + str(report)) if report is not None: - userId = report.id + playerId = report.id - if env.deepPlayerQueueDB.owned(userId): + if env.deepPlayerQueueDB.owned(playerId): logging.info("DeepPlayerQueue object exists and has owner") return - player = updatePlayerData(userId) + player = updatePlayerData(playerId) if player is None: logging.info("No player data") @@ -149,10 +149,10 @@ def updateOldestReport(): if player.engine: logging.info("Player is now engine") - env.modReportDB.close(userId) + env.modReportDB.close(playerId) return - predictions = predictPlayer(userId) + predictions = predictPlayer(playerId) if predictions is None: logging.info("No predictions to process") @@ -160,10 +160,10 @@ def updateOldestReport(): if player.reportScore is None: env.modReportDB.close(player.id) - env.deepPlayerQueueDB.removeUserId(player.id) + env.deepPlayerQueueDB.removePlayerId(player.id) else: deepPlayerQueue = DeepPlayerQueue.new( - userId=userId, + playerId=playerId, origin='reportupdate', gamePredictions=predictions) diff --git a/tools.py b/tools.py index 340efd3..2de8fb3 100644 --- a/tools.py +++ b/tools.py @@ -22,43 +22,43 @@ parser = argparse.ArgumentParser(description=__doc__) ## Training parser.add_argument("--trainbasic", dest="trainbasic", nargs="?", - default=False, const=True, help="train basic game model") + default=False, const=True, help="train basic game model") parser.add_argument("--trainanalysed", dest="trainanalysed", nargs="?", - default=False, const=True, help="train analysed game model") + default=False, const=True, help="train analysed game model") parser.add_argument("--filtered", dest="filtered", nargs="?", - default=False, const=True , help="use filtered dataset for training") + default=False, const=True , help="use filtered dataset for training") parser.add_argument("--newmodel", dest="newmodel", nargs="?", - default=False, const=True, help="throw out current model. build new") + default=False, const=True, help="throw out current model. build new") ## Database building parser.add_argument("--buildbasictable", dest="buildbasictable", nargs="?", - default=False, const=True, + default=False, const=True, help="build table of basic game activations") parser.add_argument("--buildanalysedtable", dest="buildanalysedtable", nargs="?", - default=False, const=True, + default=False, const=True, help="build table of analysed game activations") parser.add_argument("--buildpositiontable", dest="buildpositiontable", nargs="?", - default=False, const=True, + default=False, const=True, help="build table of analysed positions") parser.add_argument("--updatedatabase", dest="updatedatabase", nargs="?", - default=False, const=True, + default=False, const=True, help="collect game analyses for players. Build database collection") parser.add_argument("--buildaveragereport", dest="buildaveragereport", nargs="?", - default=False, const=True, + default=False, const=True, help="build an average report for all players in the database") ## Evaluation and testing parser.add_argument("--eval", dest="eval", nargs="?", - default=False, const=True, + default=False, const=True, help="evaluate the performance of neural networks") parser.add_argument("--test", dest="test", nargs="?", - default=False, const=True, help="test on a single player") + default=False, const=True, help="test on a single player") parser.add_argument("--discover", dest="discover", nargs="?", - default=False, const=True, + default=False, const=True, help="search for cheaters in the database that haven't been marked") parser.add_argument("--quiet", dest="loglevel", - default=logging.DEBUG, action="store_const", const=logging.INFO, + default=logging.DEBUG, action="store_const", const=logging.INFO, help="reduce the number of logged messages") config = parser.parse_args() @@ -96,10 +96,10 @@ # test on a single user in the DB if config.test: for userId in ['ralph27_velasco']: - player = env.playerDB.byUserId(userId) + player = env.playerDB.byPlayerId(userId) gameStore = GameStore.new() - gameStore.addGames(env.gameDB.byUserIdAnalysed(userId)) - gameStore.addAnalysedGames(env.analysedGameDB.byUserId(userId)) + gameStore.addGames(env.gameDB.byPlayerIdAnalysed(userId)) + gameStore.addAnalysedGames(env.analysedGameDB.byPlayerId(userId)) env.api.postReport(env.irwin.report(player, gameStore)) logging.debug("posted") diff --git a/utils/updatePlayerDatabase.py b/utils/updatePlayerDatabase.py index 6d9ec71..0866849 100644 --- a/utils/updatePlayerDatabase.py +++ b/utils/updatePlayerDatabase.py @@ -13,4 +13,4 @@ def updatePlayerDatabase(env): playerData = env.api.getPlayerData(p.id) if playerData is not None: env.playerDB.write(Player.fromPlayerData(playerData)) - env.gameDB.lazyWriteGames(Game.fromPlayerData(playerData)) \ No newline at end of file + env.gameDB.lazyWriteMany(Game.fromPlayerData(playerData)) \ No newline at end of file diff --git a/webapp/DefaultResponse.py b/webapp/DefaultResponse.py new file mode 100644 index 0000000..82f1dc5 --- /dev/null +++ b/webapp/DefaultResponse.py @@ -0,0 +1,17 @@ +from flask import Response, json + +Success = Response( + response=json({ + 'success': True, + 'message': 'action completed successfully' + }), + status=200, + mimetype='application/json') + +BadRequest = Response( + response=json({ + 'success': False, + 'message': 'bad request' + }), + status=200, + mimetype='application/json') \ No newline at end of file diff --git a/webapp/controllers/api/blueprint.py b/webapp/controllers/api/blueprint.py index 3a614a7..3e33f59 100644 --- a/webapp/controllers/api/blueprint.py +++ b/webapp/controllers/api/blueprint.py @@ -1,49 +1,53 @@ import logging -from flask import Blueprint, request, jsonify, json, Response - -default_responses = { - 'success': Response( - response=json({ - 'success': True, - 'message': 'action completed successfully' - }), - status=200, - mimetype='application/json' - ), - 'bad_request': Response( - response=json({ - 'success': False, - 'message': 'bad request' - }), - status=200, - mimetype='application/json') -} +from flask import Blueprint, request, jsonify, json +from webapp import DefaultResponse + +from modules.game.Player import Player, Game +from modules.auth.Priv import RequestJob, CompleteJob, PostJob +from modules.queue.Origin import OriginReport, OriginModerator, OriginRandom def buildApiBlueprint(env): apiBlueprint = Blueprint('Api', __name__, url_prefix='/api') - @apiBlueprint.route('/request_job', methods=['GET', 'POST']) - def api_request_job(): + @apiBlueprint.route('/request_job', methods=['GET']) + def apiRequestJob(): req = request.get_json(silent=True) - account, authorised = env.auth.authoriseReq(req, 'request_job') + authable, authorised = env.auth.authoriseReq(req, RequestJob) if authorised: - logging.info(str(account) + ' is authorised') - return env.queue.nextDeepAnalysis(account.id) + logging.info(str(authable) + ' is authorised') + return env.queue.nextDeepAnalysis(authable.id) - @apiBlueprint.route('/complete_job', method=['GET', 'POST']) - def api_complete_job(): + @apiBlueprint.route('/complete_job', methods=['POST']) + def apiCompleteJob(): req = request.get_json(silent=True) - account, authorised = env.auth.authoriseReq(req, 'complete_job') + authable, authorised = env.auth.authoriseReq(req, CompleteJob) if authorised: analysisId = req.get('id') - logging.info(str(account) + ' requested to complete job ' + analysisId) + logging.info(str(authable) + ' requested to complete job ' + analysisId) if analysisId is not None: env.gameApi.insertAnalysedGames(req.get('game_analyses')) env.queue.queueNerualAnalysis(analysisId) env.queue.completeEngineAnalysis(analysisId) - return default_responses['success'] - return default_responses['bad_request'] \ No newline at end of file + return DefaultResponse.Success + return DefaultResponse.BadRequest + + @apiBlueprint.route('/post_job', methods=['POST']) + def apiPostJob(): + req = request.get_json(silent=True) + + authable, authorised = env.auth.authoriseReq(req, PostJob) + + if authorised: + games = Game.fromJson(req) + player = Player.fromJson(req) + origin = req.get('origin') + + if None not in [player, origin] and len(games) > 0: + env.playerDB.write(player) + env.gameDB.lazyWriteMany(games) + else: + return DefaultResponse.BadRequest \ No newline at end of file