get training to pass runtime checks

basic-queue-manager.py

@@ -16,7 +16,7 @@
 parser = argparse.ArgumentParser(description=__doc__)
 
 parser.add_argument("--quiet", dest="loglevel",
-                    default=logging.DEBUG, action="store_const", const=logging.INFO,
+                default=logging.DEBUG, action="store_const", const=logging.INFO,
                     help="reduce the number of logged messages")
 config = parser.parse_args()
 
@@ -39,7 +39,7 @@
     basicPlayerQueue = env.basicPlayerQueueDB.nextUnprocessed()
     if basicPlayerQueue is not None:
         logging.info("Basic Queue: " + str(basicPlayerQueue))
-        userId = basicPlayerQueue.id
+        playerId = basicPlayerQueue.id
         origin = basicPlayerQueue.origin
     else:
         logging.info("Basic Queue empty. Pausing")
@@ -49,29 +49,29 @@
     # if there is already a deep queue item open
     # don't update. This will push the request
     # down the queue
-    if env.deepPlayerQueueDB.exists(userId):
+    if env.deepPlayerQueueDB.exists(playerId):
         continue
 
     # get analysed (by fishnet/lichess) games from the db
     gameStore = GameStore.new()
-    gameStore.addGames(env.gameDB.byUserIdAnalysed(userId))
-    gameTensors = gameStore.gameTensors(userId)
+    gameStore.addGames(env.gameDB.byPlayerIdAnalysed(playerId))
+    gameTensors = gameStore.gameTensors(playerId)
 
     if len(gameTensors) > 0:
         gamePredictions = env.irwin.predictBasicGames(gameTensors)
-        gameActivations = [GameBasicActivation.fromPrediction(gameId, userId, prediction, False)
+        gameActivations = [GameBasicActivation.fromPrediction(gameId, playerId, prediction, False)
             for gameId, prediction in gamePredictions]
         env.gameBasicActivationDB.lazyWriteMany(gameActivations)
         deepPlayerQueue = DeepPlayerQueue.new(
-            userId=userId,
+            playerId=playerId,
             origin=origin,
             gamePredictions=gamePredictions)
         logging.info("Writing DeepPlayerQueue: " + str(deepPlayerQueue))
         env.deepPlayerQueueDB.write(deepPlayerQueue)
     else:
         logging.info("No gameTensors")
         deepPlayerQueue = DeepPlayerQueue.new(
-            userId=userId,
+            playerId=playerId,
             origin=origin,
             gamePredictions=[])
         logging.info("Writing DeepPlayerQueue: " + str(deepPlayerQueue))

conf/ConfigWrapper.py

@@ -6,17 +6,14 @@ class ConfigWrapper:
     """
     Used for loading and accessing values from a json config file.
     """
-    @validated
     def __init__(self, d: Dict):
         self.d = d
 
     @staticmethod
-    @validated
     def new(filename: str):
         with open(filename) as confFile:
             return ConfigWrapper(json.load(confFile))
 
-    @validated
     def __getitem__(self, key: str):
         """
         allows for accessing like, conf["index items like this"]
@@ -27,7 +24,6 @@ def __getitem__(self, key: str):
         except ValueError:
             return self.__getattr__(key)
 
-    @validated
     def __getattr__(self, key: str):
         """
         allows for accessing like, conf.index.like.this

deep-queue-manager.py

@@ -15,9 +15,9 @@
 parser = argparse.ArgumentParser(description=__doc__)
 
 parser.add_argument("--name", dest="name",
-                    default=None, type=str, help="name of the thread")
+                default=None, type=str, help="name of the thread")
 parser.add_argument("--quiet", dest="loglevel",
-                    default=logging.DEBUG, action="store_const", const=logging.INFO,
+                default=logging.DEBUG, action="store_const", const=logging.INFO,
                     help="reduce the number of logged messages")
 
 config = parser.parse_args()
@@ -45,8 +45,8 @@
         sleep(30)
         continue
     logging.info("Deep Queue: " + str(deepPlayerQueue))
-    userId = deepPlayerQueue.id
-    playerData = env.api.getPlayerData(userId)
+    playerId = deepPlayerQueue.id
+    playerData = env.api.getPlayerData(playerId)
 
     if playerData is None:
         logging.warning("getPlayerData returned None")
@@ -55,12 +55,12 @@
 
     # get player data, this is really only
     # useful for knowing which games must be analysed
-    player = env.playerDB.byId(userId)
+    player = env.playerDB.byId(playerId)
 
     # pull what we already have on the player
     gameStore = GameStore.new()
-    gameStore.addGames(env.gameDB.byUserId(userId))
-    gameStore.addAnalysedGames(env.analysedGameDB.byUserId(userId))
+    gameStore.addGames(env.gameDB.byPlayerId(playerId))
+    gameStore.addAnalysedGames(env.analysedGameDB.byPlayerId(playerId))
 
     # Filter games and assessments for relevant info
     try:
@@ -70,12 +70,12 @@
         env.deepPlayerQueueDB.complete(deepPlayerQueue)
         continue # if this doesn't gather any useful data, skip
 
-    env.gameDB.lazyWriteGames(gameStore.games)
+    env.gameDB.lazyWriteMany(gameStore.games)
 
     logging.info("Already Analysed: " + str(len(gameStore.analysedGames)))
 
     # decide which games should be analysed
-    gameTensors = gameStore.gameTensorsWithoutAnalysis(userId)
+    gameTensors = gameStore.gameTensorsWithoutAnalysis(playerId)
 
     if len(gameTensors) > 0:
         gamePredictions = env.irwin.predictBasicGames(gameTensors) # [(gameId, prediction)]
@@ -100,7 +100,7 @@
                 game=game,
                 engine=env.engine,
                 infoHandler=env.infoHandler,
-                white=game.white == userId,
+                white=game.white == playerId,
                 nodes=env.config['stockfish']['nodes'],
                 analysedPositionDB=env.analysedPositionDB
             ))
@@ -109,7 +109,7 @@
 
     env.analysedGameDB.lazyWriteAnalysedGames(gameStore.analysedGames)
 
-    logging.info('Posting report for ' + userId)
+    logging.info('Posting report for ' + playerId)
     env.api.postReport(env.irwin.report(
         player=player,
         gameStore=gameStore,

default_imports.py

@@ -5,9 +5,6 @@
 from numpy import float16, float32, float64
 from numpy import float as npfloat
 
-from enforce import runtime_validation as validated
-from enforce.exceptions import RuntimeTypeError
-
 Number = TypeVar('Number', int, float, npfloat, float16, float32, float64)
 
 ## Logging

modules/auth/Auth.py

@@ -1,18 +1,18 @@
 from default_imports import *
 
 from modules.auth.Env import Env
-from modules.auth.User import User, Username, Password
-from modules.auth.Token import Token
+from modules.auth.User import User, UserID, Username, Password
+from modules.auth.Token import Token, TokeniD
 from modules.auth.Priv import Priv
 
-TokenID = NewType('TokenID', str)
-
 Authable = TypeVar('Authable', User, Token)
 
-@validated
+Authorised = NewType('Authorised', bool)
+
+AuthID = TypeVar('AuthID', UserID, TokenID)
+
 class Auth(NamedTuple('Auth', [('env', Env)])):
-    @validated
-	def loginUser(self, username: Username, password: Password) -> Tuple[Opt[User], bool]:
+    	def loginUser(self, username: Username, password: Password) -> Tuple[Opt[User], Authorised]:
 		"""
 		Attempts to log in a user.
 		Returns True is successful.
@@ -24,8 +24,7 @@ def loginUser(self, username: Username, password: Password) -> Tuple[Opt[User],
 			return (user, user.checkPassword(password))
 		return (None, False)
 
-    @validated
-	def registerUser(self, name: str, password: Password, privs: List[Priv] = []) -> Opt[User]:
+    	def registerUser(self, name: Username, password: Password, privs: List[Priv] = []) -> Opt[User]:
 		"""
 		Will attempt to register a user.
 		Returns User object if successful, otherwise None.
@@ -36,8 +35,7 @@ def registerUser(self, name: str, password: Password, privs: List[Priv] = []) ->
 			return user
 		return None
 
-    @validated
-	def authoriseTokenId(self, tokenId: TokenID, priv: Priv) -> Tuple[Opt[Token], bool]:
+    	def authoriseTokenId(self, tokenId: TokenID, priv: Priv) -> Tuple[Opt[Token], Authorised]:
 		"""
 		Given a tokenId, will check if the tokenId has priv.
 		"""
@@ -46,8 +44,7 @@ def authoriseTokenId(self, tokenId: TokenID, priv: Priv) -> Tuple[Opt[Token], bo
 			return (token, token.hasPriv(priv))
 		return (None, False)
 
-    @validated
-	def authoriseUser(self, username: Username, password: Password, priv: Priv) -> Tuple[Opt[User], bool]:
+    	def authoriseUser(self, username: Username, password: Password, priv: Priv) -> Tuple[Opt[User], Authorised]:
 		"""
 		Checks if user has priv in list of privs. 
 		"""
@@ -56,20 +53,23 @@ def authoriseUser(self, username: Username, password: Password, priv: Priv) -> T
 			return (user, loggedIn and user.hasPriv(priv))
 		return (None, False)
 
-    @validated
-	def authoriseRequest(self, req: Dict, priv: Priv) -> Tuple[Opt[Authable], bool]:
+    	def authoriseRequest(self, req: Opt[Dict], priv: Priv) -> Tuple[Opt[Authable], Authorised]:
 		"""
 		Checks if a request is verified with priv.
 		"""
 		if req is not None:
-			tokenId = reg.get('auth', {}).get('token')
-			if tokenId is not None:
-				return self.authoriseTokenId(tokenId, priv)
+            # Attempt to authorise token first
+            authReq = req.get('auth')
+            if authReq is not None:
+    			tokenId = authReq.get('token')
+    			if tokenId is not None:
+    				return self.authoriseTokenId(tokenId, priv)
 
-			username = req.get('auth', {}).get('username')
-			password = req.get('auth', {}).get('password')
+                # Then attempt to authorise user/password
+    			username = authReq.get('username')
+    			password = authReq.get('password')
 
-			if None not in [username, password]:
-				return self.authoriseUser(username, password, priv)
+    			if None not in [username, password]:
+    				return self.authoriseUser(username, password, priv)
 
 		return (None, False)

modules/auth/Env.py

@@ -8,9 +8,8 @@
 from pymongo.database import Database
 
 class Env:
-    @validated
-	def __init__(self, config: ConfigWrapper, db: Database):
+    	def __init__(self, config: ConfigWrapper, db: Database):
 		self.db = db
         self.config = config
-		self.userDB = UserDB(self.db[self.config.coll.user])
-		self.tokenDB = TokenDB(self.db[self.config.coll.token])
+		self.userDB = UserDB(self.db[self.config["auth coll user"]])
+		self.tokenDB = TokenDB(self.db[self.config["auth coll token"]])

modules/auth/Priv.py

@@ -2,23 +2,21 @@
 
 Permission = NewType('Permission', str)
 
-Priv = validated(NamedTuple('Priv' [
+Priv = NamedTuple('Priv' [
         ('permission', Permission)
-    ]))
+    ])
 
 RequestJob = Priv('request_job') # client can request work
 CompleteJob = Priv('complete_job') # client can post results of work
 PostJob = Priv('post_job') # lichess can post a job for analysis
 
 class PrivBSONHandler:
 	@staticmethod
-    @validated
-	def reads(bson: Dict) -> Priv:
+    	def reads(bson: Dict) -> Priv:
 		return Priv(
 			permission=bson['_id'])
 
 	@staticmethod
-    @validated
-	def writes(priv: Priv) -> Dict:
+    	def writes(priv: Priv) -> Dict:
 		return {
 			'_id': priv.permission}

modules/auth/Token.py

@@ -6,39 +6,32 @@
 
 TokenID = NewType('TokenID', str)
 
-@validated
 class Token(NamedTuple('Token', [
         ('id', TokenID), 
         ('privs', List[Priv])
     ])):
-    @validated
-	def hasPriv(self, priv: Priv) -> bool:
+    	def hasPriv(self, priv: Priv) -> bool:
 		return priv in self.privs
 
 class TokenBSONHandler:
 	@staticmethod
-    @validated
-	def reads(bson: Dict) -> Token:
+    	def reads(bson: Dict) -> Token:
 		return Token(
 			id = bson['_id'],
 			privs = [PrivBSONHandler.reads(p) for p in bson['privs']])
 
 	@staticmethod
-    @validated
-	def writes(token: Token) -> Dict:
+    	def writes(token: Token) -> Dict:
 		return {
 			'_id': token.id,
 			'privs': [PrivBSONHandler.writes(p) for p in token.privs]}
 
-@validated
 class TokenDB(NamedTuple('TokenDB', [
         ('coll', Collection)
     ])):
-    @validated
-	def write(self, token: Token):
+    	def write(self, token: Token):
 		self.coll.update_one({'_id': token.id}, {'$set': TokenBSONHandler.writes(token)}, upsert=True)
 
-    @validated
-	def byId(self, _id: TokenId) -> Opt[Token]:
+    	def byId(self, _id: TokenId) -> Opt[Token]:
 		doc = self.coll.find_one({'_id': _id})
 		return None if doc is None else TokenBSONHandler.reads(doc)