diff --git a/aimmo-game-creator/game_manager.py b/aimmo-game-creator/game_manager.py index cf8253b6b..e3e7a03c3 100644 --- a/aimmo-game-creator/game_manager.py +++ b/aimmo-game-creator/game_manager.py @@ -2,16 +2,18 @@ import os import subprocess import time -import kubernetes from abc import ABCMeta, abstractmethod -import pykube import requests from eventlet.greenpool import GreenPool from eventlet.semaphore import Semaphore +import kubernetes + LOGGER = logging.getLogger(__name__) +K8S_NAMESPACE = 'default' + class _GameManagerData(object): """This class is thread safe""" @@ -166,9 +168,10 @@ class KubernetesGameManager(GameManager): """Manages games running on Kubernetes cluster""" def __init__(self, *args, **kwargs): - self._api = pykube.HTTPClient(pykube.KubeConfig.from_service_account()) kubernetes.config.load_incluster_config() - self._api_instance = kubernetes.client.ExtensionsV1beta1Api() + self.extension_api = kubernetes.client.ExtensionsV1beta1Api() + self.api = kubernetes.client.CoreV1Api() + super(KubernetesGameManager, self).__init__(*args, **kwargs) self._create_ingress_paths_for_existing_games() @@ -178,121 +181,72 @@ def _create_ingress_paths_for_existing_games(self): self._add_path_to_ingress(game_id) @staticmethod - def _create_game_name(id): + def _create_game_name(game_id): """ Creates a name that will be used as the pod name as well as in other places. - :param id: Integer indicating the GAME_ID of the game. + :param game_id: Integer indicating the GAME_ID of the game. :return: A string with the game appended with the id. """ - return "game-{}".format(id) - - def _create_game_rc(self, id, environment_variables): - environment_variables["SOCKETIO_RESOURCE"] = KubernetesGameManager._create_game_name(id) - environment_variables["GAME_ID"] = id - environment_variables["GAME_URL"] = "http://game-{}".format(id) - environment_variables["PYKUBE_KUBERNETES_SERVICE_HOST"] = "kubernetes" - environment_variables["IMAGE_SUFFIX"] = os.environ.get("IMAGE_SUFFIX", "latest") - rc = pykube.ReplicationController( - self._api, - { - "kind": "ReplicationController", - "apiVersion": "v1", - "metadata": { - "name": KubernetesGameManager._create_game_name(id), - "namespace": "default", - "labels": { - "app": "aimmo-game", - "game_id": id, - }, - }, - "spec": { - "replicas": 1, - "selector": { - "app": "aimmo-game", - "game_id": id, - }, - "template": { - "metadata": { - "labels": { - "app": "aimmo-game", - "game_id": id, - }, - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": env_name, - "value": env_value, - } for env_name, env_value in environment_variables.items() - ], - "image": "ocadotechnology/aimmo-game:{}".format(os.environ.get("IMAGE_SUFFIX", "latest")), - "ports": [ - { - "containerPort": 5000, - }, - ], - "name": "aimmo-game", - "resources": { - "limits": { - "cpu": "300m", - "memory": "128Mi", - }, - "requests": { - "cpu": "10m", - "memory": "64Mi", - }, - }, - "securityContext": { - "capabilities": { - "drop": [ - "all" - ], - "add": [ - "NET_BIND_SERVICE" - ] - } - } - }, - ], - }, - }, - }, - }, - ) - rc.create() - - def _create_game_service(self, id, _config): - service = pykube.Service( - self._api, - { - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": KubernetesGameManager._create_game_name(id), - "labels": { - "app": "aimmo-game", - "game_id": id, - }, - }, - "spec": { - "selector": { - "app": "aimmo-game", - "game_id": id, - }, - "ports": [ - { - "protocol": "TCP", - "port": 80, - "targetPort": 5000, - }, - ], - "type": "NodePort", - }, - }, - ) - service.create() + return "game-{}".format(game_id) + + def _make_rc(self, environment_variables, game_id): + container = kubernetes.client.V1Container( + env=[kubernetes.client.V1EnvVar( + name=env_name, + value=env_value) for env_name, env_value in environment_variables.items()], + image='ocadotechnology/aimmo-game:{}'.format(os.environ.get('IMAGE_SUFFIX', 'latest')), + ports=[kubernetes.client.V1ContainerPort(container_port=5000)], + name='aimmo-game', + resources=kubernetes.client.V1ResourceRequirements( + limits={'cpu': '300m', 'memory': '128Mi'}, + requests={'cpu': '10m', 'memory': '64Mi'}), + security_context=kubernetes.client.V1SecurityContext( + capabilities=kubernetes.client.V1Capabilities( + drop=['all'], + add=['NET_BIND_SERVICE']))) + + pod_manifest = kubernetes.client.V1PodSpec(containers=[container]) + pod_metadata = kubernetes.client.V1ObjectMeta(labels={'app': 'aimmo-game', 'game_id': game_id}) + pod_template_manifest = kubernetes.client.V1PodTemplateSpec(spec=pod_manifest, metadata=pod_metadata) + + rc_manifest = kubernetes.client.V1ReplicationControllerSpec( + template=pod_template_manifest, + selector={'app': 'aimmo-game', + 'game_id': game_id}, + replicas=1) + + rc_metadata = kubernetes.client.V1ObjectMeta( + name=KubernetesGameManager._create_game_name(game_id), + namespace=K8S_NAMESPACE, + labels={'app': 'aimmo-game', 'game_id': game_id}) + + return kubernetes.client.V1ReplicationController(spec=rc_manifest, metadata=rc_metadata) + + def _create_game_rc(self, game_id, environment_variables): + environment_variables['SOCKETIO_RESOURCE'] = KubernetesGameManager._create_game_name(game_id) + environment_variables['GAME_ID'] = game_id + environment_variables['GAME_URL'] = 'http://game-{}'.format(game_id) + environment_variables['IMAGE_SUFFIX'] = os.environ.get('IMAGE_SUFFIX', 'latest') + environment_variables['K8S_NAMESPACE'] = K8S_NAMESPACE + + rc = self._make_rc(environment_variables, game_id) + self.api.create_namespaced_replication_controller(K8S_NAMESPACE, rc) + + def _make_service(self, game_id): + service_manifest = kubernetes.client.V1ServiceSpec( + selector={'app': 'aimmo-game', 'game_id': game_id}, + ports=[kubernetes.client.V1ServicePort(protocol='TCP', port=80, target_port=5000)], + type='NodePort') + + service_metadata = kubernetes.client.V1ObjectMeta( + name=KubernetesGameManager._create_game_name(game_id), + labels={'app': 'aimmo-game', 'game_id': game_id}) + + return kubernetes.client.V1Service(metadata=service_metadata, spec=service_manifest) + + def _create_game_service(self, game_id): + service = self._make_service(game_id) + self.api.create_namespaced_service(K8S_NAMESPACE, service) def _add_path_to_ingress(self, game_id): backend = kubernetes.client.V1beta1IngressBackend(KubernetesGameManager._create_game_name(game_id), 80) @@ -307,13 +261,13 @@ def _add_path_to_ingress(self, game_id): } ] - self._api_instance.patch_namespaced_ingress("aimmo-ingress", "default", patch) + self.extension_api.patch_namespaced_ingress("aimmo-ingress", "default", patch) def _remove_path_from_ingress(self, game_id): backend = kubernetes.client.V1beta1IngressBackend(KubernetesGameManager._create_game_name(game_id), 80) path = kubernetes.client.V1beta1HTTPIngressPath(backend, "/{}".format(KubernetesGameManager._create_game_name(game_id))) - ingress = self._api_instance.list_namespaced_ingress("default").items[0] + ingress = self.extension_api.list_namespaced_ingress("default").items[0] paths = ingress.spec.rules[0].http.paths try: index_to_delete = paths.index(path) @@ -327,28 +281,37 @@ def _remove_path_from_ingress(self, game_id): } ] - self._api_instance.patch_namespaced_ingress("aimmo-ingress", "default", patch) + self.extension_api.patch_namespaced_ingress("aimmo-ingress", "default", patch) + + def _remove_resources(self, game_id, resource_type): + resource_functions = {'Pod': (self.api.list_namespaced_pod, self.api.delete_namespaced_pod), + 'ReplicationController': (self.api.list_namespaced_replication_controller, + self.api.delete_namespaced_replication_controller), + 'Service': (self.api.list_namespaced_service, self.api.delete_namespaced_service)} + + list_resource_function, delete_resource_function = resource_functions[resource_type] + + app_label = 'app=aimmo-game' + game_label = 'game_id={}'.format(game_id) + + resources = list_resource_function(namespace=K8S_NAMESPACE, + label_selector=','.join([app_label, game_label])) + + for resource in resources.items: + LOGGER.info('Removing: {}'.format(resource.metadata.name)) + delete_resource_function(resource.metadata.name, K8S_NAMESPACE, kubernetes.client.V1DeleteOptions()) def create_game(self, game_id, game_data): - try: - self._create_game_service(game_id, game_data) - except pykube.exceptions.HTTPError as err: - if "already exists" in err.message: - LOGGER.warning("Service for game {} already existed".format(game_id)) - else: - raise + self._create_game_service(game_id) self._create_game_rc(game_id, game_data) self._add_path_to_ingress(game_id) LOGGER.info("Game started - {}".format(game_id)) def delete_game(self, game_id): self._remove_path_from_ingress(game_id) - for object_type in (pykube.ReplicationController, pykube.Service, pykube.Pod): - for game in object_type.objects(self._api).\ - filter(selector={"app": "aimmo-game", - "game_id": game_id}): - LOGGER.info("Removing {}: {}".format(object_type.__name__, game.name)) - game.delete() + self._remove_resources(game_id, 'ReplicationController') + self._remove_resources(game_id, 'Pod') + self._remove_resources(game_id, 'Service') GAME_MANAGERS = { diff --git a/aimmo-game-creator/requirements.txt b/aimmo-game-creator/requirements.txt index 28a2bd2b7..9eaa31f29 100644 --- a/aimmo-game-creator/requirements.txt +++ b/aimmo-game-creator/requirements.txt @@ -1,3 +1,2 @@ -pykube eventlet kubernetes == 5.0.0 diff --git a/aimmo-game-creator/setup.py b/aimmo-game-creator/setup.py index 3a64cc541..d1571af71 100644 --- a/aimmo-game-creator/setup.py +++ b/aimmo-game-creator/setup.py @@ -8,7 +8,6 @@ include_package_data=True, install_requires=[ 'eventlet', - 'pykube', 'kubernetes == 5.0.0' ], tests_require=[ diff --git a/aimmo-game/requirements.txt b/aimmo-game/requirements.txt index 9d18f0818..6ab41a68f 100644 --- a/aimmo-game/requirements.txt +++ b/aimmo-game/requirements.txt @@ -1,7 +1,7 @@ eventlet flask python-socketio==2.0.0 -pykube requests six flask-cors +kubernetes diff --git a/aimmo-game/setup.py b/aimmo-game/setup.py index fc4839f87..ce0669174 100644 --- a/aimmo-game/setup.py +++ b/aimmo-game/setup.py @@ -12,7 +12,7 @@ 'python-socketio==2.0.0', 'requests', 'six', - 'pykube', + 'kubernetes' ], tests_require=[ 'httmock', diff --git a/aimmo-game/simulation/worker_managers/kubernetes_worker_manager.py b/aimmo-game/simulation/worker_managers/kubernetes_worker_manager.py index 254b2d403..1d60809a5 100644 --- a/aimmo-game/simulation/worker_managers/kubernetes_worker_manager.py +++ b/aimmo-game/simulation/worker_managers/kubernetes_worker_manager.py @@ -2,95 +2,86 @@ import os import time -from pykube import HTTPClient, KubeConfig, Pod +import kubernetes.client +import kubernetes.config + from .worker_manager import WorkerManager LOGGER = logging.getLogger(__name__) +# Default here to stop import errors if imported when running locally +K8S_NAMESPACE = os.environ.get('K8S_NAMESPACE', '') class KubernetesWorkerManager(WorkerManager): """Kubernetes worker manager.""" def __init__(self, *args, **kwargs): - self.api = HTTPClient(KubeConfig.from_service_account()) + kubernetes.config.load_incluster_config() + self.api = kubernetes.client.CoreV1Api() self.game_id = os.environ['GAME_ID'] self.game_url = os.environ['GAME_URL'] super(KubernetesWorkerManager, self).__init__(*args, **kwargs) + def make_pod(self, player_id): + container = kubernetes.client.V1Container( + env=[kubernetes.client.V1EnvVar( + name='DATA_URL', + value='%s/player/%d' % (self.game_url, player_id))], + name='aimmo-game-worker', + image='ocadotechnology/aimmo-game-worker:%s' % os.environ.get('IMAGE_SUFFIX', 'latest'), + ports=[kubernetes.client.V1ContainerPort( + container_port=5000, + protocol='TCP')], + resources=kubernetes.client.V1ResourceRequirements( + limits={'cpu': '10m', 'memory': '64Mi'}, + requests={'cpu': '7m', 'memory': '32Mi'}), + security_context=kubernetes.client.V1SecurityContext( + capabilities=kubernetes.client.V1Capabilities( + drop=['all'], + add=['NET_BIND_SERVICE']))) + pod_manifest = kubernetes.client.V1PodSpec(containers=[container]) + + metadata = kubernetes.client.V1ObjectMeta( + labels={ + 'app': 'aimmo-game-worker', + 'game': self.game_id, + 'player': str(player_id)}, + generate_name="aimmo-%s-worker-%s-" % (self.game_id, player_id)) + + return kubernetes.client.V1Pod(metadata=metadata, spec=pod_manifest) + + def _wait_for_pod_creation(self, pod_name, player_id): + + for _ in range(90): + pod = self.api.read_namespaced_pod(pod_name, K8S_NAMESPACE) + LOGGER.info('Pod status: {}'.format(pod.status)) + if pod.status.phase == 'Running': + return pod + + time.sleep(1) + + raise EnvironmentError('Could not start worker %s.' % player_id) + def create_worker(self, player_id): - pod = Pod( - self.api, - { - 'kind': 'Pod', - 'apiVersion': 'v1', - 'metadata': { - 'generateName': "aimmo-%s-worker-%s-" % (self.game_id, player_id), - 'labels': { - 'app': 'aimmo-game-worker', - 'game': self.game_id, - 'player': str(player_id), - }, - }, - 'spec': { - 'containers': [ - { - 'env': [ - { - 'name': 'DATA_URL', - 'value': "%s/player/%d" % (self.game_url, player_id), - }, - ], - 'name': 'aimmo-game-worker', - 'image': 'ocadotechnology/aimmo-game-worker:%s' % os.environ.get('IMAGE_SUFFIX', 'latest'), - 'ports': [ - { - 'containerPort': 5000, - 'protocol': 'TCP' - } - ], - 'resources': { - 'limits': { - 'cpu': '10m', - 'memory': '64Mi', - }, - 'requests': { - 'cpu': '7m', - 'memory': '32Mi', - }, - }, - 'securityContext': { - 'capabilities': { - 'drop': [ - 'all' - ], - 'add': [ - 'NET_BIND_SERVICE' - ] - } - } - }, - ], - }, - } - ) - pod.create() - iterations = 0 - while pod.obj['status']['phase'] == 'Pending': - if iterations > 30: - raise EnvironmentError('Could not start worker %s, details %s' % (player_id, pod.obj)) - LOGGER.debug('Waiting for worker %s', player_id) - time.sleep(5) - pod.reload() - iterations += 1 - worker_url = "http://%s:5000" % pod.obj['status']['podIP'] - LOGGER.info("Worker started for %s, listening at %s", player_id, worker_url) + pod_obj = self.make_pod(player_id) + LOGGER.info('Making new worker pod: {}'.format(pod_obj.metadata.name)) + pod = self.api.create_namespaced_pod(namespace=K8S_NAMESPACE, body=pod_obj) + pod_name = pod.metadata.name + pod = self._wait_for_pod_creation(pod_name, player_id) + + worker_url = 'http://%s:5000' % pod.status.pod_ip + LOGGER.info('Worker ip: {}'.format(pod.status.pod_ip)) + LOGGER.info('Worker started for %s, listening at %s', player_id, worker_url) return worker_url def remove_worker(self, player_id): - for pod in Pod.objects(self.api).filter(selector={ - 'app': 'aimmo-game-worker', - 'game': self.game_id, - 'player': str(player_id), - }): - LOGGER.debug('Removing pod %s', pod.obj['spec']) - pod.delete() + app_label = 'app=aimmo-game-worker' + game_label = 'game={}'.format(self.game_id) + player_label = 'player={}'.format(player_id) + + pods = self.api.list_namespaced_pod(namespace=K8S_NAMESPACE, + label_selector=','.join([app_label, game_label, player_label])) + + for pod in pods.items: + LOGGER.info('Deleting pod: {}'.format(pod.metadata.name)) + self.api.delete_namespaced_pod(pod.metadata.name, K8S_NAMESPACE, kubernetes.client.V1DeleteOptions()) diff --git a/hashes.txt b/hashes.txt new file mode 100644 index 000000000..06d45ce36 --- /dev/null +++ b/hashes.txt @@ -0,0 +1,20 @@ +pbkdf2_sha256$20000$NgJaw0HOkFrr$878NQj1Sa3QRJcLElkABvWYH2aZw/f3zxO4h+jMPyDs=pbkdf2_sha256$20000$NgJaw0HOkFrr$878NQj1Sa3QRJcLElkABvWYH2aZw/f3zxO4h+jMPyDs=pbkdf2_sha256$20000$NgJaw0HOkFrr$878NQj1Sa3QRJcLElkABvWYH2aZw/f3zxO4h+jMPyDs=pbkdf2_sha256$20000$NgJaw0HOkFrr$878NQj1Sa3QRJcLElkABvWYH2aZw/f3zxO4h+jMPyDs=pbkdf2_sha256$20000$RUOl3AJuEUPM$+cYcucsG6G1B1JOWMhIckxI/9e5gMXkoyS/QNmgX7oc=pbkdf2_sha256$20000$RUOl3AJuEUPM$+cYcucsG6G1B1JOWMhIckxI/9e5gMXkoyS/QNmgX7oc=pbkdf2_sha256$20000$RUOl3AJuEUPM$+cYcucsG6G1B1JOWMhIckxI/9e5gMXkoyS/QNmgX7oc=pbkdf2_sha256$20000$RUOl3AJuEUPM$+cYcucsG6G1B1JOWMhIckxI/9e5gMXkoyS/QNmgX7oc=pbkdf2_sha256$20000$wHTKEsCkxyDi$xxM29NiGTEk3II1taYwv2K6VMnGo7YuaTPSQdGfOCpA= +pbkdf2_sha256$20000$wHTKEsCkxyDi$xxM29NiGTEk3II1taYwv2K6VMnGo7YuaTPSQdGfOCpA= +pbkdf2_sha256$20000$wHTKEsCkxyDi$xxM29NiGTEk3II1taYwv2K6VMnGo7YuaTPSQdGfOCpA= +pbkdf2_sha256$20000$wHTKEsCkxyDi$xxM29NiGTEk3II1taYwv2K6VMnGo7YuaTPSQdGfOCpA= +pbkdf2_sha256$20000$H2OTJz63tzPs$c//KTAaXA7TNkSB4TMFdMbILLhcbervir24RJr3cHPg= +pbkdf2_sha256$20000$H2OTJz63tzPs$c//KTAaXA7TNkSB4TMFdMbILLhcbervir24RJr3cHPg= +pbkdf2_sha256$20000$H2OTJz63tzPs$c//KTAaXA7TNkSB4TMFdMbILLhcbervir24RJr3cHPg= +pbkdf2_sha256$20000$H2OTJz63tzPs$c//KTAaXA7TNkSB4TMFdMbILLhcbervir24RJr3cHPg= +pbkdf2_sha256$20000$3dVulaHvWFzG$d5Z1J+QJHFam6Vki8CmOVf1gvbO3eormsWb/KSA056U= +pbkdf2_sha256$20000$3dVulaHvWFzG$d5Z1J+QJHFam6Vki8CmOVf1gvbO3eormsWb/KSA056U= +pbkdf2_sha256$20000$3dVulaHvWFzG$d5Z1J+QJHFam6Vki8CmOVf1gvbO3eormsWb/KSA056U= +pbkdf2_sha256$20000$3dVulaHvWFzG$d5Z1J+QJHFam6Vki8CmOVf1gvbO3eormsWb/KSA056U= +pbkdf2_sha256$20000$srBtXFUfTwuZ$S7EtwLaKhH7twLVpce6TbbqOdvqMIqYTOEIFWfTfqws= +pbkdf2_sha256$20000$BLeURTbhiQtB$78lt2o5gDJPqEhA7VxaZGUK5YYr1LIBibMrEvtIGfZI= +pbkdf2_sha256$20000$0Z7TO2J6vDFq$GdTdB5DXKuVckObCSSJpDDDocxykvkpNsmY+tWUeHxc= +pbkdf2_sha256$20000$RRiRDureHQV1$QkFJ7dggkbK0m4xD92UogP2MAuAyTuiosG06YGDU93g= +pbkdf2_sha256$20000$QqUo7GOOaWT4$eESqqPW8vJXYXLPVtLe7CvDDYkv3JGNbyEHeQq7nCNA= +pbkdf2_sha256$20000$Ws6L16vqon1Z$HNF46SOMExkV1UMmhxuSOZknOQC7BzeryXNvbZZvBOo= +pbkdf2_sha256$20000$k1hL0oJgd6fr$U05BF8Mw3/TPW3I1pvrUQTcSe79YzsiJBn1tysi/mAI= +pbkdf2_sha256$20000$CDwShmhGu8si$KDgi25CBJ2W/UzsXQJKLlgcuo2hA35DfdTh5osduvyM=