Implement a dev-room container to simulate a real room locally

Makefile

@@ -34,6 +34,13 @@ build:
 build-docker:
 	@docker build -t maestro .
 
+build-dev-room:
+	@cd dev-room && docker build -t maestro-dev-room .
+	@eval $(minikube docker-env -u)
+
+delete-dev-room:
+	@docker rmi -f maestro-dev-room
+
 cross-build-linux-amd64:
 	@env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ./bin/maestro-linux-amd64
 	@chmod a+x ./bin/maestro-linux-amd64
@@ -84,13 +91,13 @@ deps-test: start-deps-test wait-for-test-pg drop-test migrate-test minikube
 deps-test-ci: deps drop-test migrate-test minikube-ci
 
 drop:
-	@-psql -d postgres -h localhost -p 8765 -U postgres -c "SELECT pg_terminate_backend(pid.pid) FROM pg_stat_activity, (SELECT pid FROM pg_stat_activity where pid <> pg_backend_pid()) pid WHERE datname='maestro';"
-	@psql -d postgres -h localhost -p 8765 -U postgres -f scripts/drop.sql > /dev/null
+	@-docker exec maestro_postgres_1 psql -d postgres -h localhost -p 5432 -U postgres -c "SELECT pg_terminate_backend(pid.pid) FROM pg_stat_activity, (SELECT pid FROM pg_stat_activity where pid <> pg_backend_pid()) pid WHERE datname='maestro';"
+	@docker exec maestro_postgres_1 psql -d postgres -h localhost -p 5432 -U postgres -f scripts/drop.sql > /dev/null
 	@echo "Database created successfully!"
 
 drop-test:
-	@-psql -d postgres -h localhost -p 8585 -U postgres -c "SELECT pg_terminate_backend(pid.pid) FROM pg_stat_activity, (SELECT pid FROM pg_stat_activity where pid <> pg_backend_pid()) pid WHERE datname='maestro_test';"
-	@psql -d postgres -h localhost -p 8585 -U postgres -f scripts/drop-test.sql > /dev/null
+	@-docker exec maestro_test_postgres_1 psql -d postgres -h localhost -p 5432 -U postgres -c "SELECT pg_terminate_backend(pid.pid) FROM pg_stat_activity, (SELECT pid FROM pg_stat_activity where pid <> pg_backend_pid()) pid WHERE datname='maestro_test';"
+	@docker exec maestro_test_postgres_1 psql -d postgres -h localhost -p 5432 -U postgres -f scripts/drop-test.sql > /dev/null
 	@echo "Test Database created successfully!"
 
 unit: unit-board clear-coverage-profiles unit-run gather-unit-profiles

README.md

@@ -134,16 +134,25 @@ To run on dev:
 
 1) Start [Minikube](https://kubernetes.io/docs/tutorials/stateless-application/hello-minikube/)
 
-2) Start dependencies
+2) Build dev-room docker image on Minikube env. (optional)
+```
+eval $(minikube docker-env)
+make build-dev-room
+eval $(minikube docker-env -u)
+```
+
+This image contain a simple Python API to simulate room behaviour. Its code can be found [here](/dev-room).
+
+3) Start dependencies
 ```
 make deps
 make drop
-make migration
+make migrate
 ```
 
-3) Start worker with `make work` and api with `make run-dev`
+4) Start worker with `make work` and api with `make run-dev`
 
-4) With [maestro-cli](https://github.com/topfreegames/maestro-cli), access maestro with context `local`. For example:
+5) With [maestro-cli](https://github.com/topfreegames/maestro-cli), access maestro with context `local`. For example:
 
 ```
 maestro -c local create manifests/scheduler-config-1.yaml

dev-room/.dockerignore

@@ -0,0 +1,3 @@
+env
+__pycache__
+.vscode

dev-room/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3
+
+WORKDIR /usr/src/app
+ADD . .
+
+RUN pip install --no-cache-dir -r requirements.txt
+CMD ["gunicorn", "-b", "0.0.0.0:8080", "app:app"]

dev-room/api/__init__.py

@@ -0,0 +1,3 @@
+from api.address import Address
+from api.healthcheck import Healthcheck
+from api.status import Status

dev-room/api/address.py

@@ -0,0 +1,22 @@
+import falcon
+import json
+import requests
+
+import app
+import constant
+
+class Address (object):
+    def on_get(self, req, resp):
+        """Handles GET requests"""
+        app.logger.debug("address")
+        app.logger.debug("request to {}/{}".format(constant.MAESTRO_ADDR, constant.ROOM_ADDR_ENDPOINT))
+
+        try:
+            r = requests.get("{}/{}".format(constant.MAESTRO_ADDR, constant.ROOM_ADDR_ENDPOINT))
+            if r.status_code == 200:
+                resp.status = falcon.HTTP_200
+                resp.body = r.text
+
+        except Exception as ex:
+            resp.status = falcon.HTTP_500
+            resp.body = (json.dumps({"error": str(ex)}))

dev-room/api/exceptions.py

@@ -0,0 +1,7 @@
+class WrongStatus(Exception):
+    def __init__(self, text="Wrong status update. Try one of 'ready' or 'occupied'"):
+        self.text=text
+        super().__init__(self)
+
+    def __str__(self):
+        return str(self.text)

dev-room/api/healthcheck.py

@@ -0,0 +1,11 @@
+import falcon
+import json
+
+import app
+
+class Healthcheck (object):
+    def on_get(self, req, resp):
+        """Handles GET requests"""
+        app.logger.debug("healthcheck")
+        resp.status = falcon.HTTP_200
+        resp.body = (json.dumps({"healthy": True}))

dev-room/api/status.py

@@ -0,0 +1,33 @@
+import falcon
+import json
+import requests
+import time
+
+import app
+import constant
+from api.exceptions import WrongStatus
+
+class Status (object):
+    def on_put(self, req, resp):
+        """Handles PUT requests"""
+        app.logger.debug("status")
+
+        try:
+            body = req.stream.read()
+            status = str.lower(json.loads(body)["status"])
+
+            if status not in ["ready", "occupied"]:
+                raise WrongStatus()
+
+            app.logger.debug("request to {}/{} with status={}".format(constant.MAESTRO_ADDR, constant.ROOM_STATUS_ENDPOINT, status))
+            r = requests.put("{}/{}".format(constant.MAESTRO_ADDR, constant.ROOM_STATUS_ENDPOINT), json={"timestamp": int(time.time()), "status": status})
+            if r.status_code == 200:
+                resp.status = falcon.HTTP_200
+                resp.body = r.text
+
+        except Exception as ex:
+            if type(ex) == WrongStatus:
+                resp.status = falcon.HTTP_400
+            else:
+                resp.status = falcon.HTTP_500
+            resp.body = (json.dumps({"error": str(ex)}))

dev-room/app.py

@@ -0,0 +1,32 @@
+# External deps
+import time
+import requests
+import falcon
+import logging
+from logging.config import fileConfig
+
+# Internal deps 
+import constant
+from api import Healthcheck, Address, Status
+
+
+fileConfig('logging_config.ini')
+logger = logging.getLogger()
+
+while True:
+    try:
+        print("polling {}/{}".format(constant.MAESTRO_ADDR, constant.ROOM_ADDR_ENDPOINT))
+        r = requests.get("{}/{}".format(constant.MAESTRO_ADDR, constant.ROOM_ADDR_ENDPOINT))
+        if r.status_code == 200:
+            r = requests.put("{}/{}".format(constant.MAESTRO_ADDR, constant.ROOM_STATUS_ENDPOINT), json={"timestamp": int(time.time()), "status": "ready"})
+            break
+    except Exception as ex:
+        print(str(ex))
+        pass
+    time.sleep(constant.POLLING_INTERVAL_IN_SECONDS)
+
+# Start falcon API
+app = falcon.API()
+app.add_route('/healthcheck', Healthcheck())
+app.add_route('/address', Address())
+app.add_route('/status', Status())

dev-room/constant.py

@@ -0,0 +1,10 @@
+import os
+
+# Config env
+MAESTRO_ADDR = "http://{}".format(os.environ.get("MAESTRO_HOST_PORT"))
+MAESTRO_SCHEDULER_NAME = os.environ.get("MAESTRO_SCHEDULER_NAME")
+MAESTRO_ROOM_ID = os.environ.get("MAESTRO_ROOM_ID")
+ROOM_MGMT_ENDPOINT = "scheduler/{}/rooms/{}".format(MAESTRO_SCHEDULER_NAME, MAESTRO_ROOM_ID)
+ROOM_ADDR_ENDPOINT = "{}/address".format(ROOM_MGMT_ENDPOINT)
+ROOM_STATUS_ENDPOINT = "{}/status".format(ROOM_MGMT_ENDPOINT)
+POLLING_INTERVAL_IN_SECONDS = int(os.environ.get("POLLING_INTERVAL_IN_SECONDS")) if os.environ.get("POLLING_INTERVAL_IN_SECONDS") else 10

dev-room/logging_config.ini

@@ -0,0 +1,21 @@
+[loggers]
+keys=root
+
+[handlers]
+keys=stream_handler
+
+[formatters]
+keys=formatter
+
+[logger_root]
+level=DEBUG
+handlers=stream_handler
+
+[handler_stream_handler]
+class=StreamHandler
+level=DEBUG
+formatter=formatter
+args=(sys.stderr,)
+
+[formatter_formatter]
+format=%(asctime)s %(name)-12s %(levelname)-8s %(message)s

dev-room/requirements.txt

@@ -0,0 +1,17 @@
+astroid==2.0.2
+certifi==2018.4.16
+chardet==3.0.4
+falcon==1.4.1
+gunicorn==19.9.0
+idna==2.7
+isort==4.3.4
+lazy-object-proxy==1.3.1
+mccabe==0.6.1
+pylint==2.1.0
+python-mimeparse==1.6.0
+requests==2.19.1
+six==1.11.0
+typed-ast==1.1.0
+typing==3.6.4
+urllib3==1.23
+wrapt==1.10.11

manifests/scheduler-config-5.json

@@ -0,0 +1,47 @@
+{
+   "name": "scheduler-name",
+   "game": "game-name",
+   "image": "maestro-dev-room:latest",
+   "imagePullPolicy": "Never",
+   "ports": [
+      {
+         "containerPort": 8080,
+         "protocol": "TCP",
+         "name": "tcp"
+      }
+   ],
+   "limits": {
+      "memory": "128Mi",
+      "cpu": "20m"
+   },
+   "shutdownTimeout": 10,
+   "autoscaling": {
+      "min": 4,
+      "up": {
+         "delta": 1,
+         "trigger": {
+            "usage": 70,
+            "time": 1
+         },
+         "cooldown": 1
+      },
+      "down": {
+         "delta": 1,
+         "trigger": {
+            "usage": 50,
+            "time": 1
+         },
+         "cooldown": 1
+      }
+   },
+   "env": [
+       {
+          "name": "MAESTRO_HOST_PORT",
+          "value": "192.168.64.1:8080"
+       },
+       {
+          "name": "POLLING_INTERVAL_IN_SECONDS",
+          "value": "20"
+       }
+   ]
+}

models/config_yaml.go

@@ -23,12 +23,13 @@ type ConfigYAMLv1 struct {
 	PortRange       *PortRange                       `yaml:"portRange"`
 
 	// Container level, to keep compatibility
-	Image    string     `yaml:"image" json:"image"`
-	Ports    []*Port    `yaml:"ports" json:"ports"`
-	Limits   *Resources `yaml:"limits" json:"limits"`
-	Requests *Resources `yaml:"requests" json:"requests"`
-	Env      []*EnvVar  `yaml:"env" json:"env"`
-	Cmd      []string   `yaml:"cmd" json:"cmd"`
+	Image           string     `yaml:"image" json:"image"`
+	ImagePullPolicy string     `yaml:"imagePullPolicy" json:"imagePullPolicy"`
+	Ports           []*Port    `yaml:"ports" json:"ports"`
+	Limits          *Resources `yaml:"limits" json:"limits"`
+	Requests        *Resources `yaml:"requests" json:"requests"`
+	Env             []*EnvVar  `yaml:"env" json:"env"`
+	Cmd             []string   `yaml:"cmd" json:"cmd"`
 }
 
 // ConfigYAMLv2 is the ConfigYAML after refactor to n containers per pod
@@ -61,12 +62,13 @@ type ConfigYAML struct {
 	PortRange       *PortRange                       `yaml:"portRange"`
 
 	// Container level, to keep compatibility
-	Image    string     `yaml:"image" json:"image"`
-	Ports    []*Port    `yaml:"ports" json:"ports"`
-	Limits   *Resources `yaml:"limits" json:"limits"`
-	Requests *Resources `yaml:"requests" json:"requests"`
-	Env      []*EnvVar  `yaml:"env" json:"env"`
-	Cmd      []string   `yaml:"cmd" json:"cmd"`
+	Image           string     `yaml:"image" json:"image"`
+	ImagePullPolicy string     `yaml:"imagePullPolicy" json:"imagePullPolicy"`
+	Ports           []*Port    `yaml:"ports" json:"ports"`
+	Limits          *Resources `yaml:"limits" json:"limits"`
+	Requests        *Resources `yaml:"requests" json:"requests"`
+	Env             []*EnvVar  `yaml:"env" json:"env"`
+	Cmd             []string   `yaml:"cmd" json:"cmd"`
 
 	// Container level, for schedulers with more than one container per pod
 	Containers []*Container `yaml:"containers" json:"containers"`
@@ -96,6 +98,7 @@ func (c *ConfigYAML) ToYAML() []byte {
 			NodeToleration:  c.NodeToleration,
 			OccupiedTimeout: c.OccupiedTimeout,
 			Forwarders:      c.Forwarders,
+			ImagePullPolicy: c.ImagePullPolicy,
 			Image:           c.Image,
 			Ports:           c.Ports,
 			Limits:          c.Limits,
@@ -175,6 +178,10 @@ func (c *ConfigYAML) EnsureDefaultValues() {
 	if c.AutoScaling.Up.Trigger.Limit == 0 {
 		c.AutoScaling.Up.Trigger.Limit = 90
 	}
+
+	if c.ImagePullPolicy == "" {
+		c.ImagePullPolicy = "Always"
+	}
 }
 
 // Version returns the config version

models/pod.go

@@ -65,6 +65,7 @@ spec:
   {{range .Containers}}
   - name: {{.Name}}
     image: {{.Image}}
+    imagePullPolicy: {{.ImagePullPolicy}}
     hostNetwork: "true"
     ports:
       {{range .Ports}}
@@ -143,11 +144,12 @@ func NewPod(
 	}
 
 	container := &Container{
-		Image:   configYaml.Image,
-		Name:    name,
-		Env:     envs,
-		Ports:   configYaml.Ports,
-		Command: configYaml.Cmd,
+		Image:           configYaml.Image,
+		ImagePullPolicy: configYaml.ImagePullPolicy,
+		Name:            name,
+		Env:             envs,
+		Ports:           configYaml.Ports,
+		Command:         configYaml.Cmd,
 	}
 
 	if configYaml.Limits != nil {