Skip to content
Permalink
Browse files

feat(compose): add interactive mode of GNES board using Flask

  • Loading branch information...
hanxiao committed Jul 20, 2019
1 parent 5876c15 commit b34a765aa851ccabf45f73a740e82b95de3f1c1a
@@ -35,8 +35,13 @@ def route(args):


def compose(args):
from ..composer.base import YamlGraph
YamlGraph(args).build_all()
from ..composer.base import YamlComposer
from ..composer.flask import YamlComposerFlask

if args.flask:
YamlComposerFlask(args).run()
else:
YamlComposer(args).build_all()


def frontend(args):
@@ -34,6 +34,8 @@ def set_base_parser():


def set_composer_parser(parser=None):
from pkg_resources import resource_stream

if not parser:
parser = set_base_parser()
parser.add_argument('--port',
@@ -45,8 +47,9 @@ def set_composer_parser(parser=None):
default='GNES instance',
help='name of the instance')
parser.add_argument('--yaml_path', type=argparse.FileType('r'),
required=True,
help='yaml config of the service')
default=resource_stream(
'gnes', '/'.join(('resources', 'config', 'compose', 'default.yml'))),
help='yaml config of the service')
parser.add_argument('--html_path', type=argparse.FileType('w', encoding='utf8'),
default='./gnes-board.html',
help='output path of the HTML file, will contain all possible generations')
@@ -69,6 +72,19 @@ def set_composer_parser(parser=None):
return parser


def set_composer_flask_parser(parser=None):
if not parser:
parser = set_base_parser()
set_composer_parser(parser)
parser.add_argument('--flask', action='store_true', default=False,
help='using Flask to serve GNES composer in interactive mode')
parser.add_argument('--cors', type=str, default='*',
help='setting "Access-Control-Allow-Origin" for HTTP requests')
parser.add_argument('--http_port', type=int, default=8080,
help='server port for receiving HTTP requests')
return parser


def set_service_parser(parser=None):
from ..service.base import SocketType, BaseService
if not parser:
@@ -253,5 +269,5 @@ def get_main_parser():
set_preprocessor_service_parser(sp.add_parser('preprocess', help='start a preprocessor service'))
set_http_service_parser(sp.add_parser('client_http', help='start a http service'))
set_cli_client_parser(sp.add_parser('client_cli', help='start a grpc client'))
set_composer_parser(sp.add_parser('compose', help='start a GNES composer to simplify config generation'))
set_composer_flask_parser(sp.add_parser('compose', help='start a GNES composer to simplify config generation'))
return parser
@@ -21,7 +21,6 @@
from concurrent.futures import ThreadPoolExecutor

import grpc
from aiohttp import web
from google.protobuf.json_format import MessageToJson

from ..helper import set_logger
@@ -34,6 +33,7 @@ def __init__(self, args=None):
self.logger = set_logger(self.__class__.__name__, self.args.verbose)

def start(self):
from aiohttp import web
loop = asyncio.get_event_loop()
executor = ThreadPoolExecutor(max_workers=self.args.max_workers)

@@ -19,7 +19,7 @@
_yaml = YAML()


class YamlGraph:
class YamlComposer:
comp2file = {
'Encoder': 'encode',
'Router': 'route',
@@ -50,7 +50,7 @@ def __init__(self, layer_id: int = 0):

@staticmethod
def get_value(comp: Dict, key: str):
return comp.get(key, YamlGraph.Layer.default_values[key])
return comp.get(key, YamlComposer.Layer.default_values[key])

@property
def is_homogenous(self):
@@ -83,7 +83,7 @@ def __repr__(self):

def __init__(self, args):

self._layers = [] # type: List['YamlGraph.Layer']
self._layers = [] # type: List['YamlComposer.Layer']
self.logger = set_logger(self.__class__.__name__)
with args.yaml_path:
tmp = _yaml.load(args.yaml_path)
@@ -136,8 +136,8 @@ def add_layer(self, layer: 'Layer' = None) -> None:
def add_comp(self, comp: Dict) -> None:
self._layers[-1].append(comp)

def build_layers(self) -> List['YamlGraph.Layer']:
all_layers = [] # type: List['YamlGraph.Layer']
def build_layers(self) -> List['YamlComposer.Layer']:
all_layers = [] # type: List['YamlComposer.Layer']
for idx, layer in enumerate(self._layers[1:] + [self._layers[0]], 1):
last_layer = self._layers[idx - 1]
for l in self._add_router(last_layer, layer):
@@ -149,7 +149,7 @@ def build_layers(self) -> List['YamlGraph.Layer']:
return all_layers

@staticmethod
def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gnes/gnes:latest',
def build_dockerswarm(all_layers: List['YamlComposer.Layer'], docker_img: str = 'gnes/gnes:latest',
volumes: Dict = None, networks: Dict = None) -> str:
with resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-swarm.yml'))) as r:
swarm_lines = _yaml.load(r)
@@ -158,7 +158,7 @@ def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gn
for c_idx, c in enumerate(layer.components):
c_name = '%s%d%d' % (c['name'], l_idx, c_idx)
args = ['--%s %s' % (a, str(v) if ' ' not in str(v) else ('"%s"' % str(v))) for a, v in c.items() if
a in YamlGraph.comp2args[c['name']] and v]
a in YamlComposer.comp2args[c['name']] and v]
if 'yaml_path' in c and c['yaml_path'] is not None:
args.append('--yaml_path /%s_yaml' % c_name)
config_dict['%s_yaml' % c_name] = {'file': c['yaml_path']}
@@ -191,16 +191,16 @@ def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gn
args += ['--host_in %s' % host_in_name]
# '--host_out %s' % host_out_name]

cmd = '%s %s' % (YamlGraph.comp2file[c['name']], ' '.join(args))
cmd = '%s %s' % (YamlComposer.comp2file[c['name']], ' '.join(args))
swarm_lines['services'][c_name] = CommentedMap({
'image': docker_img,
'command': cmd,
})

rep_c = YamlGraph.Layer.get_value(c, 'replicas')
rep_c = YamlComposer.Layer.get_value(c, 'replicas')
if rep_c > 1:
swarm_lines['services'][c_name]['deploy'] = CommentedMap({
'replicas': YamlGraph.Layer.get_value(c, 'replicas'),
'replicas': YamlComposer.Layer.get_value(c, 'replicas'),
'restart_policy': {
'condition': 'on-failure',
'max_attempts': 3,
@@ -223,30 +223,30 @@ def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gn
return stream.getvalue()

@staticmethod
def build_kubernetes(all_layers: List['YamlGraph.Layer'], *args, **kwargs):
def build_kubernetes(all_layers: List['YamlComposer.Layer'], *args, **kwargs):
pass

@staticmethod
def build_shell(all_layers: List['YamlGraph.Layer'], log_redirect: str = None) -> str:
def build_shell(all_layers: List['YamlComposer.Layer'], log_redirect: str = None) -> str:
shell_lines = []
for layer in all_layers:
for c in layer.components:
rep_c = YamlGraph.Layer.get_value(c, 'replicas')
rep_c = YamlComposer.Layer.get_value(c, 'replicas')
shell_lines.append('printf "starting service %s with %s replicas...\\n"' % (
colored(c['name'], 'green'), colored(rep_c, 'yellow')))
for _ in range(rep_c):
cmd = YamlGraph.comp2file[c['name']]
cmd = YamlComposer.comp2file[c['name']]
args = ' '.join(
['--%s %s' % (a, str(v) if ' ' not in str(v) else ('"%s"' % str(v))) for a, v in c.items() if
a in YamlGraph.comp2args[c['name']] and v])
a in YamlComposer.comp2args[c['name']] and v])
shell_lines.append('gnes %s %s %s &' % (
cmd, args, '>> %s 2>&1' % log_redirect if log_redirect else ''))

with resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-shell.sh'))) as r:
return r.read().decode().replace('{{gnes-template}}', '\n'.join(shell_lines))

@staticmethod
def build_mermaid(all_layers: List['YamlGraph.Layer'], mermaid_leftright: bool = False) -> str:
def build_mermaid(all_layers: List['YamlComposer.Layer'], mermaid_leftright: bool = False) -> str:
mermaid_graph = []
cls_dict = defaultdict(set)
for l_idx, layer in enumerate(all_layers[1:] + [all_layers[0]], 1):
@@ -255,20 +255,20 @@ def build_mermaid(all_layers: List['YamlGraph.Layer'], mermaid_leftright: bool =
for c_idx, c in enumerate(last_layer.components):
# if len(last_layer.components) > 1:
# self.mermaid_graph.append('\tsubgraph %s%d' % (c['name'], c_idx))
for j in range(YamlGraph.Layer.get_value(c, 'replicas')):
for j in range(YamlComposer.Layer.get_value(c, 'replicas')):
for c1_idx, c1 in enumerate(layer.components):
if c1['port_in'] == c['port_out']:
p = '((%s%s))' if c['name'] == 'Router' else '(%s%s)'
p1 = '((%s%s))' if c1['name'] == 'Router' else '(%s%s)'
for j1 in range(YamlGraph.Layer.get_value(c1, 'replicas')):
for j1 in range(YamlComposer.Layer.get_value(c1, 'replicas')):
_id, _id1 = '%s%s%s' % (last_layer.layer_id, c_idx, j), '%s%s%s' % (
layer.layer_id, c1_idx, j1)
conn_type = (
c['socket_out'].split('_')[0] + '/' + c1['socket_in'].split('_')[0]).lower()
s_id = '%s%s' % (c_idx if len(last_layer.components) > 1 else '',
j if YamlGraph.Layer.get_value(c, 'replicas') > 1 else '')
j if YamlComposer.Layer.get_value(c, 'replicas') > 1 else '')
s1_id = '%s%s' % (c1_idx if len(layer.components) > 1 else '',
j1 if YamlGraph.Layer.get_value(c1, 'replicas') > 1 else '')
j1 if YamlComposer.Layer.get_value(c1, 'replicas') > 1 else '')
mermaid_graph.append(
'\t%s%s%s-- %s -->%s%s%s' % (
c['name'], _id, p % (c['name'], s_id), conn_type, c1['name'], _id1,
@@ -319,11 +319,15 @@ def std_or_print(f, content):
'timestamp': time.strftime("%a, %d %b %Y %H:%M:%S"),
'version': __version__
}

cmds['html'] = self.build_html(cmds)

std_or_print(self.args.graph_path, cmds['mermaid'])
std_or_print(self.args.shell_path, cmds['shell'])
std_or_print(self.args.swarm_path, cmds['docker'])
std_or_print(self.args.k8s_path, cmds['k8s'])
std_or_print(self.args.html_path, self.build_html(cmds))
std_or_print(self.args.html_path, cmds['html'])
return cmds

@staticmethod
def _get_random_port(min_port: int = 49152, max_port: int = 65536) -> str:
@@ -333,7 +337,7 @@ def _get_random_port(min_port: int = 49152, max_port: int = 65536) -> str:
def _get_random_host(comp_name: str) -> str:
return str(comp_name + str(random.randrange(0, 100)))

def _add_router(self, last_layer: 'YamlGraph.Layer', layer: 'YamlGraph.Layer') -> List['YamlGraph.Layer']:
def _add_router(self, last_layer: 'YamlComposer.Layer', layer: 'YamlComposer.Layer') -> List['YamlComposer.Layer']:
def rule1():
# a shortcut fn: push connect the last and current
last_layer.components[0]['socket_out'] = str(SocketType.PUSH_BIND)
@@ -346,7 +350,7 @@ def rule2():

def rule3():
# a shortcut fn: (N)-2-(N) with push pull connection
router_layer = YamlGraph.Layer(layer_id=self._num_layer)
router_layer = YamlComposer.Layer(layer_id=self._num_layer)
self._num_layer += 1
last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT)
r = CommentedMap({'name': 'Router',
@@ -375,7 +379,7 @@ def rule5():

def rule6():
last_layer.components[0]['socket_out'] = str(SocketType.PUB_BIND)
router_layer = YamlGraph.Layer(layer_id=self._num_layer)
router_layer = YamlComposer.Layer(layer_id=self._num_layer)
self._num_layer += 1
for c in layer.components:
income = self.Layer.get_value(c, 'income')
@@ -394,7 +398,7 @@ def rule6():
def rule7():
last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT)

router_layer = YamlGraph.Layer(layer_id=self._num_layer)
router_layer = YamlComposer.Layer(layer_id=self._num_layer)
self._num_layer += 1
r0 = CommentedMap({'name': 'Router',
'yaml_path': None,
@@ -406,7 +410,7 @@ def rule7():
router_layers.append(router_layer)
last_layer.components[0]['port_out'] = r0['port_in']

router_layer = YamlGraph.Layer(layer_id=self._num_layer)
router_layer = YamlComposer.Layer(layer_id=self._num_layer)
self._num_layer += 1
for c in layer.components:
r = CommentedMap({'name': 'Router',
@@ -423,7 +427,7 @@ def rule7():
def rule10():
last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT)

router_layer = YamlGraph.Layer(layer_id=self._num_layer)
router_layer = YamlComposer.Layer(layer_id=self._num_layer)
self._num_layer += 1
r0 = CommentedMap({'name': 'Router',
'yaml_path': None,
@@ -441,7 +445,7 @@ def rule10():

def rule8():
last_layer.components[0]['socket_out'] = str(SocketType.PUSH_CONNECT)
router_layer = YamlGraph.Layer(layer_id=self._num_layer)
router_layer = YamlComposer.Layer(layer_id=self._num_layer)
self._num_layer += 1
r = CommentedMap({'name': 'Router',
'yaml_path': None,
@@ -475,7 +479,7 @@ def rule8():
else:
self._num_layer -= 1

router_layer = YamlGraph.Layer(layer_id=self._num_layer)
router_layer = YamlComposer.Layer(layer_id=self._num_layer)
self._num_layer += 1
router_layer.append(r)
router_layers.append(router_layer)
@@ -0,0 +1,47 @@
import tempfile

from .base import YamlComposer
from ..cli.parser import set_composer_parser


class YamlComposerFlask:
def __init__(self, args):
self.args = args

def create_flask_app(self):
try:
from flask import Flask, request, abort, redirect, url_for
from flask_compress import Compress
from flask_cors import CORS
except ImportError:
raise ImportError('Flask or its dependencies are not fully installed, '
'they are required for serving HTTP requests.'
'Please use "pip install Flask" to install it.')

# support up to 10 concurrent HTTP requests
app = Flask(__name__)

@app.route('/', methods=['GET'])
def get_homepage():
return YamlComposer(set_composer_parser().parse_args([])).build_all()['html']

@app.route('/refresh', methods=['POST'])
def regenerate():
data = request.form if request.form else request.json
f = tempfile.NamedTemporaryFile('w', delete=False).name
with open(f, 'w', encoding='utf8') as fp:
fp.write(data['yaml-config'])
try:
return YamlComposer(set_composer_parser().parse_args([
'--yaml_path', f
])).build_all()['html']
except Exception:
return 'Bad YAML input, please kindly check the format, indent and content of your YAML file!'

CORS(app, origins=self.args.cors)
Compress().init_app(app)
return app

def run(self):
app = self.create_flask_app()
app.run(port=self.args.http_port, threaded=True, host='0.0.0.0')
@@ -200,14 +200,28 @@
YAML config
</div>
<div class="card-body">
<button type="button" class="btn btn-primary" data-clipboard-target="#yaml-code">
Copy to Clipboard
</button>
<pre>
<code class="yaml" id="yaml-code">
<form action="/refresh" method="post">
<div class="card-title">
<div class="btn-group" role="group" aria-label="Basic example">
<button type="button" class="btn btn-secondary"
data-clipboard-target="#simple-yaml-config">
Copy
</button>
<input type="submit" class="btn btn-primary" value="Generate">
</div>
</div>
<div class="card-text">


<div class="form-group">
<textarea name="yaml-config" class="form-control" id="simple-yaml-config" rows="15"
placeholder="your YAML config here" required autofocus>
{{gnes-yaml}}
</code>
</pre>
</textarea>
</div>

</div>
</form>
</div>
</div>
</div>
@@ -218,7 +232,7 @@
<div class="jumbotron">

<p class="lead">This is the workflow generated from your input YAML config, which helps you
to understand how microservices work together in GNES.</p>
to understand how microservices work together in GNES.</p>
</div>
<div class="card">
<div class="card-header">

0 comments on commit b34a765

Please sign in to comment.
You can’t perform that action at this time.