Skip to content
Permalink
Browse files

feat(compose): add minimum http server without flask dep

  • Loading branch information...
hanxiao committed Aug 22, 2019
1 parent 1cae761 commit bc2e441d94edf65ff675f5721aec4b0899115cf0
Showing with 104 additions and 19 deletions.
  1. +3 −0 gnes/cli/api.py
  2. +5 −2 gnes/cli/parser.py
  3. +23 −16 gnes/composer/base.py
  4. +67 −0 gnes/composer/http.py
  5. +0 −1 setup.py
  6. +6 −0 tests/test_compose.py
@@ -83,8 +83,11 @@ def _client_bm(args):
def compose(args):
from ..composer.base import YamlComposer
from ..composer.flask import YamlComposerFlask
from ..composer.http import YamlComposerHttp

if args.flask:
YamlComposerFlask(args).run()
elif args.serve:
YamlComposerHttp(args).run()
else:
YamlComposer(args).build_all()
@@ -103,8 +103,11 @@ def set_composer_flask_parser(parser=None):
if not parser:
parser = set_base_parser()
set_composer_parser(parser)
parser.add_argument('--flask', action='store_true', default=False,
help='using Flask to serve a composer in interactive mode, aka GNES board')
group = parser.add_mutually_exclusive_group()
group.add_argument('--flask', action='store_true', default=False,
help='start a Flask server and serve the composer in interactive mode, aka GNES board')
group.add_argument('--serve', action='store_true', default=False,
help='start a basic HTTP server and serve the composer in interactive mode, aka GNES board')
parser.add_argument('--http_port', type=int, default=8080,
help='server port for receiving HTTP requests')
return parser
@@ -54,8 +54,10 @@ class Layer:
default_values = {
'name': None,
'yaml_path': None,
'py_path': None,
'image': None,
'replicas': 1,
'income': 'pull'
'income': 'pull',
}

def __init__(self, layer_id: int = 0):
@@ -115,12 +117,15 @@ def __init__(self, args):

if 'services' in tmp:
self.add_layer()
self.add_comp(CommentedMap({'name': 'Frontend', 'grpc_port': self._port}))
c = CommentedMap({'name': 'Frontend', 'grpc_port': self._port})
if self.check_fields(c):
self.add_comp(c)
for comp in tmp['services']:
self.add_layer()
if isinstance(comp, list):
for c in comp:
self.add_comp(c)
if self.check_fields(c):
self.add_comp(c)
elif self.check_fields(comp):
self.add_comp(comp)
else:
@@ -134,13 +139,12 @@ def check_fields(self, comp: Dict) -> bool:
if comp['name'] not in self.comp2file:
raise AttributeError(
'a component must be one of: %s, but given %s' % (', '.join(self.comp2file.keys()), comp['name']))
if 'yaml_path' not in comp:
self.logger.warning(
'found empty "yaml_path", '
'i will use a default config and would probably result in an empty model')
for k in comp:
if k not in self.Layer.default_values:
self.logger.warning('your yaml contains an unrecognized key named "%s"' % k)
for k, v in self.Layer.default_values.items():
if k not in comp:
comp[k] = v
return True

def add_layer(self, layer: 'Layer' = None) -> None:
@@ -226,7 +230,7 @@ def build_dockerswarm(all_layers: List['YamlComposer.Layer'], docker_img: str =

cmd = '%s %s' % (YamlComposer.comp2file[c['name']], ' '.join(args))
swarm_lines['services'][c_name] = CommentedMap({
'image': docker_img,
'image': c['image'] or docker_img,
'command': cmd,
})

@@ -333,14 +337,10 @@ def build_html(generate_dict: Dict[str, str]) -> str:

def build_all(self):
def std_or_print(f, content):
if content:
if f:
with f as fp:
fp.write(content)
self.logger.info('generated content is written to %s' % f)
else:
self.logger.warning('no file path is defined, i will just print it to stdout')
print(content)
if content and f:
with f as fp:
fp.write(content)
self.logger.info('generated content is written to %s' % f)

all_layers = self.build_layers()
cmds = {
@@ -394,6 +394,7 @@ def rule3():
'socket_out': str(SocketType.PUSH_BIND),
'port_in': last_layer.components[0]['port_out'],
'port_out': self._get_random_port()})
self.check_fields(r)
for c in layer.components:
c['socket_in'] = str(SocketType.PULL_CONNECT)
c['port_in'] = r['port_out']
@@ -428,6 +429,7 @@ def rule6():
SocketType.PUB_BIND),
'port_in': last_layer.components[0]['port_out'],
'port_out': self._get_random_port()})
self.check_fields(r)
c['socket_in'] = str(SocketType.PULL_CONNECT) if income == 'pull' else str(SocketType.SUB_CONNECT)
c['port_in'] = r['port_out']
router_layer.append(r)
@@ -444,6 +446,7 @@ def rule7():
'socket_out': str(SocketType.PUB_BIND),
'port_in': self._get_random_port(),
'port_out': self._get_random_port()})
self.check_fields(r0)
router_layer.append(r0)
router_layers.append(router_layer)
last_layer.components[0]['port_out'] = r0['port_in']
@@ -459,6 +462,7 @@ def rule7():
'port_out': self._get_random_port()})
c['socket_in'] = str(SocketType.PULL_CONNECT)
c['port_in'] = r['port_out']
self.check_fields(r)
router_layer.append(r)
router_layers.append(router_layer)

@@ -473,6 +477,7 @@ def rule10():
'socket_out': str(SocketType.PUB_BIND),
'port_in': self._get_random_port(),
'port_out': self._get_random_port()})
self.check_fields(r0)
router_layer.append(r0)
router_layers.append(router_layer)
last_layer.components[0]['port_out'] = r0['port_in']
@@ -490,6 +495,7 @@ def rule8():
'socket_out': str(SocketType.PUSH_BIND),
'port_in': self._get_random_port(),
'port_out': self._get_random_port()})
self.check_fields(r)

for c in last_layer.components:
last_income = self.Layer.get_value(c, 'income')
@@ -502,6 +508,7 @@ def rule8():
'port_in': self._get_random_port(),
'port_out': r['port_in']})
c['port_out'] = r_c['port_in']
self.check_fields(r_c)
router_layer.append(r_c)
elif last_income == 'pull':
c['socket_out'] = str(SocketType.PUSH_CONNECT)
@@ -0,0 +1,67 @@
import io
from http.server import BaseHTTPRequestHandler, HTTPServer
from urllib.parse import parse_qs

from .base import YamlComposer
from ..cli.parser import set_composer_parser
from ..helper import set_logger


class YamlComposerHttp:
def __init__(self, args):
self.args = args
self.logger = set_logger(self.__class__.__name__, self.args.verbose)

class _HttpServer(BaseHTTPRequestHandler):
args = set_composer_parser().parse_args([])
default_html = YamlComposer(args).build_all()['html']

def _set_response(self, msg: str, code: int = 200):
self.send_response(code)
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write(msg.encode('utf-8'))

def do_GET(self):
if str(self.path) != '/':
self._set_response('<h1>"%s" is not a valid entrypoint</h1>' % self.path, 400)
return
self._set_response(self.default_html)

def do_POST(self):
if str(self.path) != '/generate':
self._set_response('<h1>"%s" is not a valid entrypoint</h1>' % self.path, 400)
return
content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
data = self.rfile.read(content_length) # <--- Gets the data itself

data = {k: v[0] for k, v in parse_qs(data.decode('utf-8')).items()}
if not data or 'yaml-config' not in data:
self._set_response('<h1>Bad POST request</h1> your POST request does not contain "yaml-config" field!',
406)
try:
self.args.yaml_path = io.StringIO(data['yaml-config'])
if data.get('mermaid_direction', 'top-down').lower() == 'left-right':
self.args.mermaid_leftright = True
else:
self.args.mermaid_leftright = False
if 'docker-image' in data:
self.args.docker_img = data['docker-image']
else:
self.args.docker_img = 'gnes/gnes:latest-alpine'

self._set_response(YamlComposer(self.args).build_all()['html'])
except Exception as e:
self._set_response(
'<h1>Bad YAML input</h1> please kindly check the format, '
'indent and content of your YAML file! <h3>Traceback: </h3><p><code>%s</code></p>' % e,
400)

def run(self):
httpd = HTTPServer(('0.0.0.0', self.args.http_port), self._HttpServer)
try:
httpd.serve_forever()
except KeyboardInterrupt:
pass
finally:
httpd.server_close()
@@ -54,7 +54,6 @@
'ruamel.yaml>=0.15.89',
'pyzmq>=17.1.0',
'aiohttp',
'flask'
]

# using pip install gnes[xx] is depreciated
@@ -4,6 +4,7 @@
from gnes.cli.parser import set_composer_parser, set_composer_flask_parser
from gnes.composer.base import YamlComposer
from gnes.composer.flask import YamlComposerFlask
from gnes.composer.http import YamlComposerHttp


class TestCompose(unittest.TestCase):
@@ -33,6 +34,11 @@ def _test_topology(self, yaml_path: str, num_layer_before: int, num_layer_after:
os.path.exists(self.html_path)
print(a.build_dockerswarm(r))

@unittest.SkipTest
def test_flask_local(self):
args = set_composer_flask_parser().parse_args(['--serve'])
YamlComposerHttp(args).run()

@unittest.SkipTest
def test_flask_local(self):
args = set_composer_flask_parser().parse_args(['--flask'])

0 comments on commit bc2e441

Please sign in to comment.
You can’t perform that action at this time.