Skip to content
This repository has been archived by the owner on Feb 22, 2020. It is now read-only.


feat(composer): add shell script generator
Browse files Browse the repository at this point in the history
  • Loading branch information
hanhxiao committed Jul 16, 2019
1 parent 033a4b9 commit 08aa30f
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 9 deletions.
36 changes: 28 additions & 8 deletions gnes/composer/
Expand Up @@ -5,6 +5,7 @@
import pkg_resources
from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap
from termcolor import colored

from ..helper import set_logger
from ..service.base import SocketType
Expand All @@ -14,11 +15,11 @@

class YamlGraph:
comp2file = {
'Encoder': 'service.encoder.EncoderService',
'Router': 'service.router.RouterService',
'Indexer': 'service.indexer.IndexerService',
'Frontend': 'service.grpc.GRPCFrontend',
'Preprocessor': 'service.preprocessor.PreprocessorService'
'Encoder': 'encode',
'Router': 'route',
'Indexer': 'index',
'Frontend': 'frontend',
'Preprocessor': 'preprocess'

class Layer:
Expand Down Expand Up @@ -99,8 +100,8 @@ def check_fields(self, comp: Dict) -> bool:
'a component must be one of: %s, but given %s' % (', '.join(self.comp2file.keys()), comp['name']))
if 'yaml_path' not in comp and 'dump_path' not in comp:
'you did not specify "yaml_path" and "dump_path", '
'will use a default config and would probably result in an empty model')
'found empty "yaml_path" and "dump_path", '
'i will use a default config and would probably result in an empty model')
for k in comp:
if k not in self.Layer.default_values:
self.logger.warning('your yaml contains an unrecognized key named "%s"' % k)
Expand All @@ -125,6 +126,25 @@ def build_layers(self) -> List['YamlGraph.Layer']:
all_layers[0] = copy.deepcopy(self._layers[0])
return all_layers

def build_shell(all_layers: List['YamlGraph.Layer']):
shell_lines = []
taboo = {'name', 'replicas'}
for layer in all_layers:
for c in layer.components:
rep_c = YamlGraph.Layer.get_value(c, 'replicas')
shell_lines.append('printf "starting service %s with %s replicas...\\n"' % (
colored(c['name'], 'green'), colored(rep_c, 'yellow')))
for j in range(rep_c):
cmd = YamlGraph.comp2file[c['name']]
args = ' '.join(['--%s %s' % (a, v if ' ' not in v else ('"%s"' % v)) for a, v in c.items() if a not in taboo and v])
shell_lines.append('gnes %s %s &' % (cmd, args))

r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'compose', '')))
with r:
return'{{gnes-template}}', '\n'.join(shell_lines))

def build_mermaid(all_layers: List['YamlGraph.Layer'], show_topdown: bool = True):
mermaid_graph = []
Expand Down Expand Up @@ -164,7 +184,7 @@ def build_html(self, mermaid_str: str):
r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'static', 'gnes-graph.html')))
with r, self.args.html_path as fp:
html ='@mermaid-graph', mermaid_str)
html ='{{gnes-template}}', mermaid_str)

Expand Down
7 changes: 7 additions & 0 deletions gnes/resources/compose/
@@ -0,0 +1,7 @@
#!/usr/bin/env bash

set -e

trap 'kill $(jobs -p)' EXIT

2 changes: 1 addition & 1 deletion gnes/resources/static/gnes-graph.html
Expand Up @@ -13,7 +13,7 @@
<div class="mermaid">
<script src=""></script>
Expand Down
1 change: 1 addition & 0 deletions tests/
Expand Up @@ -28,6 +28,7 @@ def _test_topology(self, yaml_path: str, num_layer_before: int, num_layer_after:
for c in r:

# def tearDown(self):
Expand Down

0 comments on commit 08aa30f

Please sign in to comment.