Skip to content
Permalink
Browse files

fix(compose): fix compose bug of pub-sub rule, duplicate yaml_path

  • Loading branch information...
hanxiao committed Jul 24, 2019
1 parent 7350ebc commit 139a02d91983a07eac4d8a9650b1ab48946fc864
Showing with 28 additions and 18 deletions.
  1. +4 −3 README.md
  2. +12 −13 gnes/composer/base.py
  3. +2 −2 gnes/resources/compose/gnes-board.html
  4. +10 −0 tests/test_compose.py
@@ -157,6 +157,7 @@ Either way, if you see the following message after `$ gnes` or `$ docker run gne
* [Runtime](#runtime)
- [Build your first GNES app](#build-your-first-gnes-app)
- [Scale your GNES app](#scale-your-gnes-app)
- [Customize GNES on your need](#customize-gnes-on-your-need)
- [Take-home messages](#take-home-messages)
* [👨‍💻️What's next?](#-whats-next)

@@ -422,7 +423,7 @@ services:
</pre>
</td>
<td width="70%">
<img src=".github/mermaid-diagram-20190724110437.svg" alt="GNES workflow of example 3" width="60%">
<img src=".github/mermaid-diagram-20190724110437.svg" alt="GNES workflow of example 3" width="50%">
</td>
</tr>
<tr>
@@ -442,7 +443,7 @@ services:
</tr>
<tr>
<td width="30%">
Vector-only indexing with 3 shards
Index-time with 3 vector-index shards
<pre lang="yaml">
port: 5566
services:
@@ -458,7 +459,7 @@ services:
</tr>
<tr>
<td width="30%">
Query-time with 2 vector-index shards and 3 full-text-index shards
Query-time with 2 vector-index shards followed by 3 full-text-index shards
<pre lang="yaml">
port: 5566
services:
@@ -22,7 +22,6 @@
from pkg_resources import resource_stream
from ruamel.yaml import YAML, StringIO
from ruamel.yaml.comments import CommentedMap
from termcolor import colored

from .. import __version__
from ..cli.parser import set_grpc_frontend_parser, \
@@ -104,7 +103,7 @@ def __init__(self, args):
tmp = _yaml.load(args.yaml_path)
stream = StringIO()
_yaml.dump(tmp, stream)
self.original_yaml = stream.getvalue()
self.original_yaml = stream.getvalue().strip()

self._name = tmp.get('name', args.name)
self._port = tmp.get('port', args.port)
@@ -173,7 +172,7 @@ def build_dockerswarm(all_layers: List['YamlComposer.Layer'], docker_img: str =
for c_idx, c in enumerate(layer.components):
c_name = '%s%d%d' % (c['name'], l_idx, c_idx)
args = ['--%s %s' % (a, str(v) if ' ' not in str(v) else ('"%s"' % str(v))) for a, v in c.items() if
a in YamlComposer.comp2args[c['name']] and v]
a in YamlComposer.comp2args[c['name']] and a != 'yaml_path' and v]
if 'yaml_path' in c and c['yaml_path'] is not None:
args.append('--yaml_path /%s_yaml' % c_name)
config_dict['%s_yaml' % c_name] = {'file': c['yaml_path']}
@@ -235,7 +234,7 @@ def build_dockerswarm(all_layers: List['YamlComposer.Layer'], docker_img: str =
swarm_lines['configs'] = config_dict
stream = StringIO()
_yaml.dump(swarm_lines, stream)
return stream.getvalue()
return stream.getvalue().strip()

@staticmethod
def build_kubernetes(all_layers: List['YamlComposer.Layer'], *args, **kwargs):
@@ -247,8 +246,8 @@ def build_shell(all_layers: List['YamlComposer.Layer'], log_redirect: str = None
for layer in all_layers:
for c in layer.components:
rep_c = YamlComposer.Layer.get_value(c, 'replicas')
shell_lines.append('printf "starting service %s with %s replicas...\\n"' % (
colored(c['name'], 'green'), colored(rep_c, 'yellow')))
shell_lines.append('printf "starting service \\e[1;33m%s\\e[0m with \e[1;33m%s\e[0m replicas...\\n"' % (
c['name'], rep_c))
for _ in range(rep_c):
cmd = YamlComposer.comp2file[c['name']]
args = ' '.join(
@@ -258,7 +257,7 @@ def build_shell(all_layers: List['YamlComposer.Layer'], log_redirect: str = None
cmd, args, '>> %s 2>&1' % log_redirect if log_redirect else ''))

with resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-shell.sh'))) as r:
return r.read().decode().replace('{{gnes-template}}', '\n'.join(shell_lines))
return r.read().decode().replace('{{gnes-template}}', '\n'.join(shell_lines)).strip()

@staticmethod
def build_mermaid(all_layers: List['YamlComposer.Layer'], mermaid_leftright: bool = False) -> str:
@@ -301,7 +300,7 @@ def build_mermaid(all_layers: List['YamlComposer.Layer'], mermaid_leftright: boo
class_def = ['class %s %s;' % (','.join(v), k) for k, v in cls_dict.items()]
mermaid_str = '\n'.join(
['graph %s' % ('LR' if mermaid_leftright else 'TD')] + mermaid_graph + style + class_def)
return mermaid_str
return mermaid_str.strip()

@staticmethod
def build_html(generate_dict: Dict[str, str]) -> str:
@@ -310,7 +309,7 @@ def build_html(generate_dict: Dict[str, str]) -> str:
for k, v in generate_dict.items():
if v:
html = html.replace('{{gnes-%s}}' % k, v)
return html
return html.strip()

def build_all(self):
def std_or_print(f, content):
@@ -516,13 +515,13 @@ def rule9():
rule1()
elif layer.is_homo_multi_component:
# 1-to-(N)
last_income = self.Layer.get_value(last_layer.components[0], 'income')
if last_income == 'pull':
income = self.Layer.get_value(layer.components[0], 'income')
if income == 'pull':
rule1()
elif last_income == 'sub':
elif income == 'sub':
rule2()
else:
raise NotImplementedError('replica type: %s is not recognized!' % last_income)
raise NotImplementedError('replica type: %s is not recognized!' % income)
elif layer.is_heto_single_component:
# 1-to-(1)&(1)&(1)
rule4()
@@ -294,9 +294,9 @@
You can use it to start a Docker Swarm distributed on multiple machines.</p>
<hr class="my-4">
<p> 1. Install Docker and Docker Swarm<br>
2. Create a new file say <code>my-compose.yml</code><br>
2. Create a new file say <code>my-gnes.yml</code><br>
3. Copy the following content to it<br>
4. Run <code>docker stack deploy --compose-file my-compose.yml</code>.</p>
4. Run <code>docker stack deploy --compose-file my-gnes.yml</code>.</p>
</div>
<div class="card">
<div class="card-header">
@@ -33,6 +33,16 @@ def _test_topology(self, yaml_path: str, num_layer_before: int, num_layer_after:
os.path.exists(self.html_path)
print(a.build_dockerswarm(r))

@unittest.SkipTest
def test_flask_local(self):
yaml_path = os.path.join(self.dirname, 'yaml', 'topology1.yml')
args = set_composer_flask_parser().parse_args([
'--flask',
'--yaml_path', yaml_path,
'--html_path', self.html_path
])
YamlComposerFlask(args).run()

def test_flask(self):
yaml_path = os.path.join(self.dirname, 'yaml', 'topology1.yml')
args = set_composer_flask_parser().parse_args([

0 comments on commit 139a02d

Please sign in to comment.
You can’t perform that action at this time.