Skip to content
Permalink
Browse files

refactor(compose): move compose template to resources

  • Loading branch information...
hanxiao committed Jul 19, 2019
1 parent a917bcc commit 9bbb3c0529ea92a947202bef1abfcd4a2f7bef77
Showing with 112 additions and 94 deletions.
  1. +3 −3 docker-compose/component/encoder.bas-pca.yml
  2. +3 −3 docker-compose/component/encoder.bas.yml
  3. +2 −2 docker-compose/component/encoder.inception.yml
  4. +2 −2 docker-compose/component/encoder.resnet.yml
  5. +2 −2 docker-compose/component/encoder.vgg.yml
  6. +2 −2 docker-compose/component/encoder.w2v.yml
  7. +1 −1 docker-compose/component/img_preprocessor_fasterRCNN.yml
  8. +1 −1 docker-compose/component/preprocessor.yml
  9. +1 −1 docs/chapter/microservice.rst
  10. +33 −29 gnes/composer/base.py
  11. 0 gnes/resources/{static → compose}/gnes-board.html
  12. +4 −0 gnes/resources/compose/gnes-shell.sh
  13. +8 −0 gnes/resources/compose/gnes-swarm.yml
  14. +3 −3 gnes/resources/config/encoder/default.yml
  15. +4 −4 gnes/resources/config/gnes/default.yml
  16. +1 −1 gnes/resources/config/preprocessor/default.yml
  17. +2 −2 tests/yaml/base-elmo-nes.yml
  18. +3 −3 tests/yaml/base-encoder.yml
  19. +2 −2 tests/yaml/base-eu-nes.yml
  20. +2 −2 tests/yaml/base-flair-nes.yml
  21. +2 −2 tests/yaml/base-gpt-nes.yml
  22. +2 −2 tests/yaml/base-gpt2-nes.yml
  23. +2 −2 tests/yaml/base-nes.yml
  24. +1 −1 tests/yaml/base-segmentation-image-prep.yml
  25. +2 −2 tests/yaml/bert-binary-encoder.yml
  26. +2 −2 tests/yaml/elmo-binary-encoder.yml
  27. +2 −2 tests/yaml/flair-binary-encoder.yml
  28. +2 −2 tests/yaml/gpt-binary-encoder.yml
  29. +2 −2 tests/yaml/gpt2-binary-encoder.yml
  30. +1 −1 tests/yaml/hash-encoder.yml
  31. +1 −1 tests/yaml/inception-encoder.yml
  32. +1 −1 tests/yaml/lopq-encoder-2-np.yml
  33. +1 −1 tests/yaml/lopq-encoder-2-tf.yml
  34. +1 −1 tests/yaml/lopq-encoder-3.yml
  35. +1 −1 tests/yaml/lopq-encoder.yml
  36. +1 −1 tests/yaml/mobilenet-encoder.yml
  37. +1 −1 tests/yaml/preprocessor-ffmpeg.yml
  38. +1 −1 tests/yaml/preprocessor-shotdetect.yml
  39. +1 −1 tests/yaml/preprocessor1.yml
  40. +1 −1 tests/yaml/preprocessor2.yml
  41. +1 −1 tests/yaml/resnet-encoder.yml
  42. +3 −1 tests/yaml/topology7.yml
  43. +1 −1 tests/yaml/vgg-encoder.yml
@@ -13,7 +13,7 @@ component:
model_dir: /ext_data/chinese_L-12_H-768_A-12
max_seq_len: 40
pooling_layer: -12
property:
gnes_config:
is_trained: true
bert_client:
!BertEncoder
@@ -22,11 +22,11 @@ component:
port: 5555
port_out: 5556
ignore_all_checks: true
property:
gnes_config:
is_trained: true
- !PCALocalEncoder
parameter:
output_dim: 300
num_locals: 30
property:
gnes_config:
batch_size: 2048
@@ -13,7 +13,7 @@ component:
model_dir: /ext_data/chinese_L-12_H-768_A-12
max_seq_len: 40
pooling_layer: -12
property:
gnes_config:
is_trained: true
bert_client:
!BertEncoder
@@ -22,13 +22,13 @@ component:
port: 5555
port_out: 5556
ignore_all_checks: true
property:
gnes_config:
is_trained: true
- !PCALocalEncoder
parameter:
output_dim: 300
num_locals: 30
property:
gnes_config:
batch_size: 2048
- !PQEncoder
parameter:
@@ -26,13 +26,13 @@ component:
- torch.nn.functional.adaptive_avg_pool2d(x, (1, 1))
- torch.nn.functional.dropout(x, training=False)
- x.view(x.size(0), -1)
property:
gnes_config:
is_trained: true
- !PCALocalEncoder
parameter:
output_dim: 200
num_locals: 10
property:
gnes_config:
batch_size: 2048
- !PQEncoder
parameter:
@@ -15,13 +15,13 @@ component:
- layer4
- avgpool
- x.reshape(x.size(0), -1)
property:
gnes_config:
is_trained: true
- !PCALocalEncoder
parameter:
output_dim: 200
num_locals: 10
property:
gnes_config:
batch_size: 2048
- !PQEncoder
parameter:
@@ -9,13 +9,13 @@ component:
- avg_pool
- x.view(x.size(0), -1)
- classifier[0]
property:
gnes_config:
is_trained: true
- !PCALocalEncoder
parameter:
output_dim: 200
num_locals: 10
property:
gnes_config:
batch_size: 2048
- !PQEncoder
parameter:
@@ -3,13 +3,13 @@ component:
- !Word2VecEncoder
parameter:
model_dir: /ext_data/sgns.wiki.bigram-char.refine
property:
gnes_config:
is_trained: true
- !PCALocalEncoder
parameter:
output_dim: 200
num_locals: 10
property:
gnes_config:
batch_size: 2048
- !PQEncoder
parameter:
@@ -4,5 +4,5 @@ parameter:
model_name: fasterrcnn_resnet50_fpn
target_img_size: 224
_use_cuda: false
property:
gnes_config:
is_trained: true
@@ -3,5 +3,5 @@ parameter:
start_doc_id: 0
random_doc_id: True
deliminator: "[.。!?!?]+"
property:
gnes_config:
is_trained: true
@@ -4,7 +4,7 @@ Microservices
GNES is all-in-microservice: encoder, indexer, preprocessor and router are all running statelessly and independently in their own containers. They communicate via versioned APIs and collaborate under the orchestration of Docker Swarm/Kubernetes etc. Scaling, load-balancing, automated recovering, they come off-the-shelf in GNES.


.. image:: ../.github/gnes-all-in-ms.svg
.. image:: ../../.github/gnes-all-in-ms.svg
:align: center
:width: 500px

@@ -21,7 +21,7 @@ class YamlGraph:
'Encoder': 'encode',
'Router': 'route',
'Indexer': 'index',
'Frontend': 'frontend',
'gRPCFrontend': 'frontend',
'Preprocessor': 'preprocess'
}

@@ -78,15 +78,19 @@ def __init__(self, args):
tmp = _yaml.load(args.yaml_path)
stream = StringIO()
_yaml.dump(tmp, stream)
self.original_yaml = stream.getvalue()
self.name = tmp.get('name', args.name)
self.port = tmp.get('port', args.port)
self.original_yaml = stream.getvalue()

self._name = tmp.get('name', args.name)
self._port = tmp.get('port', args.port)
self._networks = tmp.get('networks', {})
self._volumes = tmp.get('volumes', {})

self.args = args
self._num_layer = 0

if 'services' in tmp:
self.add_layer()
self.add_comp(CommentedMap({'name': 'Frontend', 'grpc_port': self.port}))
self.add_comp(CommentedMap({'name': 'gRPCFrontend', 'grpc_port': self._port}))
for comp in tmp['services']:
self.add_layer()
if isinstance(comp, list):
@@ -135,11 +139,11 @@ def build_layers(self) -> List['YamlGraph.Layer']:

@staticmethod
def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gnes/gnes:latest',
volumes: Dict = None) -> str:
swarm_lines = {'version': '3.4', 'services': {}}
volumes: Dict = None, networks: Dict = None) -> str:
with pkg_resources.resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-swarm.yml'))) as r:
swarm_lines = _yaml.load(r)
taboo = {'name', 'replicas', 'yaml_path'}
config_dict = {}
network_dict = {'gnes-network': {'driver': 'overlay', 'attachable': True}}
for l_idx, layer in enumerate(all_layers):
for c_idx, c in enumerate(layer.components):
c_name = '%s%d%d' % (c['name'], l_idx, c_idx)
@@ -149,18 +153,17 @@ def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gn
args.append('--yaml_path /%s_yaml' % c_name)
config_dict['%s_yaml' % c_name] = {'file': c['yaml_path']}

if l_idx + 1 < len(all_layers):
next_layer = all_layers[l_idx + 1]
_l_idx = l_idx + 1
else:
next_layer = all_layers[0]
_l_idx = 0

host_out_name = ''
for _c_idx, _c in enumerate(next_layer.components):
if _c['port_in'] == c['port_out']:
host_out_name = '%s%d%d' % (_c['name'], _l_idx, _c_idx)
break
# if l_idx + 1 < len(all_layers):
# next_layer = all_layers[l_idx + 1]
# _l_idx = l_idx + 1
# else:
# next_layer = all_layers[0]
# _l_idx = 0
# host_out_name = ''
# for _c_idx, _c in enumerate(next_layer.components):
# if _c['port_in'] == c['port_out']:
# host_out_name = '%s%d%d' % (_c['name'], _l_idx, _c_idx)
# break

if l_idx - 1 >= 0:
last_layer = all_layers[l_idx - 1]
@@ -179,31 +182,32 @@ def build_dockerswarm(all_layers: List['YamlGraph.Layer'], docker_img: str = 'gn
# '--host_out %s' % host_out_name]

cmd = '%s %s' % (YamlGraph.comp2file[c['name']], ' '.join(args))
swarm_lines['services'][c_name] = {
swarm_lines['services'][c_name] = CommentedMap({
'image': docker_img,
'command': cmd,
}
})

rep_c = YamlGraph.Layer.get_value(c, 'replicas')
if rep_c > 1:
swarm_lines['services'][c_name]['deploy'] = {
swarm_lines['services'][c_name]['deploy'] = CommentedMap({
'replicas': YamlGraph.Layer.get_value(c, 'replicas'),
'restart_policy': {
'condition': 'on-failure',
'max_attempts': 3,
}
}
})

if 'yaml_path' in c and c['yaml_path'] is not None:
swarm_lines['services'][c_name]['configs'] = ['%s_yaml' % c_name]

if c['name'] == 'Frontend':
if c['name'] == 'gRPCFrontend':
swarm_lines['services'][c_name]['ports'] = ['%d:%d' % (c['grpc_port'], c['grpc_port'])]

if volumes:
swarm_lines['volumes'] = volumes
if networks:
swarm_lines['networks'] = volumes
swarm_lines['configs'] = config_dict
swarm_lines['networks'] = network_dict
stream = StringIO()
_yaml.dump(swarm_lines, stream)
return stream.getvalue()
@@ -278,8 +282,7 @@ def build_mermaid(all_layers: List['YamlGraph.Layer'], mermaid_leftright: bool =

@staticmethod
def build_html(generate_dict: Dict[str, str]) -> str:
r = pkg_resources.resource_stream('gnes', '/'.join(('resources', 'static', 'gnes-board.html')))
with r:
with pkg_resources.resource_stream('gnes', '/'.join(('resources', 'compose', 'gnes-board.html'))) as r:
html = r.read().decode()
for k, v in generate_dict.items():
if v:
@@ -302,7 +305,8 @@ def std_or_print(f, content):
'mermaid': self.build_mermaid(all_layers, self.args.mermaid_leftright),
'shell': self.build_shell(all_layers, self.args.shell_log_redirect),
'yaml': self.original_yaml,
'docker': self.build_dockerswarm(all_layers, self.args.docker_img),
'docker': self.build_dockerswarm(all_layers, self.args.docker_img,
volumes=self._volumes, networks=self._networks),
'k8s': self.build_kubernetes(all_layers),
'timestamp': time.strftime("%a, %d %b %Y %H:%M:%S"),
'version': __version__
File renamed without changes.
@@ -1,5 +1,9 @@
#!/usr/bin/env bash

## Prerequirment of this script
## You need to install GNES locally on this local machine
## pip install gnes

set -e

trap 'kill $(jobs -p)' EXIT
@@ -0,0 +1,8 @@
version: '3.4'
services: {}
volumes: {}
networks:
gnes-net:
driver: overlay
attachable: true
configs: {}
@@ -13,7 +13,7 @@ component:
model_dir: /ext_data/chinese_L-12_H-768_A-12
max_seq_len: NONE
pooling_layer: -12
property:
gnes_config:
is_trained: true
bert_client:
!BertEncoder
@@ -22,13 +22,13 @@ component:
port: 5555
port_out: 5556
ignore_all_checks: true
property:
gnes_config:
is_trained: true
- !PCALocalEncoder
parameter:
output_dim: 32
num_locals: 8
property:
gnes_config:
batch_size: 2048
- !PQEncoder
parameter:
@@ -16,7 +16,7 @@ component:
model_dir: /ext_data/chinese_L-12_H-768_A-12
max_seq_len: NONE
pooling_layer: -12
property:
gnes_config:
is_trained: true
bert_client:
!BertEncoder
@@ -25,13 +25,13 @@ component:
port: 5555
port_out: 5556
ignore_all_checks: true
property:
gnes_config:
is_trained: true
- !PCALocalEncoder
parameter:
output_dim: 32
num_locals: 8
property:
gnes_config:
batch_size: 2048
- !PQEncoder
parameter:
@@ -44,5 +44,5 @@ component:
text_indexer: !LVDBIndexer
parameter:
data_path: ./test_db
property:
gnes_config:
work_dir: /ext_data/demo-model
@@ -3,5 +3,5 @@ parameter:
start_doc_id: 0
random_doc_id: True
deliminator: "[.。!?!?]+"
property:
gnes_config:
is_trained: true
@@ -8,13 +8,13 @@ component:
model_dir: $ELMO_CI_MODEL
pooling_layer: -1
pooling_stragy: REDUCE_MEAN
property:
gnes_config:
is_trained: true
- !PCALocalEncoder
parameter:
output_dim: 32
num_locals: 8
property:
gnes_config:
batch_size: 2048
- !PQEncoder
parameter:

0 comments on commit 9bbb3c0

Please sign in to comment.
You can’t perform that action at this time.