Skip to content
Permalink
Browse files

feat(parser): improve yaml_path parsing

  • Loading branch information...
hanxiao committed Aug 6, 2019
1 parent 12a5304 commit 77a2ea42374a29615ac70d42d4c3e2abc91b7034
Showing with 28 additions and 18 deletions.
  1. +22 −18 gnes/cli/parser.py
  2. +6 −0 tests/test_batching.py
@@ -19,6 +19,23 @@
import argparse


def resolve_yaml_path(path):
# priority, filepath > classname > default
import os
import io
if hasattr(path, 'read'):
# already a readable stream
return path
elif os.path.exists(path):
return open(path, encoding='utf8')
elif path.isidentifier():
# possible class name
return io.StringIO('!%s {}' % path)
else:
raise argparse.ArgumentTypeError('%s can not be resolved, it should be a readable stream,'
' or a valid file path, or a supported class name.' % path)


def set_base_parser():
from .. import __version__
from termcolor import colored
@@ -49,7 +66,7 @@ def set_composer_parser(parser=None):
type=str,
default='GNES app',
help='name of the instance')
parser.add_argument('--yaml_path', type=argparse.FileType('r'),
parser.add_argument('--yaml_path', type=resolve_yaml_path,
default=resource_stream(
'gnes', '/'.join(('resources', 'config', 'compose', 'default.yml'))),
help='yaml config of the service')
@@ -134,14 +151,12 @@ def _set_client_parser(parser=None):
def set_loadable_service_parser(parser=None):
if not parser:
parser = set_base_parser()
from pkg_resources import resource_stream
from ..service.base import SocketType
set_service_parser(parser)

parser.add_argument('--yaml_path', type=argparse.FileType('r'),
default=resource_stream(
'gnes', '/'.join(('resources', 'config', 'encoder', 'default.yml'))),
help='yaml config of the service')
parser.add_argument('--yaml_path', type=resolve_yaml_path,
help='yaml config of the service, it should be a readable stream,'
' or a valid file path, or a supported class name.')

parser.set_defaults(socket_in=SocketType.PULL_BIND,
socket_out=SocketType.PUSH_BIND)
@@ -151,39 +166,28 @@ def set_loadable_service_parser(parser=None):
def set_preprocessor_service_parser(parser=None):
if not parser:
parser = set_base_parser()
from pkg_resources import resource_stream
set_loadable_service_parser(parser)
parser.get_default('yaml_path').close()
parser.set_defaults(yaml_path=resource_stream(
'gnes', '/'.join(('resources', 'config', 'preprocessor', 'default.yml'))))

parser.set_defaults(read_only=True)
return parser


def set_router_service_parser(parser=None):
from pkg_resources import resource_stream
if not parser:
parser = set_base_parser()
set_loadable_service_parser(parser)
parser.get_default('yaml_path').close()
parser.set_defaults(yaml_path=resource_stream(
'gnes', '/'.join(('resources', 'config', 'router', 'default.yml'))))
parser.set_defaults(yaml_path='BaseRouter')

parser.set_defaults(read_only=True)
return parser


def set_indexer_service_parser(parser=None):
from ..service.base import SocketType
from pkg_resources import resource_stream

if not parser:
parser = set_base_parser()
set_loadable_service_parser(parser)
parser.get_default('yaml_path').close()
parser.set_defaults(yaml_path=resource_stream(
'gnes', '/'.join(('resources', 'config', 'indexer', 'default.yml'))))

# encoder's port_out is indexer's port_in
parser.set_defaults(port_in=parser.get_default('port_out'),
@@ -98,4 +98,10 @@ def test_mini_batch(self):
def _do_mini_batch(_, y):
return y

# this will follow self.batch_size, which is None
@batching
def _do_mini_batch2(_, y):
return y

self.assertEqual(_do_mini_batch(None, x), [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9]])
self.assertEqual(_do_mini_batch2(None, x), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])

0 comments on commit 77a2ea4

Please sign in to comment.
You can’t perform that action at this time.