diff --git a/gnes/cli/parser.py b/gnes/cli/parser.py index 8b69d988..75ea08c6 100644 --- a/gnes/cli/parser.py +++ b/gnes/cli/parser.py @@ -157,7 +157,7 @@ def set_loadable_service_parser(parser=None): from ..service.base import SocketType set_service_parser(parser) - parser.add_argument('--yaml_path', type=resolve_yaml_path, + parser.add_argument('--yaml_path', type=resolve_yaml_path, required=True, help='yaml config of the service, it should be a readable stream,' ' or a valid file path, or a supported class name.') @@ -179,8 +179,6 @@ def set_router_service_parser(parser=None): if not parser: parser = set_base_parser() set_loadable_service_parser(parser) - parser.set_defaults(yaml_path='BaseRouter') - parser.set_defaults(read_only=True) return parser diff --git a/gnes/composer/base.py b/gnes/composer/base.py index 6e4b9160..c92a6faf 100644 --- a/gnes/composer/base.py +++ b/gnes/composer/base.py @@ -43,11 +43,11 @@ class YamlComposer: } comp2args = { - 'Encoder': set_loadable_service_parser().parse_args([]), - 'Router': set_router_service_parser().parse_args([]), - 'Indexer': set_indexer_service_parser().parse_args([]), + 'Encoder': set_loadable_service_parser().parse_args(['--yaml_path', 'BaseEncoder']), + 'Router': set_router_service_parser().parse_args(['--yaml_path', 'BaseRouter']), + 'Indexer': set_indexer_service_parser().parse_args(['--yaml_path', 'BaseIndexer']), 'gRPCFrontend': set_grpc_frontend_parser().parse_args([]), - 'Preprocessor': set_preprocessor_service_parser().parse_args([]) + 'Preprocessor': set_preprocessor_service_parser().parse_args(['--yaml_path', 'BasePreprocessor']) } class Layer: diff --git a/tests/test_preprocessor.py b/tests/test_preprocessor.py index 835f76f3..a63359fd 100644 --- a/tests/test_preprocessor.py +++ b/tests/test_preprocessor.py @@ -14,14 +14,15 @@ def setUp(self): self.single_cn = '矫矫珍木巅，得无金丸惧。' self.single_en = 'When forty winters shall besiege thy brow. And dig deep trenches in thy beautys field.' self.dirname = os.path.dirname(__file__) + self.yaml_path = os.path.join(self.dirname, 'yaml', 'test-preprocessor.yml') def test_preprocessor_service_empty(self): - args = set_preprocessor_service_parser().parse_args([]) + args = set_preprocessor_service_parser().parse_args(['--yaml_path', 'BasePreprocessor']) with PreprocessorService(args): pass def test_preprocessor_service_echo(self): - args = set_preprocessor_service_parser().parse_args([]) + args = set_preprocessor_service_parser().parse_args(['--yaml_path', 'BasePreprocessor']) c_args = _set_client_parser().parse_args([ '--port_in', str(args.port_out), '--port_out', str(args.port_in) @@ -38,7 +39,7 @@ def test_preprocessor_service_echo(self): print(r) def test_preprocessor_service_realdata(self): - args = set_preprocessor_service_parser().parse_args([]) + args = set_preprocessor_service_parser().parse_args(['--yaml_path', self.yaml_path]) c_args = _set_client_parser().parse_args([ '--port_in', str(args.port_out), '--port_out', str(args.port_in) diff --git a/tests/test_router.py b/tests/test_router.py index 4e751d36..8fe587ec 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -22,7 +22,7 @@ def setUp(self): self.concat_router_yaml = 'ConcatEmbedRouter' def test_service_empty(self): - args = set_router_service_parser().parse_args([]) + args = set_router_service_parser().parse_args(['--yaml_path', 'BaseRouter']) with RouterService(args): pass @@ -287,25 +287,29 @@ def test_multimap_multireduce(self): '--socket_in', str(SocketType.SUB_CONNECT), '--socket_out', str(SocketType.PUSH_CONNECT), '--port_in', str(p21.port_out), - '--port_out', str(r41.port_in) + '--port_out', str(r41.port_in), + '--yaml_path', 'BaseRouter' ]) r312 = set_router_service_parser().parse_args([ '--socket_in', str(SocketType.SUB_CONNECT), '--socket_out', str(SocketType.PUSH_CONNECT), '--port_in', str(p21.port_out), - '--port_out', str(r41.port_in) + '--port_out', str(r41.port_in), + '--yaml_path', 'BaseRouter' ]) r321 = set_router_service_parser().parse_args([ '--socket_in', str(SocketType.SUB_CONNECT), '--socket_out', str(SocketType.PUSH_CONNECT), '--port_in', str(p22.port_out), - '--port_out', str(r42.port_in) + '--port_out', str(r42.port_in), + '--yaml_path', 'BaseRouter' ]) r322 = set_router_service_parser().parse_args([ '--socket_in', str(SocketType.SUB_CONNECT), '--socket_out', str(SocketType.PUSH_CONNECT), '--port_in', str(p22.port_out), - '--port_out', str(r42.port_in) + '--port_out', str(r42.port_in), + '--yaml_path', 'BaseRouter' ]) c_args = _set_client_parser().parse_args([ diff --git a/tests/test_stream_grpc.py b/tests/test_stream_grpc.py index f500fd20..0ae92da3 100644 --- a/tests/test_stream_grpc.py +++ b/tests/test_stream_grpc.py @@ -49,6 +49,7 @@ def test_grpc_frontend(self): '--port_out', str(args.port_in), '--socket_in', str(SocketType.PULL_CONNECT), '--socket_out', str(SocketType.PUSH_CONNECT), + '--yaml_path', 'BaseRouter' ]) with RouterService(p_args), GRPCFrontend(args), grpc.insecure_channel( @@ -72,6 +73,7 @@ def test_async_block(self): '--port_out', '8899', '--socket_in', str(SocketType.PULL_CONNECT), '--socket_out', str(SocketType.PUSH_CONNECT), + '--yaml_path', 'BaseRouter' ]) p2_args = set_router_service_parser().parse_args([ @@ -79,6 +81,7 @@ def test_async_block(self): '--port_out', str(args.port_in), '--socket_in', str(SocketType.PULL_BIND), '--socket_out', str(SocketType.PUSH_CONNECT), + '--yaml_path', 'BaseRouter' ]) with GRPCFrontend(args), Router1(p1_args), Router2(p2_args), grpc.insecure_channel( diff --git a/tests/yaml/test-preprocessor.yml b/tests/yaml/test-preprocessor.yml new file mode 100644 index 00000000..fe7f0fca --- /dev/null +++ b/tests/yaml/test-preprocessor.yml @@ -0,0 +1,7 @@ +!TextPreprocessor +parameter: + start_doc_id: 0 + random_doc_id: True + deliminator: "[.。！？!?]+" +gnes_config: + is_trained: true \ No newline at end of file