diff --git a/README.md b/README.md index f99f633..f90d802 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,8 @@ # pythonsdk Python SDK for gaia pipelines. + +## Upgrade Protc / gRPC +``` +protoc -I . --python_out=. gaiasdk/plugin.proto +python3 -m grpc_tools.protoc -I . --python_out=. --grpc_python_out=. gaiasdk/plugin.proto +``` diff --git a/gaiasdk/plugin.proto b/gaiasdk/plugin.proto new file mode 100644 index 0000000..b65bcec --- /dev/null +++ b/gaiasdk/plugin.proto @@ -0,0 +1,59 @@ +// plugin.proto +// Defines the gRPC interface between gaia and the user defined +// pipelines (plugins). All rpc Methods are called from Gaia and +// executed in the plugin. + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "io.gaiapipeline.proto"; +option java_outer_classname = "GRPCPlugin"; + +package proto; + +// Job represents a single job +message Job { + uint32 unique_id = 1; + string title = 2; + string description = 3; + repeated uint32 dependson = 4; + repeated Argument args = 5; + ManualInteraction interaction = 6; +} + +// Argument represents an argument passed from a pipeline +// to gaia and/or from gaia to the pipeline. +message Argument { + string description = 1; + string type = 2; + string key = 3; + string value = 4; +} + +// ManualInteraction represents a manual human interaction +message ManualInteraction { + string description = 1; + string type = 2; + string value = 3; +} + +// JobResult represents the result of an executed job +message JobResult { + uint32 unique_id = 1; + bool failed = 2; + bool exit_pipeline = 3; + string message = 4; +} + +// Empty message +message Empty {} + +service Plugin { + // GetJobs returns a stream of Job objects. + // Used to expose jobs to gaia. + rpc GetJobs(Empty) returns (stream Job); + + // ExecuteJob signals the plugin to execute the given job. + // Used to execute one job from a pipeline. + rpc ExecuteJob(Job) returns (JobResult); +} \ No newline at end of file diff --git a/gaiasdk/plugin_pb2.py b/gaiasdk/plugin_pb2.py index efc22ab..b0e9cb0 100644 --- a/gaiasdk/plugin_pb2.py +++ b/gaiasdk/plugin_pb2.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: gaiasdk/plugin.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +"""Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database @@ -15,264 +14,15 @@ -DESCRIPTOR = _descriptor.FileDescriptor( - name='gaiasdk/plugin.proto', - package='proto', - syntax='proto3', - serialized_options=_b('\n\025io.gaiapipeline.protoB\nGRPCPluginP\001'), - serialized_pb=_b('\n\x14gaiasdk/plugin.proto\x12\x05proto\"\x9d\x01\n\x03Job\x12\x11\n\tunique_id\x18\x01 \x01(\r\x12\r\n\x05title\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x11\n\tdependson\x18\x04 \x03(\r\x12\x1d\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x0f.proto.Argument\x12-\n\x0binteraction\x18\x06 \x01(\x0b\x32\x18.proto.ManualInteraction\"I\n\x08\x41rgument\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0b\n\x03key\x18\x03 \x01(\t\x12\r\n\x05value\x18\x04 \x01(\t\"E\n\x11ManualInteraction\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\r\n\x05value\x18\x03 \x01(\t\"V\n\tJobResult\x12\x11\n\tunique_id\x18\x01 \x01(\r\x12\x0e\n\x06\x66\x61iled\x18\x02 \x01(\x08\x12\x15\n\rexit_pipeline\x18\x03 \x01(\x08\x12\x0f\n\x07message\x18\x04 \x01(\t\"\x07\n\x05\x45mpty2[\n\x06Plugin\x12%\n\x07GetJobs\x12\x0c.proto.Empty\x1a\n.proto.Job0\x01\x12*\n\nExecuteJob\x12\n.proto.Job\x1a\x10.proto.JobResultB%\n\x15io.gaiapipeline.protoB\nGRPCPluginP\x01\x62\x06proto3') -) - - - - -_JOB = _descriptor.Descriptor( - name='Job', - full_name='proto.Job', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='unique_id', full_name='proto.Job.unique_id', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='title', full_name='proto.Job.title', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='description', full_name='proto.Job.description', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='dependson', full_name='proto.Job.dependson', index=3, - number=4, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='args', full_name='proto.Job.args', index=4, - number=5, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='interaction', full_name='proto.Job.interaction', index=5, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=32, - serialized_end=189, -) - - -_ARGUMENT = _descriptor.Descriptor( - name='Argument', - full_name='proto.Argument', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='description', full_name='proto.Argument.description', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='type', full_name='proto.Argument.type', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='key', full_name='proto.Argument.key', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='proto.Argument.value', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=191, - serialized_end=264, -) - +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14gaiasdk/plugin.proto\x12\x05proto\"\x9d\x01\n\x03Job\x12\x11\n\tunique_id\x18\x01 \x01(\r\x12\r\n\x05title\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x11\n\tdependson\x18\x04 \x03(\r\x12\x1d\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x0f.proto.Argument\x12-\n\x0binteraction\x18\x06 \x01(\x0b\x32\x18.proto.ManualInteraction\"I\n\x08\x41rgument\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0b\n\x03key\x18\x03 \x01(\t\x12\r\n\x05value\x18\x04 \x01(\t\"E\n\x11ManualInteraction\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\r\n\x05value\x18\x03 \x01(\t\"V\n\tJobResult\x12\x11\n\tunique_id\x18\x01 \x01(\r\x12\x0e\n\x06\x66\x61iled\x18\x02 \x01(\x08\x12\x15\n\rexit_pipeline\x18\x03 \x01(\x08\x12\x0f\n\x07message\x18\x04 \x01(\t\"\x07\n\x05\x45mpty2[\n\x06Plugin\x12%\n\x07GetJobs\x12\x0c.proto.Empty\x1a\n.proto.Job0\x01\x12*\n\nExecuteJob\x12\n.proto.Job\x1a\x10.proto.JobResultB%\n\x15io.gaiapipeline.protoB\nGRPCPluginP\x01\x62\x06proto3') -_MANUALINTERACTION = _descriptor.Descriptor( - name='ManualInteraction', - full_name='proto.ManualInteraction', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='description', full_name='proto.ManualInteraction.description', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='type', full_name='proto.ManualInteraction.type', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='proto.ManualInteraction.value', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=266, - serialized_end=335, -) -_JOBRESULT = _descriptor.Descriptor( - name='JobResult', - full_name='proto.JobResult', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='unique_id', full_name='proto.JobResult.unique_id', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='failed', full_name='proto.JobResult.failed', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='exit_pipeline', full_name='proto.JobResult.exit_pipeline', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='message', full_name='proto.JobResult.message', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=337, - serialized_end=423, -) - - -_EMPTY = _descriptor.Descriptor( - name='Empty', - full_name='proto.Empty', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=425, - serialized_end=432, -) - -_JOB.fields_by_name['args'].message_type = _ARGUMENT -_JOB.fields_by_name['interaction'].message_type = _MANUALINTERACTION -DESCRIPTOR.message_types_by_name['Job'] = _JOB -DESCRIPTOR.message_types_by_name['Argument'] = _ARGUMENT -DESCRIPTOR.message_types_by_name['ManualInteraction'] = _MANUALINTERACTION -DESCRIPTOR.message_types_by_name['JobResult'] = _JOBRESULT -DESCRIPTOR.message_types_by_name['Empty'] = _EMPTY -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - +_JOB = DESCRIPTOR.message_types_by_name['Job'] +_ARGUMENT = DESCRIPTOR.message_types_by_name['Argument'] +_MANUALINTERACTION = DESCRIPTOR.message_types_by_name['ManualInteraction'] +_JOBRESULT = DESCRIPTOR.message_types_by_name['JobResult'] +_EMPTY = DESCRIPTOR.message_types_by_name['Empty'] Job = _reflection.GeneratedProtocolMessageType('Job', (_message.Message,), { 'DESCRIPTOR' : _JOB, '__module__' : 'gaiasdk.plugin_pb2' @@ -308,39 +58,21 @@ }) _sym_db.RegisterMessage(Empty) - -DESCRIPTOR._options = None - -_PLUGIN = _descriptor.ServiceDescriptor( - name='Plugin', - full_name='proto.Plugin', - file=DESCRIPTOR, - index=0, - serialized_options=None, - serialized_start=434, - serialized_end=525, - methods=[ - _descriptor.MethodDescriptor( - name='GetJobs', - full_name='proto.Plugin.GetJobs', - index=0, - containing_service=None, - input_type=_EMPTY, - output_type=_JOB, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='ExecuteJob', - full_name='proto.Plugin.ExecuteJob', - index=1, - containing_service=None, - input_type=_JOB, - output_type=_JOBRESULT, - serialized_options=None, - ), -]) -_sym_db.RegisterServiceDescriptor(_PLUGIN) - -DESCRIPTOR.services_by_name['Plugin'] = _PLUGIN - +_PLUGIN = DESCRIPTOR.services_by_name['Plugin'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\025io.gaiapipeline.protoB\nGRPCPluginP\001' + _JOB._serialized_start=32 + _JOB._serialized_end=189 + _ARGUMENT._serialized_start=191 + _ARGUMENT._serialized_end=264 + _MANUALINTERACTION._serialized_start=266 + _MANUALINTERACTION._serialized_end=335 + _JOBRESULT._serialized_start=337 + _JOBRESULT._serialized_end=423 + _EMPTY._serialized_start=425 + _EMPTY._serialized_end=432 + _PLUGIN._serialized_start=434 + _PLUGIN._serialized_end=525 # @@protoc_insertion_point(module_scope) diff --git a/gaiasdk/plugin_pb2_grpc.py b/gaiasdk/plugin_pb2_grpc.py index 4112101..32889e6 100644 --- a/gaiasdk/plugin_pb2_grpc.py +++ b/gaiasdk/plugin_pb2_grpc.py @@ -1,65 +1,103 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" import grpc from gaiasdk import plugin_pb2 as gaiasdk_dot_plugin__pb2 class PluginStub(object): - # missing associated documentation comment in .proto file - pass + """Missing associated documentation comment in .proto file.""" - def __init__(self, channel): - """Constructor. + def __init__(self, channel): + """Constructor. - Args: - channel: A grpc.Channel. - """ - self.GetJobs = channel.unary_stream( - '/proto.Plugin/GetJobs', - request_serializer=gaiasdk_dot_plugin__pb2.Empty.SerializeToString, - response_deserializer=gaiasdk_dot_plugin__pb2.Job.FromString, - ) - self.ExecuteJob = channel.unary_unary( - '/proto.Plugin/ExecuteJob', - request_serializer=gaiasdk_dot_plugin__pb2.Job.SerializeToString, - response_deserializer=gaiasdk_dot_plugin__pb2.JobResult.FromString, - ) + Args: + channel: A grpc.Channel. + """ + self.GetJobs = channel.unary_stream( + '/proto.Plugin/GetJobs', + request_serializer=gaiasdk_dot_plugin__pb2.Empty.SerializeToString, + response_deserializer=gaiasdk_dot_plugin__pb2.Job.FromString, + ) + self.ExecuteJob = channel.unary_unary( + '/proto.Plugin/ExecuteJob', + request_serializer=gaiasdk_dot_plugin__pb2.Job.SerializeToString, + response_deserializer=gaiasdk_dot_plugin__pb2.JobResult.FromString, + ) class PluginServicer(object): - # missing associated documentation comment in .proto file - pass + """Missing associated documentation comment in .proto file.""" - def GetJobs(self, request, context): - """GetJobs returns a stream of Job objects. - Used to expose jobs to gaia. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') + def GetJobs(self, request, context): + """GetJobs returns a stream of Job objects. + Used to expose jobs to gaia. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') - def ExecuteJob(self, request, context): - """ExecuteJob signals the plugin to execute the given job. - Used to execute one job from a pipeline. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') + def ExecuteJob(self, request, context): + """ExecuteJob signals the plugin to execute the given job. + Used to execute one job from a pipeline. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') def add_PluginServicer_to_server(servicer, server): - rpc_method_handlers = { - 'GetJobs': grpc.unary_stream_rpc_method_handler( - servicer.GetJobs, - request_deserializer=gaiasdk_dot_plugin__pb2.Empty.FromString, - response_serializer=gaiasdk_dot_plugin__pb2.Job.SerializeToString, - ), - 'ExecuteJob': grpc.unary_unary_rpc_method_handler( - servicer.ExecuteJob, - request_deserializer=gaiasdk_dot_plugin__pb2.Job.FromString, - response_serializer=gaiasdk_dot_plugin__pb2.JobResult.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'proto.Plugin', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) + rpc_method_handlers = { + 'GetJobs': grpc.unary_stream_rpc_method_handler( + servicer.GetJobs, + request_deserializer=gaiasdk_dot_plugin__pb2.Empty.FromString, + response_serializer=gaiasdk_dot_plugin__pb2.Job.SerializeToString, + ), + 'ExecuteJob': grpc.unary_unary_rpc_method_handler( + servicer.ExecuteJob, + request_deserializer=gaiasdk_dot_plugin__pb2.Job.FromString, + response_serializer=gaiasdk_dot_plugin__pb2.JobResult.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'proto.Plugin', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class Plugin(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def GetJobs(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/proto.Plugin/GetJobs', + gaiasdk_dot_plugin__pb2.Empty.SerializeToString, + gaiasdk_dot_plugin__pb2.Job.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ExecuteJob(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/proto.Plugin/ExecuteJob', + gaiasdk_dot_plugin__pb2.Job.SerializeToString, + gaiasdk_dot_plugin__pb2.JobResult.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/setup.py b/setup.py index fa5f939..6bad6d7 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import setuptools setuptools.setup(name='gaiasdk', - version='0.0.18', + version='0.0.19', description='Gaia Python SDK for python pipelines', url='https://github.com/gaia-pipeline/pythonsdk', author='Michel Vocks',