diff --git a/ai/src/data_manager/time_series_manager.py b/ai/src/data_manager/time_series_manager.py index 159074a9f5..55df11155f 100644 --- a/ai/src/data_manager/time_series_manager.py +++ b/ai/src/data_manager/time_series_manager.py @@ -5,6 +5,7 @@ import pandas as pd from data_manager.base_manager import DataManagerBase, DataParam +from exception import AiEngineException from proto.aiengine.v1 import aiengine_pb2 @@ -29,7 +30,7 @@ def get_window_span(self): def start_training(self): if self.is_training: - raise Exception("unable to start a new training run before the previous has finished") + raise AiEngineException("unable to start a new training run before the previous has finished") self.is_training = True self.metrics.start("copy_training_table") self.massive_table_training_filled = self._fill_table(self.massive_table_sparse) @@ -63,7 +64,7 @@ def _fill_table(self, input_table: pd.DataFrame) -> pd.DataFrame: def merge_training_row(self, new_row: pd.DataFrame): if not self.is_training: - raise Exception("only valid to call merge_training_row during a training run") + raise AiEngineException("only valid to call merge_training_row during a training run") index = new_row.index[0] for column_name in list(new_row.keys()): value = new_row[column_name].array[0] @@ -108,7 +109,7 @@ def get_shape(self): # This method should only be called during training. def get_current_window(self) -> pd.DataFrame: if not self.is_training: - raise Exception("Start training before calling get_current_window()") + raise AiEngineException("Start training before calling get_current_window()") # This will get the nearest previous index that matches the timestamp, # so we don't need to specify the timestamps exactly diff --git a/ai/src/tests/test_stateful.py b/ai/src/tests/test_stateful.py index 76f0da7bc3..dca3c98bb9 100644 --- a/ai/src/tests/test_stateful.py +++ b/ai/src/tests/test_stateful.py @@ -8,6 +8,7 @@ from data_manager.base_manager import DataParam from data_manager.time_series_manager import TimeSeriesDataManager from proto.aiengine.v1 import aiengine_pb2 +from exception import AiEngineException class StatefulConnectorTests(unittest.TestCase): @@ -104,7 +105,7 @@ def test_is_calling_merge_row(self): original_fill_table = self.data_manager._fill_table # pylint: disable=protected-access def new_fill_table(): - raise Exception("Should not call this on apply_action") + raise AiEngineException("Should not call this on apply_action") try: self.data_manager._fill_table = new_fill_table # pylint: disable=protected-access diff --git a/go.mod b/go.mod index fcc146234b..d753c48819 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/grpc v1.47.0 - google.golang.org/protobuf v1.28.0 + google.golang.org/protobuf v1.28.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index e49ad409c8..c1e3209912 100644 --- a/go.sum +++ b/go.sum @@ -1224,8 +1224,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/proto/aiengine_pb/aiengine.pb.go b/pkg/proto/aiengine_pb/aiengine.pb.go index bce58d2863..21e3923340 100644 --- a/pkg/proto/aiengine_pb/aiengine.pb.go +++ b/pkg/proto/aiengine_pb/aiengine.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.0 +// protoc-gen-go v1.28.1 // protoc v3.19.4 // source: proto/aiengine/v1/aiengine.proto diff --git a/pkg/proto/aiengine_pb/aiengine_grpc.pb.go b/pkg/proto/aiengine_pb/aiengine_grpc.pb.go index 1192f11d4d..59e01e274d 100644 --- a/pkg/proto/aiengine_pb/aiengine_grpc.pb.go +++ b/pkg/proto/aiengine_pb/aiengine_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.3.0 // - protoc v3.19.4 // source: proto/aiengine/v1/aiengine.proto @@ -18,6 +18,17 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + AIEngine_Init_FullMethodName = "/aiengine.AIEngine/Init" + AIEngine_AddData_FullMethodName = "/aiengine.AIEngine/AddData" + AIEngine_AddInterpretations_FullMethodName = "/aiengine.AIEngine/AddInterpretations" + AIEngine_StartTraining_FullMethodName = "/aiengine.AIEngine/StartTraining" + AIEngine_GetInference_FullMethodName = "/aiengine.AIEngine/GetInference" + AIEngine_GetHealth_FullMethodName = "/aiengine.AIEngine/GetHealth" + AIEngine_ExportModel_FullMethodName = "/aiengine.AIEngine/ExportModel" + AIEngine_ImportModel_FullMethodName = "/aiengine.AIEngine/ImportModel" +) + // AIEngineClient is the client API for AIEngine service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -42,7 +53,7 @@ func NewAIEngineClient(cc grpc.ClientConnInterface) AIEngineClient { func (c *aIEngineClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*Response, error) { out := new(Response) - err := c.cc.Invoke(ctx, "/aiengine.AIEngine/Init", in, out, opts...) + err := c.cc.Invoke(ctx, AIEngine_Init_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -51,7 +62,7 @@ func (c *aIEngineClient) Init(ctx context.Context, in *InitRequest, opts ...grpc func (c *aIEngineClient) AddData(ctx context.Context, in *AddDataRequest, opts ...grpc.CallOption) (*Response, error) { out := new(Response) - err := c.cc.Invoke(ctx, "/aiengine.AIEngine/AddData", in, out, opts...) + err := c.cc.Invoke(ctx, AIEngine_AddData_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -60,7 +71,7 @@ func (c *aIEngineClient) AddData(ctx context.Context, in *AddDataRequest, opts . func (c *aIEngineClient) AddInterpretations(ctx context.Context, in *AddInterpretationsRequest, opts ...grpc.CallOption) (*Response, error) { out := new(Response) - err := c.cc.Invoke(ctx, "/aiengine.AIEngine/AddInterpretations", in, out, opts...) + err := c.cc.Invoke(ctx, AIEngine_AddInterpretations_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -69,7 +80,7 @@ func (c *aIEngineClient) AddInterpretations(ctx context.Context, in *AddInterpre func (c *aIEngineClient) StartTraining(ctx context.Context, in *StartTrainingRequest, opts ...grpc.CallOption) (*Response, error) { out := new(Response) - err := c.cc.Invoke(ctx, "/aiengine.AIEngine/StartTraining", in, out, opts...) + err := c.cc.Invoke(ctx, AIEngine_StartTraining_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -78,7 +89,7 @@ func (c *aIEngineClient) StartTraining(ctx context.Context, in *StartTrainingReq func (c *aIEngineClient) GetInference(ctx context.Context, in *InferenceRequest, opts ...grpc.CallOption) (*InferenceResult, error) { out := new(InferenceResult) - err := c.cc.Invoke(ctx, "/aiengine.AIEngine/GetInference", in, out, opts...) + err := c.cc.Invoke(ctx, AIEngine_GetInference_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -87,7 +98,7 @@ func (c *aIEngineClient) GetInference(ctx context.Context, in *InferenceRequest, func (c *aIEngineClient) GetHealth(ctx context.Context, in *HealthRequest, opts ...grpc.CallOption) (*Response, error) { out := new(Response) - err := c.cc.Invoke(ctx, "/aiengine.AIEngine/GetHealth", in, out, opts...) + err := c.cc.Invoke(ctx, AIEngine_GetHealth_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -96,7 +107,7 @@ func (c *aIEngineClient) GetHealth(ctx context.Context, in *HealthRequest, opts func (c *aIEngineClient) ExportModel(ctx context.Context, in *ExportModelRequest, opts ...grpc.CallOption) (*ExportModelResult, error) { out := new(ExportModelResult) - err := c.cc.Invoke(ctx, "/aiengine.AIEngine/ExportModel", in, out, opts...) + err := c.cc.Invoke(ctx, AIEngine_ExportModel_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -105,7 +116,7 @@ func (c *aIEngineClient) ExportModel(ctx context.Context, in *ExportModelRequest func (c *aIEngineClient) ImportModel(ctx context.Context, in *ImportModelRequest, opts ...grpc.CallOption) (*Response, error) { out := new(Response) - err := c.cc.Invoke(ctx, "/aiengine.AIEngine/ImportModel", in, out, opts...) + err := c.cc.Invoke(ctx, AIEngine_ImportModel_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -176,7 +187,7 @@ func _AIEngine_Init_Handler(srv interface{}, ctx context.Context, dec func(inter } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/aiengine.AIEngine/Init", + FullMethod: AIEngine_Init_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AIEngineServer).Init(ctx, req.(*InitRequest)) @@ -194,7 +205,7 @@ func _AIEngine_AddData_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/aiengine.AIEngine/AddData", + FullMethod: AIEngine_AddData_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AIEngineServer).AddData(ctx, req.(*AddDataRequest)) @@ -212,7 +223,7 @@ func _AIEngine_AddInterpretations_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/aiengine.AIEngine/AddInterpretations", + FullMethod: AIEngine_AddInterpretations_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AIEngineServer).AddInterpretations(ctx, req.(*AddInterpretationsRequest)) @@ -230,7 +241,7 @@ func _AIEngine_StartTraining_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/aiengine.AIEngine/StartTraining", + FullMethod: AIEngine_StartTraining_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AIEngineServer).StartTraining(ctx, req.(*StartTrainingRequest)) @@ -248,7 +259,7 @@ func _AIEngine_GetInference_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/aiengine.AIEngine/GetInference", + FullMethod: AIEngine_GetInference_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AIEngineServer).GetInference(ctx, req.(*InferenceRequest)) @@ -266,7 +277,7 @@ func _AIEngine_GetHealth_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/aiengine.AIEngine/GetHealth", + FullMethod: AIEngine_GetHealth_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AIEngineServer).GetHealth(ctx, req.(*HealthRequest)) @@ -284,7 +295,7 @@ func _AIEngine_ExportModel_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/aiengine.AIEngine/ExportModel", + FullMethod: AIEngine_ExportModel_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AIEngineServer).ExportModel(ctx, req.(*ExportModelRequest)) @@ -302,7 +313,7 @@ func _AIEngine_ImportModel_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/aiengine.AIEngine/ImportModel", + FullMethod: AIEngine_ImportModel_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AIEngineServer).ImportModel(ctx, req.(*ImportModelRequest)) diff --git a/pkg/proto/common_pb/common.pb.go b/pkg/proto/common_pb/common.pb.go index 9ab73645aa..77a564f6dd 100644 --- a/pkg/proto/common_pb/common.pb.go +++ b/pkg/proto/common_pb/common.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.0 +// protoc-gen-go v1.28.1 // protoc v3.19.4 // source: proto/common/v1/common.proto diff --git a/pkg/proto/runtime_pb/runtime.pb.go b/pkg/proto/runtime_pb/runtime.pb.go index 7320c6d9b1..3e21bc12d6 100644 --- a/pkg/proto/runtime_pb/runtime.pb.go +++ b/pkg/proto/runtime_pb/runtime.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.0 +// protoc-gen-go v1.28.1 // protoc v3.19.4 // source: proto/runtime/v1/runtime.proto diff --git a/pkg/spec/dataspace.go b/pkg/spec/dataspace.go index f82930789e..2f30a9464d 100644 --- a/pkg/spec/dataspace.go +++ b/pkg/spec/dataspace.go @@ -5,6 +5,7 @@ type DataspaceSpec struct { Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"` SeedData *DataSpec `json:"seed_data,omitempty" yaml:"seed_data,omitempty" mapstructure:"seed_data,omitempty"` Data *DataSpec `json:"data,omitempty" yaml:"data,omitempty" mapstructure:"data,omitempty"` + Fields []FieldsSpec `json:"fields,omitempty" yaml:"fields,omitempty" mapstructure:"fields,omitempty"` Identifiers []IdentifiersSpec `json:"identifiers,omitempty" yaml:"identifiers,omitempty" mapstructure:"identifiers,omitempty"` Measurements []MeasurementSpec `json:"measurements,omitempty" yaml:"measurements,omitempty" mapstructure:"measurements,omitempty"` Categories []CategorySpec `json:"categories,omitempty" yaml:"categories,omitempty" mapstructure:"categories,omitempty"` @@ -18,6 +19,11 @@ type DataSpec struct { Processor DataProcessorSpec `json:"processor,omitempty" yaml:"processor,omitempty" mapstructure:"processor,omitempty"` } +type FieldsSpec struct { + Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"` + Selector string `json:"selector,omitempty" yaml:"selector,omitempty" mapstructure:"selector,omitempty"` +} + type IdentifiersSpec struct { Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"` Selector string `json:"selector,omitempty" yaml:"selector,omitempty" mapstructure:"selector,omitempty"` diff --git a/pkg/spec/pods.go b/pkg/spec/pods.go index 0e3babb5d1..21ba9e5c9d 100644 --- a/pkg/spec/pods.go +++ b/pkg/spec/pods.go @@ -1,13 +1,13 @@ package spec type PodSpec struct { - Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"` - Params map[string]string `json:"params,omitempty" yaml:"params,omitempty" mapstructure:"params,omitempty"` - Time *TimeSpec `json:"time,omitempty" yaml:"time,omitempty" mapstructure:"time,omitempty"` - Dataspaces []DataspaceSpec `json:"dataspaces,omitempty" yaml:"dataspaces,omitempty" mapstructure:"dataspaces,omitempty"` - Actions []PodActionSpec `json:"actions,omitempty" yaml:"actions,omitempty" mapstructure:"actions,omitempty"` - Training *TrainingSpec `json:"training,omitempty" yaml:"training,omitempty" mapstructure:"training,omitempty"` - Monitors *MonitorSpec `json:"monitors,omitempty" yaml:"monitors,omitempty" mapstructure:"monitors,omitempty"` + Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"` + Params map[string]string `json:"params,omitempty" yaml:"params,omitempty" mapstructure:"params,omitempty"` + Time *TimeSpec `json:"time,omitempty" yaml:"time,omitempty" mapstructure:"time,omitempty"` + Dataspaces []DataspaceSpec `json:"dataspaces,omitempty" yaml:"dataspaces,omitempty" mapstructure:"dataspaces,omitempty"` + Actions []PodActionSpec `json:"actions,omitempty" yaml:"actions,omitempty" mapstructure:"actions,omitempty"` + Training *TrainingSpec `json:"training,omitempty" yaml:"training,omitempty" mapstructure:"training,omitempty"` + Monitors map[string]*MonitorSpec `json:"monitors,omitempty" yaml:"monitors,omitempty" mapstructure:"monitors,omitempty"` } type TimeSpec struct { @@ -39,22 +39,15 @@ type RewardSpec struct { } type MonitorSpec struct { - Triggers []TriggerSpec `json:"triggers,omitempty" yaml:"triggers,omitempty" mapstructure:"triggers,omitempty"` - Alerts []AlertSpec `json:"alerts,omitempty" yaml:"alerts,omitempty" mapstructure:"alerts,omitempty"` + Triggers []TriggerSpec `json:"triggers,omitempty" yaml:"triggers,omitempty" mapstructure:"triggers,omitempty"` + Notifications []NotificationSpec `json:"notifications,omitempty" yaml:"notifications,omitempty" mapstructure:"notifications,omitempty"` } type TriggerSpec struct { - Dataspace string `json:"dataspace,omitempty" yaml:"dataspace,omitempty" mapstructure:"dataspace,omitempty"` - Measurement string `json:"measurement,omitempty" yaml:"measurement,omitempty" mapstructure:"measurement,omitempty"` - Thresholds []ThresholdSpec `json:"thresholds,omitempty" yaml:"thresholds,omitempty" mapstructure:"thresholds,omitempty"` + Dataspace string `json:"dataspace,omitempty" yaml:"dataspace,omitempty" mapstructure:"dataspace,omitempty"` } -type ThresholdSpec struct { - Operator string `json:"operator,omitempty" yaml:"operator,omitempty" mapstructure:"operator,omitempty"` - Value string `json:"value,omitempty" yaml:"value,omitempty" mapstructure:"value,omitempty"` -} - -type AlertSpec struct { +type NotificationSpec struct { Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"` Type string `json:"type,omitempty" yaml:"type,omitempty" mapstructure:"type,omitempty"` Endpoint string `json:"endpoint,omitempty" yaml:"endpoint,omitempty" mapstructure:"endpoint,omitempty"`