Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Tweak monitors Spicepod schema (#580)
* Tweak monitors Spicepod schema

* Fix version

* changes

* proto fix

* lint

* fix proto

* proto fix
  • Loading branch information
phillipleblanc committed Mar 10, 2023
1 parent cfabf92 commit 693b1e1
Show file tree
Hide file tree
Showing 10 changed files with 57 additions and 44 deletions.
7 changes: 4 additions & 3 deletions ai/src/data_manager/time_series_manager.py
Expand Up @@ -5,6 +5,7 @@
import pandas as pd

from data_manager.base_manager import DataManagerBase, DataParam
from exception import AiEngineException
from proto.aiengine.v1 import aiengine_pb2


Expand All @@ -29,7 +30,7 @@ def get_window_span(self):

def start_training(self):
if self.is_training:
raise Exception("unable to start a new training run before the previous has finished")
raise AiEngineException("unable to start a new training run before the previous has finished")
self.is_training = True
self.metrics.start("copy_training_table")
self.massive_table_training_filled = self._fill_table(self.massive_table_sparse)
Expand Down Expand Up @@ -63,7 +64,7 @@ def _fill_table(self, input_table: pd.DataFrame) -> pd.DataFrame:

def merge_training_row(self, new_row: pd.DataFrame):
if not self.is_training:
raise Exception("only valid to call merge_training_row during a training run")
raise AiEngineException("only valid to call merge_training_row during a training run")
index = new_row.index[0]
for column_name in list(new_row.keys()):
value = new_row[column_name].array[0]
Expand Down Expand Up @@ -108,7 +109,7 @@ def get_shape(self):
# This method should only be called during training.
def get_current_window(self) -> pd.DataFrame:
if not self.is_training:
raise Exception("Start training before calling get_current_window()")
raise AiEngineException("Start training before calling get_current_window()")

# This will get the nearest previous index that matches the timestamp,
# so we don't need to specify the timestamps exactly
Expand Down
3 changes: 2 additions & 1 deletion ai/src/tests/test_stateful.py
Expand Up @@ -8,6 +8,7 @@
from data_manager.base_manager import DataParam
from data_manager.time_series_manager import TimeSeriesDataManager
from proto.aiengine.v1 import aiengine_pb2
from exception import AiEngineException


class StatefulConnectorTests(unittest.TestCase):
Expand Down Expand Up @@ -104,7 +105,7 @@ def test_is_calling_merge_row(self):
original_fill_table = self.data_manager._fill_table # pylint: disable=protected-access

def new_fill_table():
raise Exception("Should not call this on apply_action")
raise AiEngineException("Should not call this on apply_action")

try:
self.data_manager._fill_table = new_fill_table # pylint: disable=protected-access
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Expand Up @@ -26,7 +26,7 @@ require (
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/grpc v1.47.0
google.golang.org/protobuf v1.28.0
google.golang.org/protobuf v1.28.1
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v2 v2.4.0
)
Expand Down
3 changes: 2 additions & 1 deletion go.sum
Expand Up @@ -1224,8 +1224,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
2 changes: 1 addition & 1 deletion pkg/proto/aiengine_pb/aiengine.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

45 changes: 28 additions & 17 deletions pkg/proto/aiengine_pb/aiengine_grpc.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pkg/proto/common_pb/common.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pkg/proto/runtime_pb/runtime.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions pkg/spec/dataspace.go
Expand Up @@ -5,6 +5,7 @@ type DataspaceSpec struct {
Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"`
SeedData *DataSpec `json:"seed_data,omitempty" yaml:"seed_data,omitempty" mapstructure:"seed_data,omitempty"`
Data *DataSpec `json:"data,omitempty" yaml:"data,omitempty" mapstructure:"data,omitempty"`
Fields []FieldsSpec `json:"fields,omitempty" yaml:"fields,omitempty" mapstructure:"fields,omitempty"`
Identifiers []IdentifiersSpec `json:"identifiers,omitempty" yaml:"identifiers,omitempty" mapstructure:"identifiers,omitempty"`
Measurements []MeasurementSpec `json:"measurements,omitempty" yaml:"measurements,omitempty" mapstructure:"measurements,omitempty"`
Categories []CategorySpec `json:"categories,omitempty" yaml:"categories,omitempty" mapstructure:"categories,omitempty"`
Expand All @@ -18,6 +19,11 @@ type DataSpec struct {
Processor DataProcessorSpec `json:"processor,omitempty" yaml:"processor,omitempty" mapstructure:"processor,omitempty"`
}

type FieldsSpec struct {
Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"`
Selector string `json:"selector,omitempty" yaml:"selector,omitempty" mapstructure:"selector,omitempty"`
}

type IdentifiersSpec struct {
Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"`
Selector string `json:"selector,omitempty" yaml:"selector,omitempty" mapstructure:"selector,omitempty"`
Expand Down
29 changes: 11 additions & 18 deletions pkg/spec/pods.go
@@ -1,13 +1,13 @@
package spec

type PodSpec struct {
Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"`
Params map[string]string `json:"params,omitempty" yaml:"params,omitempty" mapstructure:"params,omitempty"`
Time *TimeSpec `json:"time,omitempty" yaml:"time,omitempty" mapstructure:"time,omitempty"`
Dataspaces []DataspaceSpec `json:"dataspaces,omitempty" yaml:"dataspaces,omitempty" mapstructure:"dataspaces,omitempty"`
Actions []PodActionSpec `json:"actions,omitempty" yaml:"actions,omitempty" mapstructure:"actions,omitempty"`
Training *TrainingSpec `json:"training,omitempty" yaml:"training,omitempty" mapstructure:"training,omitempty"`
Monitors *MonitorSpec `json:"monitors,omitempty" yaml:"monitors,omitempty" mapstructure:"monitors,omitempty"`
Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"`
Params map[string]string `json:"params,omitempty" yaml:"params,omitempty" mapstructure:"params,omitempty"`
Time *TimeSpec `json:"time,omitempty" yaml:"time,omitempty" mapstructure:"time,omitempty"`
Dataspaces []DataspaceSpec `json:"dataspaces,omitempty" yaml:"dataspaces,omitempty" mapstructure:"dataspaces,omitempty"`
Actions []PodActionSpec `json:"actions,omitempty" yaml:"actions,omitempty" mapstructure:"actions,omitempty"`
Training *TrainingSpec `json:"training,omitempty" yaml:"training,omitempty" mapstructure:"training,omitempty"`
Monitors map[string]*MonitorSpec `json:"monitors,omitempty" yaml:"monitors,omitempty" mapstructure:"monitors,omitempty"`
}

type TimeSpec struct {
Expand Down Expand Up @@ -39,22 +39,15 @@ type RewardSpec struct {
}

type MonitorSpec struct {
Triggers []TriggerSpec `json:"triggers,omitempty" yaml:"triggers,omitempty" mapstructure:"triggers,omitempty"`
Alerts []AlertSpec `json:"alerts,omitempty" yaml:"alerts,omitempty" mapstructure:"alerts,omitempty"`
Triggers []TriggerSpec `json:"triggers,omitempty" yaml:"triggers,omitempty" mapstructure:"triggers,omitempty"`
Notifications []NotificationSpec `json:"notifications,omitempty" yaml:"notifications,omitempty" mapstructure:"notifications,omitempty"`
}

type TriggerSpec struct {
Dataspace string `json:"dataspace,omitempty" yaml:"dataspace,omitempty" mapstructure:"dataspace,omitempty"`
Measurement string `json:"measurement,omitempty" yaml:"measurement,omitempty" mapstructure:"measurement,omitempty"`
Thresholds []ThresholdSpec `json:"thresholds,omitempty" yaml:"thresholds,omitempty" mapstructure:"thresholds,omitempty"`
Dataspace string `json:"dataspace,omitempty" yaml:"dataspace,omitempty" mapstructure:"dataspace,omitempty"`
}

type ThresholdSpec struct {
Operator string `json:"operator,omitempty" yaml:"operator,omitempty" mapstructure:"operator,omitempty"`
Value string `json:"value,omitempty" yaml:"value,omitempty" mapstructure:"value,omitempty"`
}

type AlertSpec struct {
type NotificationSpec struct {
Name string `json:"name,omitempty" yaml:"name,omitempty" mapstructure:"name,omitempty"`
Type string `json:"type,omitempty" yaml:"type,omitempty" mapstructure:"type,omitempty"`
Endpoint string `json:"endpoint,omitempty" yaml:"endpoint,omitempty" mapstructure:"endpoint,omitempty"`
Expand Down

0 comments on commit 693b1e1

Please sign in to comment.