Skip to content

Commit

Permalink
WIP settings system.
Browse files Browse the repository at this point in the history
- updated dbdiagrams schema
- [BREAKING] force failure if `notify.filter_attributes` or `notify.level` is set
- added Settings table (and default values during migration)
- Added Save Settings and Get Settings functions.
- Added web API endpoints for getting and saving settings.
- Deprecated old Notify* constants. Created new MetricsStatus* and MetricsNotifyLevel constants.
  • Loading branch information
AnalogJ committed Jul 17, 2022
1 parent dd0c3e6 commit 99af2b8
Show file tree
Hide file tree
Showing 14 changed files with 316 additions and 109 deletions.
File renamed without changes.
122 changes: 74 additions & 48 deletions docs/dbdiagram.io.txt
Original file line number Diff line number Diff line change
@@ -1,62 +1,88 @@

// SQLite Table(s)
Table device {
created_at timestamp

wwn varchar [pk]

//user provided
label varchar
host_id varchar

// smartctl provided
device_name varchar
manufacturer varchar
model_name varchar
interface_type varchar
interface_speed varchar
serial_number varchar
firmware varchar
rotational_speed varchar
capacity varchar
form_factor varchar
smart_support varchar
device_protocol varchar
device_type varchar

Table Device {
//GORM attributes, see: http://gorm.io/docs/conventions.html
CreatedAt time
UpdatedAt time
DeletedAt time

WWN string

DeviceName string
DeviceUUID string
DeviceSerialID string
DeviceLabel string

Manufacturer string
ModelName string
InterfaceType string
InterfaceSpeed string
SerialNumber string
Firmware string
RotationSpeed int
Capacity int64
FormFactor string
SmartSupport bool
DeviceProtocol string//protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
DeviceType string//device type is used for querying with -d/t flag, should only be used by collector.

// User provided metadata
Label string
HostId string

// Data set by Scrutiny
DeviceStatus enum
}

Table Setting {
//GORM attributes, see: http://gorm.io/docs/conventions.html

// InfluxDB Tables
Table device_temperature {
//timestamp
created_at timestamp

//tags (indexed & queryable)
device_wwn varchar [pk]

//fields
temp bigint
}

SettingKeyName string
SettingKeyDescription string
SettingDataType string

Table smart_ata_results {
//timestamp
created_at timestamp

//tags (indexed & queryable)
device_wwn varchar [pk]
smart_status varchar
scrutiny_status varchar
SettingValueNumeric int64
SettingValueString string
}


// InfluxDB Tables
Table SmartTemperature {
Date time
DeviceWWN string //(tag)
Temp int64
}

//fields
temp bigint
power_on_hours bigint
power_cycle_count bigint

Table Smart {
Date time
DeviceWWN string //(tag)
DeviceProtocol string

//Metrics (fields)
Temp int64
PowerOnHours int64
PowerCycleCount int64

//Smart Status
Status enum

//SMART Attributes (fields)
Attr_ID_AttributeId int
Attr_ID_Value int64
Attr_ID_Threshold int64
Attr_ID_Worst int64
Attr_ID_RawValue int64
Attr_ID_RawString string
Attr_ID_WhenFailed string
//Generated data
Attr_ID_TransformedValue int64
Attr_ID_Status enum
Attr_ID_StatusReason string
Attr_ID_FailureRate float64

}

Ref: device.wwn < smart_ata_results.device_wwn
Ref: Device.WWN < Smart.DeviceWWN
Ref: Device.WWN < SmartTemperature.DeviceWWN
46 changes: 13 additions & 33 deletions webapp/backend/pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package config

import (
"github.com/analogj/go-util/utils"
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
"github.com/spf13/viper"
"log"
Expand Down Expand Up @@ -39,8 +38,6 @@ func (c *configuration) Init() error {
c.SetDefault("log.file", "")

c.SetDefault("notify.urls", []string{})
c.SetDefault("notify.filter_attributes", pkg.NotifyFilterAttributesAll)
c.SetDefault("notify.level", pkg.NotifyLevelFail)

c.SetDefault("web.influxdb.scheme", "http")
c.SetDefault("web.influxdb.host", "localhost")
Expand All @@ -55,17 +52,6 @@ func (c *configuration) Init() error {
//c.SetDefault("disks.include", []string{})
//c.SetDefault("disks.exclude", []string{})

//c.SetDefault("notify.metric.script", "/opt/scrutiny/config/notify-metrics.sh")
//c.SetDefault("notify.long.script", "/opt/scrutiny/config/notify-long-test.sh")
//c.SetDefault("notify.short.script", "/opt/scrutiny/config/notify-short-test.sh")

//c.SetDefault("collect.metric.enable", true)
//c.SetDefault("collect.metric.command", "-a -o on -S on")
//c.SetDefault("collect.long.enable", true)
//c.SetDefault("collect.long.command", "-a -o on -S on")
//c.SetDefault("collect.short.enable", true)
//c.SetDefault("collect.short.command", "-a -o on -S on")

//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
c.SetConfigType("yaml")
//c.SetConfigName("drawbridge")
Expand All @@ -77,7 +63,7 @@ func (c *configuration) Init() error {
c.AutomaticEnv()

//CLI options will be added via the `Set()` function
return nil
return c.ValidateConfig()
}

func (c *configuration) ReadConfig(configFilePath string) error {
Expand Down Expand Up @@ -120,24 +106,18 @@ func (c *configuration) ReadConfig(configFilePath string) error {
// This function ensures that the merged config works correctly.
func (c *configuration) ValidateConfig() error {

////deserialize Questions
//questionsMap := map[string]Question{}
//err := c.UnmarshalKey("questions", &questionsMap)
//
//if err != nil {
// log.Printf("questions could not be deserialized correctly. %v", err)
// return err
//}
//
//for _, v := range questionsMap {
//
// typeContent, ok := v.Schema["type"].(string)
// if !ok || len(typeContent) == 0 {
// return errors.QuestionSyntaxError("`type` is required for questions")
// }
//}
//
//
//the following keys are deprecated, and no longer supported
/*
- notify.filter_attributes (replaced by metrics.status.filter_attributes SETTING)
- notify.level (replaced by metrics.notify.level and metrics.status.threshold SETTING)
*/
//TODO add docs and upgrade doc.
if c.IsSet("notify.filter_attributes") {
return errors.ConfigValidationError("`notify.filter_attributes` configuration option is deprecated. Replaced by option in Dashboard Settings page")
}
if c.IsSet("notify.level") {
return errors.ConfigValidationError("`notify.level` configuration option is deprecated. Replaced by option in Dashboard Settings page")
}

return nil
}
54 changes: 45 additions & 9 deletions webapp/backend/pkg/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,11 @@ const DeviceProtocolAta = "ATA"
const DeviceProtocolScsi = "SCSI"
const DeviceProtocolNvme = "NVMe"

const NotifyFilterAttributesAll = "all"
const NotifyFilterAttributesCritical = "critical"

const NotifyLevelFail = "fail"
const NotifyLevelFailScrutiny = "fail_scrutiny"
const NotifyLevelFailSmart = "fail_smart"

//go:generate stringer -type=AttributeStatus
// AttributeStatus bitwise flag, 1,2,4,8,16,32,etc
type AttributeStatus uint8

const (
// AttributeStatusPassed binary, 1,2,4,8,16,32,etc
AttributeStatusPassed AttributeStatus = 0
AttributeStatusFailedSmart AttributeStatus = 1
AttributeStatusWarningScrutiny AttributeStatus = 2
Expand All @@ -30,9 +24,10 @@ func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^
func AttributeStatusHas(b, flag AttributeStatus) bool { return b&flag != 0 }

//go:generate stringer -type=DeviceStatus
// DeviceStatus bitwise flag, 1,2,4,8,16,32,etc
type DeviceStatus uint8

const (
// DeviceStatusPassed binary, 1,2,4,8,16,32,etc
DeviceStatusPassed DeviceStatus = 0
DeviceStatusFailedSmart DeviceStatus = 1
DeviceStatusFailedScrutiny DeviceStatus = 2
Expand All @@ -42,3 +37,44 @@ func DeviceStatusSet(b, flag DeviceStatus) DeviceStatus { return b | flag }
func DeviceStatusClear(b, flag DeviceStatus) DeviceStatus { return b &^ flag }
func DeviceStatusToggle(b, flag DeviceStatus) DeviceStatus { return b ^ flag }
func DeviceStatusHas(b, flag DeviceStatus) bool { return b&flag != 0 }

// Metrics Specific Filtering & Threshold Constants
type MetricsNotifyLevel int64

const (
MetricsNotifyLevelWarn MetricsNotifyLevel = 1
MetricsNotifyLevelFail MetricsNotifyLevel = 2
)

type MetricsStatusFilterAttributes int64

const (
MetricsStatusFilterAttributesAll MetricsStatusFilterAttributes = 0
MetricsStatusFilterAttributesCritical MetricsStatusFilterAttributes = 1
)

// MetricsStatusThreshold bitwise flag, 1,2,4,8,16,32,etc
type MetricsStatusThreshold int64

const (
MetricsStatusThresholdSmart MetricsStatusThreshold = 1
MetricsStatusThresholdScrutiny MetricsStatusThreshold = 2

//shortcut
MetricsStatusThresholdBoth MetricsStatusThreshold = 3
)

// Deprecated
const NotifyFilterAttributesAll = "all"

// Deprecated
const NotifyFilterAttributesCritical = "critical"

// Deprecated
const NotifyLevelFail = "fail"

// Deprecated
const NotifyLevelFailScrutiny = "fail_scrutiny"

// Deprecated
const NotifyLevelFailSmart = "fail_smart"
3 changes: 3 additions & 0 deletions webapp/backend/pkg/database/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,7 @@ type DeviceRepo interface {

GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)

GetSettings(ctx context.Context) (*models.Settings, error)
SaveSettings(ctx context.Context, settings models.Settings) error
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ type Setting struct {
//GORM attributes, see: http://gorm.io/docs/conventions.html
gorm.Model

SettingKeyName string `json:"setting_key_name"`
SettingDataType string `json:"setting_data_type"`
SettingKeyName string `json:"setting_key_name"`
SettingKeyDescription string `json:"setting_key_description"`
SettingDataType string `json:"setting_data_type"`

SettingValueNumeric int64 `json:"setting_value_numeric"`
SettingValueString string `json:"setting_value_string"`
Expand Down
29 changes: 28 additions & 1 deletion webapp/backend/pkg/database/scrutiny_repository_migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100"
Expand Down Expand Up @@ -281,7 +282,33 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
Migrate: func(tx *gorm.DB) error {

// adding the settings table.
return tx.AutoMigrate(m20220716214900.Setting{})
err := tx.AutoMigrate(m20220716214900.Setting{})
if err != nil {
return err
}
//add defaults.

var defaultSettings = []m20220716214900.Setting{
{
SettingKeyName: "metrics.notify.level",
SettingKeyDescription: "Determines which device status will cause a notification (fail or warn)",
SettingDataType: "numeric",
SettingValueNumeric: int64(pkg.MetricsNotifyLevelFail), // options: 'fail' or 'warn'
},
{
SettingKeyName: "metrics.status.filter_attributes",
SettingKeyDescription: "Determines which attributes should impact device status",
SettingDataType: "numeric",
SettingValueNumeric: int64(pkg.MetricsStatusFilterAttributesAll), // options: 'all' or 'critical'
},
{
SettingKeyName: "metrics.status.threshold",
SettingKeyDescription: "Determines which threshold should impact device status",
SettingDataType: "string",
SettingValueNumeric: int64(pkg.MetricsStatusThresholdBoth), // options: 'scrutiny', 'smart', 'both'
},
}
return tx.Create(&defaultSettings).Error
},
},
})
Expand Down
33 changes: 33 additions & 0 deletions webapp/backend/pkg/database/scrutiny_repository_settings.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package database

import (
"context"
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
)

func (sr *scrutinyRepository) GetSettings(ctx context.Context) (*models.Settings, error) {
settingsEntries := []models.SettingEntry{}
if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
return nil, fmt.Errorf("Could not get settings from DB: %v", err)
}

settings := models.Settings{}
settings.PopulateFromSettingEntries(settingsEntries)

return &settings, nil
}
func (sr *scrutinyRepository) SaveSettings(ctx context.Context, settings models.Settings) error {

//get current settings
settingsEntries := []models.SettingEntry{}
if err := sr.gormClient.WithContext(ctx).Find(&settingsEntries).Error; err != nil {
return fmt.Errorf("Could not get settings from DB: %v", err)
}

// override with values from settings object
settingsEntries = settings.UpdateSettingEntries(settingsEntries)

// store in database.
return sr.gormClient.Updates(&settingsEntries).Error
}

0 comments on commit 99af2b8

Please sign in to comment.