diff --git a/api/autopilot.go b/api/autopilot.go index fdd6c4942..22598b28c 100644 --- a/api/autopilot.go +++ b/api/autopilot.go @@ -2,8 +2,6 @@ package api import ( "errors" - "fmt" - "strings" "go.sia.tech/core/types" "go.sia.tech/renterd/hostdb" @@ -136,65 +134,8 @@ type ( Usable bool `json:"usable"` UnusableReasons []string `json:"unusableReasons"` } - - HostGougingBreakdown struct { - ContractErr string `json:"contractErr"` - DownloadErr string `json:"downloadErr"` - GougingErr string `json:"gougingErr"` - PruneErr string `json:"pruneErr"` - UploadErr string `json:"uploadErr"` - } - - HostScoreBreakdown struct { - Age float64 `json:"age"` - Collateral float64 `json:"collateral"` - Interactions float64 `json:"interactions"` - StorageRemaining float64 `json:"storageRemaining"` - Uptime float64 `json:"uptime"` - Version float64 `json:"version"` - Prices float64 `json:"prices"` - } ) -func (sb HostScoreBreakdown) String() string { - return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices) -} - -func (hgb HostGougingBreakdown) Gouging() bool { - for _, err := range []string{ - hgb.ContractErr, - hgb.DownloadErr, - hgb.GougingErr, - hgb.PruneErr, - hgb.UploadErr, - } { - if err != "" { - return true - } - } - return false -} - -func (hgb HostGougingBreakdown) String() string { - var reasons []string - for _, errStr := range []string{ - hgb.ContractErr, - hgb.DownloadErr, - hgb.GougingErr, - hgb.PruneErr, - hgb.UploadErr, - } { - if errStr != "" { - reasons = append(reasons, errStr) - } - } - return strings.Join(reasons, ";") -} - -func (sb HostScoreBreakdown) Score() float64 { - return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices -} - func (c AutopilotConfig) Validate() error { if c.Hosts.MaxDowntimeHours > 99*365*24 { return ErrMaxDowntimeHoursTooHigh diff --git a/api/host.go b/api/host.go index 293403c0a..5221fb20c 100644 --- a/api/host.go +++ b/api/host.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net/url" + "strings" "go.sia.tech/core/types" "go.sia.tech/renterd/hostdb" @@ -42,6 +43,8 @@ type ( MinRecentScanFailures uint64 `json:"minRecentScanFailures"` } + // SearchHostsRequest is the request type for the /api/bus/search/hosts + // endpoint. SearchHostsRequest struct { Offset int `json:"offset"` Limit int `json:"limit"` @@ -109,3 +112,85 @@ func (opts HostsForScanningOptions) Apply(values url.Values) { values.Set("lastScan", TimeRFC3339(opts.MaxLastScan).String()) } } + +type ( + Host struct { + hostdb.Host + Blocked bool `json:"blocked"` + Checks map[string]HostCheck `json:"checks"` + } + + HostCheck struct { + Gouging HostGougingBreakdown `json:"gouging"` + Score HostScoreBreakdown `json:"score"` + Usability HostUsabilityBreakdown `json:"usability"` + } + + HostGougingBreakdown struct { + ContractErr string `json:"contractErr"` + DownloadErr string `json:"downloadErr"` + GougingErr string `json:"gougingErr"` + PruneErr string `json:"pruneErr"` + UploadErr string `json:"uploadErr"` + } + + HostScoreBreakdown struct { + Age float64 `json:"age"` + Collateral float64 `json:"collateral"` + Interactions float64 `json:"interactions"` + StorageRemaining float64 `json:"storageRemaining"` + Uptime float64 `json:"uptime"` + Version float64 `json:"version"` + Prices float64 `json:"prices"` + } + + HostUsabilityBreakdown struct { + Blocked bool `json:"blocked"` + Offline bool `json:"offline"` + LowScore bool `json:"lowScore"` + RedundantIP bool `json:"redundantIP"` + Gouging bool `json:"gouging"` + NotAcceptingContracts bool `json:"notAcceptingContracts"` + NotAnnounced bool `json:"notAnnounced"` + NotCompletingScan bool `json:"notCompletingScan"` + } +) + +func (sb HostScoreBreakdown) String() string { + return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices) +} + +func (hgb HostGougingBreakdown) Gouging() bool { + for _, err := range []string{ + hgb.ContractErr, + hgb.DownloadErr, + hgb.GougingErr, + hgb.PruneErr, + hgb.UploadErr, + } { + if err != "" { + return true + } + } + return false +} + +func (hgb HostGougingBreakdown) String() string { + var reasons []string + for _, errStr := range []string{ + hgb.ContractErr, + hgb.DownloadErr, + hgb.GougingErr, + hgb.PruneErr, + hgb.UploadErr, + } { + if errStr != "" { + reasons = append(reasons, errStr) + } + } + return strings.Join(reasons, ";") +} + +func (sb HostScoreBreakdown) Score() float64 { + return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices +} diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index c9617837b..57ec72967 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -53,10 +53,10 @@ type Bus interface { PrunableData(ctx context.Context) (prunableData api.ContractsPrunableDataResponse, err error) // hostdb - Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error) + Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) - SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error) + SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) // metrics RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error @@ -737,7 +737,7 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) { jc.Encode(hosts) } -func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (usables uint64) { +func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (usables uint64) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) for _, host := range hosts { usable, _ := isUsableHost(cfg, rs, gc, host, smallestValidScore, 0) @@ -751,7 +751,7 @@ func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types. // evaluateConfig evaluates the given configuration and if the gouging settings // are too strict for the number of contracts required by 'cfg', it will provide // a recommendation on how to loosen it. -func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (resp api.ConfigEvaluationResponse) { +func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (resp api.ConfigEvaluationResponse) { gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow) resp.Hosts = uint64(len(hosts)) @@ -866,7 +866,7 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu // optimiseGougingSetting tries to optimise one field of the gouging settings to // try and hit the target number of contracts. -func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []hostdb.HostInfo) bool { +func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []api.Host) bool { if cfg.Contracts.Amount == 0 { return true // nothing to do } diff --git a/autopilot/autopilot_test.go b/autopilot/autopilot_test.go index 9ebafe675..a21b55c7b 100644 --- a/autopilot/autopilot_test.go +++ b/autopilot/autopilot_test.go @@ -14,9 +14,10 @@ import ( func TestOptimiseGougingSetting(t *testing.T) { // create 10 hosts that should all be usable - var hosts []hostdb.HostInfo + var hosts []api.Host for i := 0; i < 10; i++ { - hosts = append(hosts, hostdb.HostInfo{ + hosts = append(hosts, api.Host{ + Host: hostdb.Host{ KnownSince: time.Unix(0, 0), PriceTable: hostdb.HostPriceTable{ @@ -42,6 +43,7 @@ func TestOptimiseGougingSetting(t *testing.T) { Scanned: true, }, Blocked: false, + Checks: nil, }) } diff --git a/autopilot/client.go b/autopilot/client.go index ba16754a5..01d0a1632 100644 --- a/autopilot/client.go +++ b/autopilot/client.go @@ -40,7 +40,7 @@ func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostHandlerResponse } // HostInfo returns information about all hosts. -func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode string, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) { +func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) { err = c.c.POST("/hosts", api.SearchHostsRequest{ Offset: offset, Limit: limit, diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 9bfc8055e..7b2ea9863 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -1297,7 +1297,7 @@ func (c *contractor) calculateMinScore(candidates []scoredHost, numContracts uin return minScore } -func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.HostInfo, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) { +func (c *contractor) candidateHosts(ctx context.Context, hosts []api.Host, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostResult, error) { start := time.Now() // fetch consensus state @@ -1311,7 +1311,7 @@ func (c *contractor) candidateHosts(ctx context.Context, hosts []hostdb.HostInfo gc := worker.NewGougingChecker(state.gs, cs, state.fee, state.cfg.Contracts.Period, state.cfg.Contracts.RenewWindow) // select unused hosts that passed a scan - var unused []hostdb.HostInfo + var unused []api.Host var excluded, notcompletedscan int for _, h := range hosts { // filter out used hosts @@ -1612,7 +1612,7 @@ func (c *contractor) tryPerformPruning(wp *workerPool) { }() } -func (c *contractor) hostForContract(ctx context.Context, fcid types.FileContractID) (host hostdb.HostInfo, metadata api.ContractMetadata, err error) { +func (c *contractor) hostForContract(ctx context.Context, fcid types.FileContractID) (host api.Host, metadata api.ContractMetadata, err error) { // fetch the contract metadata, err = c.ap.bus.Contract(ctx, fcid) if err != nil { diff --git a/autopilot/hostfilter.go b/autopilot/hostfilter.go index 8de37221a..f41a20c94 100644 --- a/autopilot/hostfilter.go +++ b/autopilot/hostfilter.go @@ -11,7 +11,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/worker" ) @@ -176,7 +175,7 @@ func (u *unusableHostResult) keysAndValues() []interface{} { // isUsableHost returns whether the given host is usable along with a list of // reasons why it was deemed unusable. -func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h hostdb.HostInfo, minScore float64, storedData uint64) (bool, unusableHostResult) { +func isUsableHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h api.Host, minScore float64, storedData uint64) (bool, unusableHostResult) { if rs.Validate() != nil { panic("invalid redundancy settings were supplied - developer error") } diff --git a/autopilot/hostinfo.go b/autopilot/hostinfo.go index e0cbecadc..d82062a80 100644 --- a/autopilot/hostinfo.go +++ b/autopilot/hostinfo.go @@ -6,7 +6,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/hostdb" "go.sia.tech/renterd/worker" ) @@ -67,7 +66,7 @@ func (c *contractor) HostInfo(ctx context.Context, hostKey types.PublicKey) (api }, nil } -func (c *contractor) hostInfoFromCache(ctx context.Context, host hostdb.HostInfo) (hi hostInfo, found bool) { +func (c *contractor) hostInfoFromCache(ctx context.Context, host api.Host) (hi hostInfo, found bool) { // grab host details from cache c.mu.Lock() hi, found = c.cachedHostInfo[host.PublicKey] diff --git a/autopilot/scanner.go b/autopilot/scanner.go index 2c3a3caea..d733c8d0c 100644 --- a/autopilot/scanner.go +++ b/autopilot/scanner.go @@ -31,7 +31,7 @@ type ( // a bit, we currently use inline interfaces to avoid having to update the // scanner tests with every interface change bus interface { - SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error) + SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) } diff --git a/autopilot/scanner_test.go b/autopilot/scanner_test.go index 1cdd096d2..860a855fe 100644 --- a/autopilot/scanner_test.go +++ b/autopilot/scanner_test.go @@ -19,7 +19,7 @@ type mockBus struct { reqs []string } -func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error) { +func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error) { b.reqs = append(b.reqs, fmt.Sprintf("%d-%d", opts.Offset, opts.Offset+opts.Limit)) start := opts.Offset @@ -32,11 +32,11 @@ func (b *mockBus) SearchHosts(ctx context.Context, opts api.SearchHostOptions) ( end = len(b.hosts) } - his := make([]hostdb.HostInfo, len(b.hosts[start:end])) + hosts := make([]api.Host, len(b.hosts[start:end])) for i, h := range b.hosts[start:end] { - his[i] = hostdb.HostInfo{Host: h} + hosts[i] = api.Host{Host: h} } - return his, nil + return hosts, nil } func (b *mockBus) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error) { diff --git a/bus/bus.go b/bus/bus.go index 7d33964be..d68e46309 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -91,13 +91,13 @@ type ( // A HostDB stores information about hosts. HostDB interface { - Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error) + Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]hostdb.HostAddress, error) RecordHostScans(ctx context.Context, scans []hostdb.HostScan) error RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error) ResetLostSectors(ctx context.Context, hk types.PublicKey) error - SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]hostdb.HostInfo, error) + SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) HostAllowlist(ctx context.Context) ([]types.PublicKey, error) HostBlocklist(ctx context.Context) ([]string, error) @@ -775,9 +775,9 @@ func (b *bus) searchHostsHandlerPOST(jc jape.Context) { return } - // TODO: on the next major release - // - set defaults in handler - // - validate request params and return 400 if invalid + // TODO: on the next major release: + // - properly default search params + // - properly validate and return 400 hosts, err := b.hdb.SearchHosts(jc.Request.Context(), req.FilterMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", req.Offset, req.Offset+req.Limit), err) != nil { return diff --git a/bus/client/hosts.go b/bus/client/hosts.go index 1ebf14e1f..8338d53f7 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -12,7 +12,7 @@ import ( ) // Host returns information about a particular host known to the server. -func (c *Client) Host(ctx context.Context, hostKey types.PublicKey) (h hostdb.HostInfo, err error) { +func (c *Client) Host(ctx context.Context, hostKey types.PublicKey) (h api.Host, err error) { err = c.c.WithContext(ctx).GET(fmt.Sprintf("/host/%s", hostKey), &h) return } @@ -30,7 +30,7 @@ func (c *Client) HostBlocklist(ctx context.Context) (blocklist []string, err err } // Hosts returns 'limit' hosts at given 'offset'. -func (c *Client) Hosts(ctx context.Context, opts api.GetHostsOptions) (hosts []hostdb.HostInfo, err error) { +func (c *Client) Hosts(ctx context.Context, opts api.GetHostsOptions) (hosts []api.Host, err error) { values := url.Values{} opts.Apply(values) err = c.c.WithContext(ctx).GET("/hosts?"+values.Encode(), &hosts) @@ -78,7 +78,7 @@ func (c *Client) ResetLostSectors(ctx context.Context, hostKey types.PublicKey) } // SearchHosts returns all hosts that match certain search criteria. -func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []hostdb.HostInfo, err error) { +func (c *Client) SearchHosts(ctx context.Context, opts api.SearchHostOptions) (hosts []api.Host, err error) { err = c.c.WithContext(ctx).POST("/search/hosts", api.SearchHostsRequest{ Offset: opts.Offset, Limit: opts.Limit, diff --git a/hostdb/hostdb.go b/hostdb/hostdb.go index 69ed80989..1f4c341de 100644 --- a/hostdb/hostdb.go +++ b/hostdb/hostdb.go @@ -114,12 +114,6 @@ type HostPriceTable struct { Expiry time.Time `json:"expiry"` } -// HostInfo extends the host type with a field indicating whether it is blocked or not. -type HostInfo struct { - Host - Blocked bool `json:"blocked"` -} - // IsAnnounced returns whether the host has been announced. func (h Host) IsAnnounced() bool { return !h.LastAnnouncement.IsZero() diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index e081c2a5d..77898d4cf 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -166,7 +166,7 @@ func TestNewTestCluster(t *testing.T) { if len(hi.Checks.UnusableReasons) != 0 { t.Fatal("usable hosts don't have any reasons set") } - if reflect.DeepEqual(hi.Host, hostdb.HostInfo{}) { + if reflect.DeepEqual(hi.Host, hostdb.Host{}) { t.Fatal("host wasn't set") } } @@ -188,7 +188,7 @@ func TestNewTestCluster(t *testing.T) { if len(hi.Checks.UnusableReasons) != 0 { t.Fatal("usable hosts don't have any reasons set") } - if reflect.DeepEqual(hi.Host, hostdb.HostInfo{}) { + if reflect.DeepEqual(hi.Host, hostdb.Host{}) { t.Fatal("host wasn't set") } allHosts[hi.Host.PublicKey] = struct{}{} diff --git a/stores/hostdb.go b/stores/hostdb.go index 95e37a26c..b91f85601 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -78,6 +78,45 @@ type ( Allowlist []dbAllowlistEntry `gorm:"many2many:host_allowlist_entry_hosts;constraint:OnDelete:CASCADE"` Blocklist []dbBlocklistEntry `gorm:"many2many:host_blocklist_entry_hosts;constraint:OnDelete:CASCADE"` + Checks []dbHostCheck `gorm:"foreignKey:DBHostID;constraint:OnDelete:CASCADE"` + } + + // dbHostCheck contains information about a host that is collected and used + // by the autopilot. + dbHostCheck struct { + Model + + DBAutopilotID uint + DBAutopilot dbAutopilot + + DBHostID uint + DBHost dbHost + + // usability + UsabilityBlocked bool + UsabilityOffline bool + UsabilityLowScore bool + UsabilityRedundantIP bool + UsabilityGouging bool + UsabilityNotAcceptingContracts bool + UsabilityNotAnnounced bool + UsabilityNotCompletingScan bool + + // score + ScoreAge float64 + ScoreCollateral float64 + ScoreInteractions float64 + ScoreStorageRemaining float64 + ScoreUptime float64 + ScoreVersion float64 + ScorePrices float64 + + // gouging + GougingContractErr string + GougingDownloadErr string + GougingGougingErr string + GougingPruneErr string + GougingUploadErr string } // dbAllowlistEntry defines a table that stores the host blocklist. @@ -263,40 +302,82 @@ func (dbConsensusInfo) TableName() string { return "consensus_infos" } // TableName implements the gorm.Tabler interface. func (dbHost) TableName() string { return "hosts" } +// TableName implements the gorm.Tabler interface. +func (dbHostCheck) TableName() string { return "host_checks" } + // TableName implements the gorm.Tabler interface. func (dbAllowlistEntry) TableName() string { return "host_allowlist_entries" } // TableName implements the gorm.Tabler interface. func (dbBlocklistEntry) TableName() string { return "host_blocklist_entries" } -// convert converts a host into a hostdb.Host. -func (h dbHost) convert() hostdb.Host { +// convert converts a host into a api.HostInfo +func (h dbHost) convert(blocked bool) api.Host { var lastScan time.Time if h.LastScan > 0 { lastScan = time.Unix(0, h.LastScan) } - return hostdb.Host{ - KnownSince: h.CreatedAt, - LastAnnouncement: h.LastAnnouncement, - NetAddress: h.NetAddress, - Interactions: hostdb.Interactions{ - TotalScans: h.TotalScans, - LastScan: lastScan, - LastScanSuccess: h.LastScanSuccess, - SecondToLastScanSuccess: h.SecondToLastScanSuccess, - Uptime: h.Uptime, - Downtime: h.Downtime, - SuccessfulInteractions: h.SuccessfulInteractions, - FailedInteractions: h.FailedInteractions, - LostSectors: h.LostSectors, + checks := make(map[string]api.HostCheck) + for _, check := range h.Checks { + checks[check.DBAutopilot.Identifier] = check.convert() + } + return api.Host{ + Host: hostdb.Host{ + KnownSince: h.CreatedAt, + LastAnnouncement: h.LastAnnouncement, + NetAddress: h.NetAddress, + Interactions: hostdb.Interactions{ + TotalScans: h.TotalScans, + LastScan: lastScan, + LastScanSuccess: h.LastScanSuccess, + SecondToLastScanSuccess: h.SecondToLastScanSuccess, + Uptime: h.Uptime, + Downtime: h.Downtime, + SuccessfulInteractions: h.SuccessfulInteractions, + FailedInteractions: h.FailedInteractions, + LostSectors: h.LostSectors, + }, + PriceTable: hostdb.HostPriceTable{ + HostPriceTable: h.PriceTable.convert(), + Expiry: h.PriceTableExpiry.Time, + }, + PublicKey: types.PublicKey(h.PublicKey), + Scanned: h.Scanned, + Settings: h.Settings.convert(), + }, + Blocked: blocked, + Checks: checks, + } +} + +func (hi dbHostCheck) convert() api.HostCheck { + return api.HostCheck{ + Gouging: api.HostGougingBreakdown{ + ContractErr: hi.GougingContractErr, + DownloadErr: hi.GougingDownloadErr, + GougingErr: hi.GougingGougingErr, + PruneErr: hi.GougingPruneErr, + UploadErr: hi.GougingUploadErr, + }, + Score: api.HostScoreBreakdown{ + Age: hi.ScoreAge, + Collateral: hi.ScoreCollateral, + Interactions: hi.ScoreInteractions, + StorageRemaining: hi.ScoreStorageRemaining, + Uptime: hi.ScoreUptime, + Version: hi.ScoreVersion, + Prices: hi.ScorePrices, }, - PriceTable: hostdb.HostPriceTable{ - HostPriceTable: h.PriceTable.convert(), - Expiry: h.PriceTableExpiry.Time, + Usability: api.HostUsabilityBreakdown{ + Blocked: hi.UsabilityBlocked, + Offline: hi.UsabilityOffline, + LowScore: hi.UsabilityLowScore, + RedundantIP: hi.UsabilityRedundantIP, + Gouging: hi.UsabilityGouging, + NotAcceptingContracts: hi.UsabilityNotAcceptingContracts, + NotAnnounced: hi.UsabilityNotAnnounced, + NotCompletingScan: hi.UsabilityNotCompletingScan, }, - PublicKey: types.PublicKey(h.PublicKey), - Scanned: h.Scanned, - Settings: h.Settings.convert(), } } @@ -405,7 +486,7 @@ func (e *dbBlocklistEntry) blocks(h dbHost) bool { } // Host returns information about a host. -func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error) { +func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) { var h dbHost tx := ss.db. @@ -415,15 +496,78 @@ func (ss *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (hostdb.H Preload("Blocklist"). Take(&h) if errors.Is(tx.Error, gorm.ErrRecordNotFound) { - return hostdb.HostInfo{}, api.ErrHostNotFound + return api.Host{}, api.ErrHostNotFound } else if tx.Error != nil { - return hostdb.HostInfo{}, tx.Error + return api.Host{}, tx.Error } - return hostdb.HostInfo{ - Host: h.convert(), - Blocked: ss.isBlocked(h), - }, nil + return h.convert(ss.isBlocked(h)), nil +} + +func (ss *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, hc api.HostCheck) (err error) { + err = ss.db.Transaction(func(tx *gorm.DB) error { + // fetch ap id + var apID uint + if err := tx. + Model(&dbAutopilot{}). + Where("identifier = ?", autopilotID). + Select("id"). + Take(&apID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrAutopilotNotFound + } else if err != nil { + return err + } + + // fetch host id + var hID uint + if err := tx. + Model(&dbHost{}). + Where("public_key = ?", publicKey(hk)). + Select("id"). + Take(&hID). + Error; errors.Is(err, gorm.ErrRecordNotFound) { + return api.ErrHostNotFound + } else if err != nil { + return err + } + + // update host info + return tx. + Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "db_autopilot_id"}, {Name: "db_host_id"}}, + UpdateAll: true, + }). + Create(&dbHostCheck{ + DBAutopilotID: apID, + DBHostID: hID, + + UsabilityBlocked: hc.Usability.Blocked, + UsabilityOffline: hc.Usability.Offline, + UsabilityLowScore: hc.Usability.LowScore, + UsabilityRedundantIP: hc.Usability.RedundantIP, + UsabilityGouging: hc.Usability.Gouging, + UsabilityNotAcceptingContracts: hc.Usability.NotAcceptingContracts, + UsabilityNotAnnounced: hc.Usability.NotAnnounced, + UsabilityNotCompletingScan: hc.Usability.NotCompletingScan, + + ScoreAge: hc.Score.Age, + ScoreCollateral: hc.Score.Collateral, + ScoreInteractions: hc.Score.Interactions, + ScoreStorageRemaining: hc.Score.StorageRemaining, + ScoreUptime: hc.Score.Uptime, + ScoreVersion: hc.Score.Version, + ScorePrices: hc.Score.Prices, + + GougingContractErr: hc.Gouging.ContractErr, + GougingDownloadErr: hc.Gouging.DownloadErr, + GougingGougingErr: hc.Gouging.GougingErr, + GougingPruneErr: hc.Gouging.PruneErr, + GougingUploadErr: hc.Gouging.UploadErr, + }). + Error + }) + return } // HostsForScanning returns the address of hosts for scanning. @@ -461,65 +605,53 @@ func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, return hostAddresses, err } -func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]hostdb.HostInfo, error) { +func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) { if offset < 0 { return nil, ErrNegativeOffset } - // Apply filter mode. - var blocked bool - query := ss.db + // validate filterMode switch filterMode { case api.HostFilterModeAllowed: - query = query.Scopes(ss.excludeBlocked) case api.HostFilterModeBlocked: - query = query.Scopes(ss.excludeAllowed) - blocked = true case api.HostFilterModeAll: - // preload allowlist and blocklist - query = query. - Preload("Allowlist"). - Preload("Blocklist") default: return nil, fmt.Errorf("invalid filter mode: %v", filterMode) } - // Add address filter. - if addressContains != "" { - query = query.Scopes(func(d *gorm.DB) *gorm.DB { - return d.Where("net_address LIKE ?", "%"+addressContains+"%") - }) + // prepare query + query := ss.db. + Model(&dbHost{}). + Scopes( + hostFilter(filterMode, ss.hasAllowlist(), ss.hasBlocklist()), + hostNetAddress(addressContains), + hostPublicKey(keyIn), + ) + + // preload allowlist and blocklist + if filterMode == api.HostFilterModeAll { + query = query. + Preload("Allowlist"). + Preload("Blocklist") } - // Only search for specific hosts. - if len(keyIn) > 0 { - pubKeys := make([]publicKey, len(keyIn)) - for i, pk := range keyIn { - pubKeys[i] = publicKey(pk) - } - query = query.Scopes(func(d *gorm.DB) *gorm.DB { - return d.Where("public_key IN ?", pubKeys) - }) - } + // preload host checks + query = query.Preload("Checks.DBAutopilot") - var hosts []hostdb.HostInfo + var hosts []api.Host var fullHosts []dbHost err := query. Offset(offset). Limit(limit). FindInBatches(&fullHosts, hostRetrievalBatchSize, func(tx *gorm.DB, batch int) error { for _, fh := range fullHosts { + var blocked bool if filterMode == api.HostFilterModeAll { - hosts = append(hosts, hostdb.HostInfo{ - Host: fh.convert(), - Blocked: ss.isBlocked(fh), - }) + blocked = ss.isBlocked(fh) } else { - hosts = append(hosts, hostdb.HostInfo{ - Host: fh.convert(), - Blocked: blocked, - }) + blocked = filterMode == api.HostFilterModeBlocked } + hosts = append(hosts, fh.convert(blocked)) } return nil }). @@ -531,7 +663,7 @@ func (ss *SQLStore) SearchHosts(ctx context.Context, filterMode, addressContains } // Hosts returns non-blocked hosts at given offset and limit. -func (ss *SQLStore) Hosts(ctx context.Context, offset, limit int) ([]hostdb.HostInfo, error) { +func (ss *SQLStore) Hosts(ctx context.Context, offset, limit int) ([]api.Host, error) { return ss.SearchHosts(ctx, api.HostFilterModeAllowed, "", nil, offset, limit) } @@ -930,49 +1062,68 @@ func (ss *SQLStore) processConsensusChangeHostDB(cc modules.ConsensusChange) { ss.unappliedAnnouncements = append(ss.unappliedAnnouncements, newAnnouncements...) } -// excludeBlocked can be used as a scope for a db transaction to exclude blocked -// hosts. -func (ss *SQLStore) excludeBlocked(db *gorm.DB) *gorm.DB { - ss.mu.Lock() - defer ss.mu.Unlock() - - if ss.hasAllowlist { - db = db.Where("EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") - } - if ss.hasBlocklist { - db = db.Where("NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") +// hostNetAddress can be used as a scope to filter hosts by their net address. +func hostNetAddress(addressContains string) func(*gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + if addressContains != "" { + return db.Where("net_address LIKE ?", "%"+addressContains+"%") + } + return db } - return db } -// excludeAllowed can be used as a scope for a db transaction to exclude allowed -// hosts. -func (ss *SQLStore) excludeAllowed(db *gorm.DB) *gorm.DB { - ss.mu.Lock() - defer ss.mu.Unlock() - - if ss.hasAllowlist { - db = db.Where("NOT EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") - } - if ss.hasBlocklist { - db = db.Where("EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") +func hostPublicKey(keyIn []types.PublicKey) func(*gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + if len(keyIn) > 0 { + pubKeys := make([]publicKey, len(keyIn)) + for i, pk := range keyIn { + pubKeys[i] = publicKey(pk) + } + return db.Where("public_key IN ?", pubKeys) + } + return db } - if !ss.hasAllowlist && !ss.hasBlocklist { - // if neither an allowlist nor a blocklist exist, all hosts are allowed - // which means we return none - db = db.Where("1 = 0") +} + +// hostFilter can be used as a scope to filter hosts based on their filter mode, +// returning either all, allowed or blocked hosts. +func hostFilter(filterMode string, hasAllowlist, hasBlocklist bool) func(*gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + switch filterMode { + case api.HostFilterModeAllowed: + if hasAllowlist { + db = db.Where("EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") + } + if hasBlocklist { + db = db.Where("NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") + } + case api.HostFilterModeBlocked: + if hasAllowlist { + db = db.Where("NOT EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") + } + if hasBlocklist { + db = db.Where("EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = hosts.id)") + } + if !hasAllowlist && !hasBlocklist { + // if neither an allowlist nor a blocklist exist, all hosts are allowed + // which means we return none + db = db.Where("1 = 0") + } + case api.HostFilterModeAll: + // do nothing + } + return db } - return db } func (ss *SQLStore) isBlocked(h dbHost) (blocked bool) { ss.mu.Lock() defer ss.mu.Unlock() - if ss.hasAllowlist && len(h.Allowlist) == 0 { + if ss.allowListCnt > 0 && len(h.Allowlist) == 0 { blocked = true } - if ss.hasBlocklist && len(h.Blocklist) > 0 { + if ss.blockListCnt > 0 && len(h.Blocklist) > 0 { blocked = true } return diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 35872ea2d..ec3bc17be 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -248,30 +248,163 @@ func TestSearchHosts(t *testing.T) { // add 3 hosts var hks []types.PublicKey - for i := 0; i < 3; i++ { - if err := ss.addCustomTestHost(types.PublicKey{byte(i)}, fmt.Sprintf("-%v-", i+1)); err != nil { + for i := 1; i <= 3; i++ { + if err := ss.addCustomTestHost(types.PublicKey{byte(i)}, fmt.Sprintf("foo.com:100%d", i)); err != nil { t.Fatal(err) } hks = append(hks, types.PublicKey{byte(i)}) } hk1, hk2, hk3 := hks[0], hks[1], hks[2] - // Search by address. - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "1", nil, 0, -1); err != nil || len(hosts) != 1 { + // search all hosts + his, err := ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 3 { + t.Fatal("unexpected") + } + + // assert offset & limit are taken into account + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 0, 1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 1, 2) + if err != nil { + t.Fatal(err) + } else if len(his) != 2 { + t.Fatal("unexpected") + } + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 3, 1) + if err != nil { + t.Fatal(err) + } else if len(his) != 0 { + t.Fatal("unexpected") + } + + // assert address and key filters are taken into account + if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "com:1001", nil, 0, -1); err != nil || len(hosts) != 1 { t.Fatal("unexpected", len(hosts), err) } - // Filter by key. - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "", []types.PublicKey{hk1, hk2}, 0, -1); err != nil || len(hosts) != 2 { + if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "", []types.PublicKey{hk2, hk3}, 0, -1); err != nil || len(hosts) != 2 { t.Fatal("unexpected", len(hosts), err) } - // Filter by address and key. - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "1", []types.PublicKey{hk1, hk2}, 0, -1); err != nil || len(hosts) != 1 { + if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "com:1002", []types.PublicKey{hk2, hk3}, 0, -1); err != nil || len(hosts) != 1 { t.Fatal("unexpected", len(hosts), err) } - // Filter by key and limit results - if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "3", []types.PublicKey{hk3}, 0, -1); err != nil || len(hosts) != 1 { + if hosts, err := ss.SearchHosts(ctx, api.HostFilterModeAll, "com:1002", []types.PublicKey{hk1}, 0, -1); err != nil || len(hosts) != 0 { t.Fatal("unexpected", len(hosts), err) } + + // assert host filter mode is taken into account + err = ss.UpdateHostBlocklistEntries(context.Background(), []string{"foo.com:1001"}, nil, false) + if err != nil { + t.Fatal(err) + } + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAllowed, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 2 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{2}) || his[1].Host.PublicKey != (types.PublicKey{3}) { + t.Fatal("unexpected", his[0].Host.PublicKey, his[1].Host.PublicKey) + } + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeBlocked, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if len(his) != 1 { + t.Fatal("unexpected") + } else if his[0].Host.PublicKey != (types.PublicKey{1}) { + t.Fatal("unexpected", his) + } + err = ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true) + if err != nil { + t.Fatal(err) + } + + // add two autopilots + ap1 := "ap1" + err = ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: ap1}) + if err != nil { + t.Fatal(err) + } + ap2 := "ap2" + err = ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: ap2}) + if err != nil { + t.Fatal(err) + } + + // add host checks, h1 gets ap1 and h2 gets both, h3 gets none + h1c := newTestHostCheck() + h1c.Score.Age = .1 + err = ss.UpdateHostCheck(context.Background(), ap1, hk1, h1c) + if err != nil { + t.Fatal(err) + } + h2c1 := newTestHostCheck() + h2c1.Score.Age = .21 + err = ss.UpdateHostCheck(context.Background(), ap1, hk2, h2c1) + if err != nil { + t.Fatal(err) + } + h2c2 := newTestHostCheck() + h2c2.Score.Age = .22 + err = ss.UpdateHostCheck(context.Background(), ap2, hk2, h2c2) + if err != nil { + t.Fatal(err) + } + + // assert there are currently 3 checks + var cnt int64 + err = ss.db.Model(&dbHostCheck{}).Count(&cnt).Error + if err != nil { + t.Fatal(err) + } else if cnt != 3 { + t.Fatal("unexpected", cnt) + } + + // fetch all hosts + his, err = ss.SearchHosts(context.Background(), api.HostFilterModeAll, "", nil, 0, -1) + if err != nil { + t.Fatal(err) + } else if cnt != 3 { + t.Fatal("unexpected", cnt) + } + + // assert h1 and h2 have the expected checks + if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c { + t.Fatal("unexpected", c1, ok) + } else if c2, ok := his[1].Checks[ap1]; !ok || c2 != h2c1 { + t.Fatal("unexpected", c2, ok) + } else if c3, ok := his[1].Checks[ap2]; !ok || c3 != h2c2 { + t.Fatal("unexpected", c3, ok) + } + + // assert cascade delete on host + err = ss.db.Exec("DELETE FROM hosts WHERE public_key = ?", publicKey(types.PublicKey{1})).Error + if err != nil { + t.Fatal(err) + } + err = ss.db.Model(&dbHostCheck{}).Count(&cnt).Error + if err != nil { + t.Fatal(err) + } else if cnt != 2 { + t.Fatal("unexpected", cnt) + } + + // assert cascade delete on autopilot + err = ss.db.Exec("DELETE FROM autopilots WHERE identifier IN (?,?)", ap1, ap2).Error + if err != nil { + t.Fatal(err) + } + err = ss.db.Model(&dbHostCheck{}).Count(&cnt).Error + if err != nil { + t.Fatal(err) + } else if cnt != 0 { + t.Fatal("unexpected", cnt) + } } // TestRecordScan is a test for recording scans. @@ -1156,3 +1289,35 @@ func newTestTransaction(ha modules.HostAnnouncement, sk types.PrivateKey) stypes buf.Write(encoding.Marshal(sk.SignHash(types.Hash256(crypto.HashObject(ha))))) return stypes.Transaction{ArbitraryData: [][]byte{buf.Bytes()}} } + +func newTestHostCheck() api.HostCheck { + return api.HostCheck{ + + Gouging: api.HostGougingBreakdown{ + ContractErr: "foo", + DownloadErr: "bar", + GougingErr: "baz", + PruneErr: "qux", + UploadErr: "quuz", + }, + Score: api.HostScoreBreakdown{ + Age: .1, + Collateral: .2, + Interactions: .3, + StorageRemaining: .4, + Uptime: .5, + Version: .6, + Prices: .7, + }, + Usability: api.HostUsabilityBreakdown{ + Blocked: true, + Offline: true, + LowScore: true, + RedundantIP: true, + Gouging: true, + NotAcceptingContracts: true, + NotAnnounced: true, + NotCompletingScan: true, + }, + } +} diff --git a/stores/metadata_test.go b/stores/metadata_test.go index c16f927d1..eb082fb54 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -440,12 +440,12 @@ func TestContractsForHost(t *testing.T) { } contracts, _ := contractsForHost(ss.db, hosts[0]) - if len(contracts) != 1 || contracts[0].Host.convert().PublicKey.String() != hosts[0].convert().PublicKey.String() { + if len(contracts) != 1 || types.PublicKey(contracts[0].Host.PublicKey).String() != types.PublicKey(hosts[0].PublicKey).String() { t.Fatal("unexpected", len(contracts), contracts) } contracts, _ = contractsForHost(ss.db, hosts[1]) - if len(contracts) != 1 || contracts[0].Host.convert().PublicKey.String() != hosts[1].convert().PublicKey.String() { + if len(contracts) != 1 || types.PublicKey(contracts[0].Host.PublicKey).String() != types.PublicKey(hosts[1].PublicKey).String() { t.Fatalf("unexpected contracts, %+v", contracts) } } diff --git a/stores/migrations.go b/stores/migrations.go index 6ccc75964..4ac6b755e 100644 --- a/stores/migrations.go +++ b/stores/migrations.go @@ -62,6 +62,12 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { return performMigration(tx, dbIdentifier, "00006_idx_objects_created_at", logger) }, }, + { + ID: "00007_host_checks", + Migrate: func(tx *gorm.DB) error { + return performMigration(tx, dbIdentifier, "00007_host_checks", logger) + }, + }, } // Create migrator. diff --git a/stores/migrations/mysql/main/migration_00007_host_checks.sql b/stores/migrations/mysql/main/migration_00007_host_checks.sql new file mode 100644 index 000000000..f96b9853c --- /dev/null +++ b/stores/migrations/mysql/main/migration_00007_host_checks.sql @@ -0,0 +1,52 @@ +-- dbHostCheck +CREATE TABLE `host_checks` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime(3) DEFAULT NULL, + + `db_autopilot_id` bigint unsigned NOT NULL, + `db_host_id` bigint unsigned NOT NULL, + + `usability_blocked` boolean NOT NULL DEFAULT false, + `usability_offline` boolean NOT NULL DEFAULT false, + `usability_low_score` boolean NOT NULL DEFAULT false, + `usability_redundant_ip` boolean NOT NULL DEFAULT false, + `usability_gouging` boolean NOT NULL DEFAULT false, + `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, + `usability_not_announced` boolean NOT NULL DEFAULT false, + `usability_not_completing_scan` boolean NOT NULL DEFAULT false, + + `score_age` double NOT NULL, + `score_collateral` double NOT NULL, + `score_interactions` double NOT NULL, + `score_storage_remaining` double NOT NULL, + `score_uptime` double NOT NULL, + `score_version` double NOT NULL, + `score_prices` double NOT NULL, + + `gouging_contract_err` text, + `gouging_download_err` text, + `gouging_gouging_err` text, + `gouging_prune_err` text, + `gouging_upload_err` text, + + PRIMARY KEY (`id`), + UNIQUE KEY `idx_host_checks_id` (`db_autopilot_id`, `db_host_id`), + INDEX `idx_host_checks_usability_blocked` (`usability_blocked`), + INDEX `idx_host_checks_usability_offline` (`usability_offline`), + INDEX `idx_host_checks_usability_low_score` (`usability_low_score`), + INDEX `idx_host_checks_usability_redundant_ip` (`usability_redundant_ip`), + INDEX `idx_host_checks_usability_gouging` (`usability_gouging`), + INDEX `idx_host_checks_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), + INDEX `idx_host_checks_usability_not_announced` (`usability_not_announced`), + INDEX `idx_host_checks_usability_not_completing_scan` (`usability_not_completing_scan`), + INDEX `idx_host_checks_score_age` (`score_age`), + INDEX `idx_host_checks_score_collateral` (`score_collateral`), + INDEX `idx_host_checks_score_interactions` (`score_interactions`), + INDEX `idx_host_checks_score_storage_remaining` (`score_storage_remaining`), + INDEX `idx_host_checks_score_uptime` (`score_uptime`), + INDEX `idx_host_checks_score_version` (`score_version`), + INDEX `idx_host_checks_score_prices` (`score_prices`), + + CONSTRAINT `fk_host_checks_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_host_checks_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; diff --git a/stores/migrations/mysql/main/schema.sql b/stores/migrations/mysql/main/schema.sql index 68b42ae47..446b2a805 100644 --- a/stores/migrations/mysql/main/schema.sql +++ b/stores/migrations/mysql/main/schema.sql @@ -422,5 +422,58 @@ CREATE TABLE `object_user_metadata` ( CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +-- dbHostCheck +CREATE TABLE `host_checks` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime(3) DEFAULT NULL, + + `db_autopilot_id` bigint unsigned NOT NULL, + `db_host_id` bigint unsigned NOT NULL, + + `usability_blocked` boolean NOT NULL DEFAULT false, + `usability_offline` boolean NOT NULL DEFAULT false, + `usability_low_score` boolean NOT NULL DEFAULT false, + `usability_redundant_ip` boolean NOT NULL DEFAULT false, + `usability_gouging` boolean NOT NULL DEFAULT false, + `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, + `usability_not_announced` boolean NOT NULL DEFAULT false, + `usability_not_completing_scan` boolean NOT NULL DEFAULT false, + + `score_age` double NOT NULL, + `score_collateral` double NOT NULL, + `score_interactions` double NOT NULL, + `score_storage_remaining` double NOT NULL, + `score_uptime` double NOT NULL, + `score_version` double NOT NULL, + `score_prices` double NOT NULL, + + `gouging_contract_err` text, + `gouging_download_err` text, + `gouging_gouging_err` text, + `gouging_prune_err` text, + `gouging_upload_err` text, + + PRIMARY KEY (`id`), + UNIQUE KEY `idx_host_checks_id` (`db_autopilot_id`, `db_host_id`), + INDEX `idx_host_checks_usability_blocked` (`usability_blocked`), + INDEX `idx_host_checks_usability_offline` (`usability_offline`), + INDEX `idx_host_checks_usability_low_score` (`usability_low_score`), + INDEX `idx_host_checks_usability_redundant_ip` (`usability_redundant_ip`), + INDEX `idx_host_checks_usability_gouging` (`usability_gouging`), + INDEX `idx_host_checks_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), + INDEX `idx_host_checks_usability_not_announced` (`usability_not_announced`), + INDEX `idx_host_checks_usability_not_completing_scan` (`usability_not_completing_scan`), + INDEX `idx_host_checks_score_age` (`score_age`), + INDEX `idx_host_checks_score_collateral` (`score_collateral`), + INDEX `idx_host_checks_score_interactions` (`score_interactions`), + INDEX `idx_host_checks_score_storage_remaining` (`score_storage_remaining`), + INDEX `idx_host_checks_score_uptime` (`score_uptime`), + INDEX `idx_host_checks_score_version` (`score_version`), + INDEX `idx_host_checks_score_prices` (`score_prices`), + + CONSTRAINT `fk_host_checks_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_host_checks_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); \ No newline at end of file diff --git a/stores/migrations/sqlite/main/migration_00007_host_checks.sql b/stores/migrations/sqlite/main/migration_00007_host_checks.sql new file mode 100644 index 000000000..da13460c6 --- /dev/null +++ b/stores/migrations/sqlite/main/migration_00007_host_checks.sql @@ -0,0 +1,52 @@ +-- dbHostCheck +CREATE TABLE `host_checks` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `created_at` datetime, + + `db_autopilot_id` INTEGER NOT NULL, + `db_host_id` INTEGER NOT NULL, + + `usability_blocked` INTEGER NOT NULL DEFAULT 0, + `usability_offline` INTEGER NOT NULL DEFAULT 0, + `usability_low_score` INTEGER NOT NULL DEFAULT 0, + `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, + `usability_gouging` INTEGER NOT NULL DEFAULT 0, + `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, + `usability_not_announced` INTEGER NOT NULL DEFAULT 0, + `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, + + `score_age` REAL NOT NULL, + `score_collateral` REAL NOT NULL, + `score_interactions` REAL NOT NULL, + `score_storage_remaining` REAL NOT NULL, + `score_uptime` REAL NOT NULL, + `score_version` REAL NOT NULL, + `score_prices` REAL NOT NULL, + + `gouging_contract_err` TEXT, + `gouging_download_err` TEXT, + `gouging_gouging_err` TEXT, + `gouging_prune_err` TEXT, + `gouging_upload_err` TEXT, + + FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, + FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +); + +-- Indexes creation +CREATE UNIQUE INDEX `idx_host_checks_id` ON `host_checks` (`db_autopilot_id`, `db_host_id`); +CREATE INDEX `idx_host_checks_usability_blocked` ON `host_checks` (`usability_blocked`); +CREATE INDEX `idx_host_checks_usability_offline` ON `host_checks` (`usability_offline`); +CREATE INDEX `idx_host_checks_usability_low_score` ON `host_checks` (`usability_low_score`); +CREATE INDEX `idx_host_checks_usability_redundant_ip` ON `host_checks` (`usability_redundant_ip`); +CREATE INDEX `idx_host_checks_usability_gouging` ON `host_checks` (`usability_gouging`); +CREATE INDEX `idx_host_checks_usability_not_accepting_contracts` ON `host_checks` (`usability_not_accepting_contracts`); +CREATE INDEX `idx_host_checks_usability_not_announced` ON `host_checks` (`usability_not_announced`); +CREATE INDEX `idx_host_checks_usability_not_completing_scan` ON `host_checks` (`usability_not_completing_scan`); +CREATE INDEX `idx_host_checks_score_age` ON `host_checks` (`score_age`); +CREATE INDEX `idx_host_checks_score_collateral` ON `host_checks` (`score_collateral`); +CREATE INDEX `idx_host_checks_score_interactions` ON `host_checks` (`score_interactions`); +CREATE INDEX `idx_host_checks_score_storage_remaining` ON `host_checks` (`score_storage_remaining`); +CREATE INDEX `idx_host_checks_score_uptime` ON `host_checks` (`score_uptime`); +CREATE INDEX `idx_host_checks_score_version` ON `host_checks` (`score_version`); +CREATE INDEX `idx_host_checks_score_prices` ON `host_checks` (`score_prices`); diff --git a/stores/migrations/sqlite/main/schema.sql b/stores/migrations/sqlite/main/schema.sql index 9875e81e3..3fca53a3a 100644 --- a/stores/migrations/sqlite/main/schema.sql +++ b/stores/migrations/sqlite/main/schema.sql @@ -149,5 +149,24 @@ CREATE UNIQUE INDEX `idx_module_event_url` ON `webhooks`(`module`,`event`,`url`) CREATE TABLE `object_user_metadata` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer DEFAULT NULL,`db_multipart_upload_id` integer DEFAULT NULL,`key` text NOT NULL,`value` text, CONSTRAINT `fk_object_user_metadata` FOREIGN KEY (`db_object_id`) REFERENCES `objects` (`id`) ON DELETE CASCADE, CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL); CREATE UNIQUE INDEX `idx_object_user_metadata_key` ON `object_user_metadata`(`db_object_id`,`db_multipart_upload_id`,`key`); +-- dbHostCheck +CREATE TABLE `host_checks` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_autopilot_id` INTEGER NOT NULL, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); +CREATE UNIQUE INDEX `idx_host_checks_id` ON `host_checks` (`db_autopilot_id`, `db_host_id`); +CREATE INDEX `idx_host_checks_usability_blocked` ON `host_checks` (`usability_blocked`); +CREATE INDEX `idx_host_checks_usability_offline` ON `host_checks` (`usability_offline`); +CREATE INDEX `idx_host_checks_usability_low_score` ON `host_checks` (`usability_low_score`); +CREATE INDEX `idx_host_checks_usability_redundant_ip` ON `host_checks` (`usability_redundant_ip`); +CREATE INDEX `idx_host_checks_usability_gouging` ON `host_checks` (`usability_gouging`); +CREATE INDEX `idx_host_checks_usability_not_accepting_contracts` ON `host_checks` (`usability_not_accepting_contracts`); +CREATE INDEX `idx_host_checks_usability_not_announced` ON `host_checks` (`usability_not_announced`); +CREATE INDEX `idx_host_checks_usability_not_completing_scan` ON `host_checks` (`usability_not_completing_scan`); +CREATE INDEX `idx_host_checks_score_age` ON `host_checks` (`score_age`); +CREATE INDEX `idx_host_checks_score_collateral` ON `host_checks` (`score_collateral`); +CREATE INDEX `idx_host_checks_score_interactions` ON `host_checks` (`score_interactions`); +CREATE INDEX `idx_host_checks_score_storage_remaining` ON `host_checks` (`score_storage_remaining`); +CREATE INDEX `idx_host_checks_score_uptime` ON `host_checks` (`score_uptime`); +CREATE INDEX `idx_host_checks_score_version` ON `host_checks` (`score_version`); +CREATE INDEX `idx_host_checks_score_prices` ON `host_checks` (`score_prices`); + -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); diff --git a/stores/sql.go b/stores/sql.go index f62dba97f..34a6d78ab 100644 --- a/stores/sql.go +++ b/stores/sql.go @@ -104,8 +104,8 @@ type ( wg sync.WaitGroup mu sync.Mutex - hasAllowlist bool - hasBlocklist bool + allowListCnt uint64 + blockListCnt uint64 closed bool knownContracts map[types.FileContractID]struct{} @@ -258,8 +258,8 @@ func NewSQLStore(cfg Config) (*SQLStore, modules.ConsensusChangeID, error) { knownContracts: isOurContract, lastSave: time.Now(), persistInterval: cfg.PersistInterval, - hasAllowlist: allowlistCnt > 0, - hasBlocklist: blocklistCnt > 0, + allowListCnt: uint64(allowlistCnt), + blockListCnt: uint64(blocklistCnt), settings: make(map[string]string), slabPruneSigChan: make(chan struct{}, 1), unappliedContractState: make(map[types.FileContractID]contractState), @@ -299,6 +299,12 @@ func isSQLite(db *gorm.DB) bool { } } +func (ss *SQLStore) hasAllowlist() bool { + ss.mu.Lock() + defer ss.mu.Unlock() + return ss.allowListCnt > 0 +} + func (ss *SQLStore) updateHasAllowlist(err *error) { if *err != nil { return @@ -311,10 +317,16 @@ func (ss *SQLStore) updateHasAllowlist(err *error) { } ss.mu.Lock() - ss.hasAllowlist = cnt > 0 + ss.allowListCnt = uint64(cnt) ss.mu.Unlock() } +func (ss *SQLStore) hasBlocklist() bool { + ss.mu.Lock() + defer ss.mu.Unlock() + return ss.blockListCnt > 0 +} + func (ss *SQLStore) updateHasBlocklist(err *error) { if *err != nil { return @@ -327,7 +339,7 @@ func (ss *SQLStore) updateHasBlocklist(err *error) { } ss.mu.Lock() - ss.hasBlocklist = cnt > 0 + ss.blockListCnt = uint64(cnt) ss.mu.Unlock() } diff --git a/worker/mocks_test.go b/worker/mocks_test.go index 7b3609c0b..baf83b39d 100644 --- a/worker/mocks_test.go +++ b/worker/mocks_test.go @@ -260,13 +260,18 @@ var errSectorOutOfBounds = errors.New("sector out of bounds") type hostMock struct { hk types.PublicKey - hi hostdb.HostInfo + hi api.Host } func newHostMock(hk types.PublicKey) *hostMock { return &hostMock{ hk: hk, - hi: hostdb.HostInfo{Host: hostdb.Host{PublicKey: hk, Scanned: true}}, + hi: api.Host{ + Host: hostdb.Host{ + PublicKey: hk, + Scanned: true, + }, + }, } } @@ -282,13 +287,13 @@ func newHostStoreMock() *hostStoreMock { return &hostStoreMock{hosts: make(map[types.PublicKey]*hostMock)} } -func (hs *hostStoreMock) Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error) { +func (hs *hostStoreMock) Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) { hs.mu.Lock() defer hs.mu.Unlock() h, ok := hs.hosts[hostKey] if !ok { - return hostdb.HostInfo{}, api.ErrHostNotFound + return api.Host{}, api.ErrHostNotFound } return h.hi, nil } diff --git a/worker/worker.go b/worker/worker.go index 89fe37a14..d0de33f71 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -110,7 +110,7 @@ type ( RecordPriceTables(ctx context.Context, priceTableUpdate []hostdb.PriceTableUpdate) error RecordContractSpending(ctx context.Context, records []api.ContractSpendingRecord) error - Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error) + Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) } ObjectStore interface {