From 39f3f7dd44bf93fea78a423c6397a39b839d9ae5 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 29 Nov 2023 14:39:16 +0100 Subject: [PATCH 1/3] stores: optimise ContractSizes --- stores/metadata.go | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/stores/metadata.go b/stores/metadata.go index 81afd1207..cf9dc7c71 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -919,26 +919,38 @@ func (s *SQLStore) ContractSets(ctx context.Context) ([]string, error) { } func (s *SQLStore) ContractSizes(ctx context.Context) (map[types.FileContractID]api.ContractSize, error) { - rows := make([]struct { + type size struct { Fcid fileContractID `json:"fcid"` Size uint64 `json:"size"` Prunable uint64 `json:"prunable"` - }, 0) + } - if err := s.db. - Raw(` -SELECT fcid, MAX(c.size) as size, CASE WHEN MAX(c.size)>COUNT(cs.db_sector_id) * ? THEN MAX(c.size)-(COUNT(cs.db_sector_id) * ?) ELSE 0 END as prunable -FROM contracts c -LEFT JOIN contract_sectors cs ON cs.db_contract_id = c.id -GROUP BY c.fcid - `, rhpv2.SectorSize, rhpv2.SectorSize). - Scan(&rows). - Error; err != nil { + var nullContracts []size + var dataContracts []size + if err := s.db.Transaction(func(tx *gorm.DB) error { + // use WHERE NOT EXISTS to catch contracts that would otherwise have been caught by a LEFT JOIN + if err := tx. + Raw(` +SELECT c.fcid, c.size, c.size as prunable FROM contracts c WHERE NOT EXISTS (SELECT 1 FROM contract_sectors cs WHERE cs.db_contract_id = c.id)`). + Scan(&nullContracts). + Error; err != nil { + return err + } + + // with those out of the way, we can use an INNER JOIN to get the rest + return tx. + Raw(` +SELECT fcid, contract_size as size, CASE WHEN contract_size > sector_size THEN contract_size - sector_size ELSE 0 END as prunable FROM ( +SELECT c.fcid, MAX(c.size) as contract_size, COUNT(cs.db_sector_id) * ? as sector_size FROM contracts c INNER JOIN contract_sectors cs ON cs.db_contract_id = c.id GROUP BY c.fcid +) i`, rhpv2.SectorSize). + Scan(&dataContracts). + Error + }); err != nil { return nil, err } sizes := make(map[types.FileContractID]api.ContractSize) - for _, row := range rows { + for _, row := range append(nullContracts, dataContracts...) { if types.FileContractID(row.Fcid) == (types.FileContractID{}) { return nil, errors.New("invalid file contract id") } @@ -962,11 +974,10 @@ func (s *SQLStore) ContractSize(ctx context.Context, id types.FileContractID) (a if err := s.db. Raw(` -SELECT MAX(c.size) as size, CASE WHEN MAX(c.size)>(COUNT(cs.db_sector_id) * ?) THEN MAX(c.size)-(COUNT(cs.db_sector_id) * ?) ELSE 0 END as prunable -FROM contracts c -LEFT JOIN contract_sectors cs ON cs.db_contract_id = c.id -WHERE c.fcid = ? -`, rhpv2.SectorSize, rhpv2.SectorSize, fileContractID(id)). +SELECT contract_size as size, CASE WHEN contract_size > sector_size THEN contract_size - sector_size ELSE 0 END as prunable FROM ( +SELECT MAX(c.size) as contract_size, COUNT(cs.db_sector_id) * ? as sector_size FROM contracts c LEFT JOIN contract_sectors cs ON cs.db_contract_id = c.id WHERE c.fcid = ? +) i +`, rhpv2.SectorSize, fileContractID(id)). Take(&size). Error; err != nil { return api.ContractSize{}, err From 8ab4b4dd65cb69c6fb0246979aaeafbc06b6a5a3 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 30 Nov 2023 10:28:34 +0100 Subject: [PATCH 2/3] stores: fix TestSlabHealthInvalidation NDF --- stores/metadata_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stores/metadata_test.go b/stores/metadata_test.go index d4756ed57..95534189d 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -3765,7 +3765,7 @@ func TestSlabHealthInvalidation(t *testing.T) { } // refresh health - now := time.Now().Round(time.Second) + now := time.Now() if err := ss.RefreshHealth(context.Background()); err != nil { t.Fatal(err) } From 7a2b3660092e91beaaac2b5c4ce60e18a2a28c45 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 30 Nov 2023 17:19:19 +0100 Subject: [PATCH 3/3] stores: update comments --- stores/metadata.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/stores/metadata.go b/stores/metadata.go index cf9dc7c71..689b0b502 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -928,7 +928,8 @@ func (s *SQLStore) ContractSizes(ctx context.Context) (map[types.FileContractID] var nullContracts []size var dataContracts []size if err := s.db.Transaction(func(tx *gorm.DB) error { - // use WHERE NOT EXISTS to catch contracts that would otherwise have been caught by a LEFT JOIN + // first, we fetch all contracts without sectors and consider their + // entire size as prunable if err := tx. Raw(` SELECT c.fcid, c.size, c.size as prunable FROM contracts c WHERE NOT EXISTS (SELECT 1 FROM contract_sectors cs WHERE cs.db_contract_id = c.id)`). @@ -937,7 +938,10 @@ SELECT c.fcid, c.size, c.size as prunable FROM contracts c WHERE NOT EXISTS (SEL return err } - // with those out of the way, we can use an INNER JOIN to get the rest + // second, we fetch how much data can be pruned from all contracts that + // do have sectors, we take a two-step approach because it allows us to + // use an INNER JOIN on contract_sectors, drastically improving the + // performance of the query return tx. Raw(` SELECT fcid, contract_size as size, CASE WHEN contract_size > sector_size THEN contract_size - sector_size ELSE 0 END as prunable FROM (