Cleanup worker for challenge tables (#1278)

* add cleanup worker for challenge tables

* move cleanupGap to config

---------

Co-authored-by: Yury <yuderbasov@gmail.com>

code/go/0chain.net/blobber/config.go

@@ -49,6 +49,7 @@ func setupConfig(configDir string, deploymentMode int) {
 	config.Configuration.ChallengeResolveFreq = viper.GetInt64("challenge_response.frequency")
 	config.Configuration.ChallengeResolveNumWorkers = viper.GetInt("challenge_response.num_workers")
 	config.Configuration.ChallengeMaxRetires = viper.GetInt("challenge_response.max_retries")
+	config.Configuration.ChallengeCleanupGap = viper.GetInt64("challenge_response.cleanup_gap")
 
 	config.Configuration.AutomaticUpdate = viper.GetBool("disk_update.automatic_update")
 	blobberUpdateIntrv := viper.GetDuration("disk_update.blobber_update_interval")

code/go/0chain.net/blobber/worker.go

@@ -24,6 +24,8 @@ func setupWorkers(ctx context.Context) {
 	allocation.StartUpdateWorker(ctx, config.Configuration.UpdateAllocationsInterval)
 	allocation.StartFinalizeWorker(ctx, config.Configuration.FinalizeAllocationsInterval)
 	allocation.SetupWorkers(ctx)
+	challenge.SetupChallengeCleanUpWorker(ctx)
+	challenge.SetupChallengeTimingsCleanupWorker(ctx)
 	updateCCTWorker(ctx)
 }
 

code/go/0chain.net/blobbercore/challenge/entity.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
 	"github.com/0chain/blobber/code/go/0chain.net/core/common"
@@ -48,6 +49,11 @@ const (
 	ChallengeFailure
 )
 
+const (
+	cleanupInterval = 30 * time.Minute
+	cleanupGap      = 1 * time.Hour
+)
+
 type ValidationTicket struct {
 	ChallengeID  string           `json:"challenge_id"`
 	BlobberID    string           `json:"blobber_id"`
@@ -206,3 +212,24 @@ func GetChallengeEntity(ctx context.Context, challengeID string) (*ChallengeEnti
 	}
 	return cr, nil
 }
+
+func SetupChallengeCleanUpWorker(ctx context.Context) {
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(cleanupInterval):
+				cleanUpWorker()
+			}
+		}
+	}()
+}
+
+func cleanUpWorker() {
+	currentRound := roundInfo.CurrentRound + int64(float64(roundInfo.LastRoundDiff)*(float64(time.Since(roundInfo.CurrentRoundCaptureTime).Milliseconds())/float64(GetRoundInterval.Milliseconds())))
+	_ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
+		db := datastore.GetStore().GetTransaction(ctx)
+		return db.Model(&ChallengeEntity{}).Unscoped().Delete(&ChallengeEntity{}, "status <> ? AND round_created_at < ?", Cancelled, currentRound-config.Configuration.ChallengeCleanupGap).Error
+	})
+}

code/go/0chain.net/blobbercore/challenge/timing.go

@@ -3,6 +3,7 @@ package challenge
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
 	"github.com/0chain/blobber/code/go/0chain.net/core/common"
@@ -186,3 +187,24 @@ func GetChallengeTiming(challengeID string) (*ChallengeTiming, error) {
 	})
 	return ch, err
 }
+
+func SetupChallengeTimingsCleanupWorker(ctx context.Context) {
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(cleanupInterval):
+				cleanUpTimingWorker()
+			}
+		}
+	}()
+}
+
+func cleanUpTimingWorker() {
+	_ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error {
+		db := datastore.GetStore().GetTransaction(ctx)
+		return db.Model(&ChallengeTiming{}).Unscoped().Delete(&ChallengeTiming{}, "created_at_blobber < ?", time.Now().Add(-cleanupGap).Unix()).Error
+	})
+}

code/go/0chain.net/blobbercore/config/config.go

@@ -22,6 +22,7 @@ func SetupDefaultConfig() {
 	viper.SetDefault("challenge_response.frequency", 10)
 	viper.SetDefault("challenge_response.num_workers", 5)
 	viper.SetDefault("challenge_response.max_retries", 10)
+	viper.SetDefault("challenge_response.cleanup_gap", 100000)
 	viper.SetDefault("rate_limiters.block_limit_daily", 1562500)
 	viper.SetDefault("rate_limiters.block_limit_request", 500)
 
@@ -92,6 +93,7 @@ type Config struct {
 	TempFilesCleanupNumWorkers    int
 	BlockLimitDaily               int64
 	BlockLimitRequest             int64
+	ChallengeCleanupGap           int64
 
 	HealthCheckWorkerFreq time.Duration
 

config/0chain_blobber.yaml

@@ -89,6 +89,7 @@ challenge_response:
   frequency: 10
   num_workers: 5
   max_retries: 20
+  cleanup_gap: 100000
 
 healthcheck:
   frequency: 60m # send healthcheck to miners every 60 minutes