add shutdown-timeout, raft-snapshot-interval

api/config.go

@@ -1,6 +1,8 @@
 package api
 
 import (
+	"time"
+
 	dbApi "github.com/tdx/rkv/db/api"
 
 	"github.com/hashicorp/raft"
@@ -23,4 +25,6 @@ type Config struct {
 	LogLevel string
 	// Backend
 	Backend dbApi.Backend
+	// Used in k8s. Delay to allow balancer remove traffice from endpoint
+	ShutdownDelay time.Duration
 }

cmd/rkvd/main.go

@@ -7,6 +7,7 @@ import (
 	"path"
 	"path/filepath"
 	"syscall"
+	"time"
 
 	"github.com/tdx/rkv"
 	rkvApi "github.com/tdx/rkv/api"
@@ -66,6 +67,7 @@ func setupFlags(cmd *cobra.Command) error {
 	cmd.Flags().Int("rpc-port", 8402, "Port for RPC connections.")
 	cmd.Flags().String("http-addr", ":8403", "Address to bind HTTP on.")
 	cmd.Flags().String("log-level", "info", "Log level.")
+	cmd.Flags().String("shutdown-delay", "", "Used for k8s balancer to remove traffic from pod")
 
 	return viper.BindPFlags(cmd.Flags())
 }
@@ -117,6 +119,8 @@ func (c *cli) setupConfig(cmd *cobra.Command, args []string) error {
 	c.Config.RPCPort = viper.GetInt("rpc-port")
 	c.Config.HTTPAddr = viper.GetString("http-addr")
 
+	c.Config.ShutdownDelay = viper.GetDuration("shutdown-delay")
+
 	return nil
 }
 
@@ -128,11 +132,25 @@ func (c *cli) run(cmd *cobra.Command, args []string) error {
 	c.logger = client.Logger("main")
 	c.logger.SetPrefix("rkvd ")
 
+	c.logger.Println("config:", "shutdown-delay=", c.Config.ShutdownDelay)
+
+	done := make(chan struct{})
 	sigc := make(chan os.Signal, 1)
-	signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
-	s := <-sigc
 
-	c.logger.Println("stopped by signal:", s.String())
+	signal.Notify(sigc, syscall.SIGTERM)
+	go func() {
+		s := <-sigc
+		c.logger.Println("got signal signal:", s.String(),
+			", delay shutdown:", c.Config.ShutdownDelay)
+
+		// allow k8s load balancer remove traffic from pod
+		if c.Config.ShutdownDelay > 0 {
+			time.Sleep(c.Config.ShutdownDelay)
+		}
+		done <- struct{}{}
+	}()
+
+	<-done
 
 	return client.Shutdown()
 }

deploy/rkvd/templates/statefulset.yaml

@@ -49,6 +49,8 @@ spec:
             bootstrap: $([ $ID = 0 ] && echo true || echo false )
             http-addr: ":{{.Values.httpPort}}"
             log-level: {{ .Values.log }}
+            raft-snapshot-interval: {{ .Values.raftSnapshotInterval }}
+            shutdown-delay: {{ .Values.shutdownDelay }}
             EOD
         volumeMounts:
         - name: datadir

deploy/rkvd/values.yaml

@@ -12,5 +12,6 @@ rpcPort: 8402
 httpPort: 8403
 replicas: 3
 storage: 1Gi
+raftSnapshotInterval: 120s
 db: bolt
-log: debug
+log: trace

example/cluster/put

@@ -25,5 +25,6 @@ curl -X POST \
         -s -w " %{http_code}" \
 	-H "Content-Type: application/json" \
 	-d '{"tab": "'$tab'","key":"'$key'","val":"'$val'"}' \
+        --connect-timeout 10 --max-time 10 \
         http://$hostname:$port/db/put
 

internal/cluster/raft/raft.go

@@ -155,7 +155,7 @@ func (d *Backend) setupRaft(dataDir string) error {
 		d.logger.Named("snapshot"),
 	)
 
-	maxPool := 5
+	maxPool := 1
 	timeout := 10 * time.Second
 	transport := raft.NewNetworkTransport(
 		d.config.StreamLayer,