add slot nums to config (OpenAtomFoundation#145) (OpenAtomFoundation#…

…1664)

* add slot nums to config (OpenAtomFoundation#145)

* fix migrate list slot logic (OpenAtomFoundation#156)

* fix sadd slot (OpenAtomFoundation#157)

Co-authored-by: liuyuecai <liuyuecai@360.cn>

---------

Co-authored-by: liuyuecai <liuyuecai@360.cn>

codis/ansible/roles/codis-proxy/templates/proxy.toml

@@ -62,11 +62,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/cmd/admin/admin.go

@@ -265,7 +265,7 @@ func (t *cmdAdmin) handleConfigConvert(d map[string]interface{}) {
 		for _, v := range slots.(map[string]interface{}) {
 			t.convertSlotsV1(temp, v)
 		}
-		for i := 0; i < models.MaxSlotNum; i++ {
+		for i := 0; i < models.GetMaxSlotNum(); i++ {
 			if temp[i] == nil {
 				continue
 			}
@@ -333,7 +333,7 @@ func (t *cmdAdmin) loadJsonConfigV3(file string) *ConfigV3 {
 
 	var slots = make(map[int]*models.SlotMapping)
 	for _, s := range config.Slots {
-		if s.Id < 0 || s.Id >= models.MaxSlotNum {
+		if s.Id < 0 || s.Id >= models.GetMaxSlotNum() {
 			log.Panicf("invalid slot id = %d", s.Id)
 		}
 		if slots[s.Id] != nil {

codis/cmd/admin/proxy.go

@@ -145,7 +145,7 @@ func (t *cmdProxy) handleFillSlots(d map[string]interface{}) {
 	}
 
 	for _, m := range slots {
-		if m.Id < 0 || m.Id >= models.MaxSlotNum {
+		if m.Id < 0 || m.Id >= models.GetMaxSlotNum() {
 			log.Panicf("invalid slot id = %d", m.Id)
 		}
 	}

codis/cmd/dashboard/main.go

@@ -83,6 +83,7 @@ Options:
 			log.PanicErrorf(err, "load config %s failed", s)
 		}
 	}
+	models.SetMaxSlotNum(config.MaxSlotNum)
 	if s, ok := utils.Argument(d, "--host-admin"); ok {
 		config.HostAdmin = s
 		log.Warnf("option --host-admin = %s", s)

codis/cmd/fe/assets/css/main.css

@@ -1,5 +1,5 @@
 body {
-    min-height: 20000px;
+    /*min-height: 20000px;*/
     min-width: 1000px;
 }
 

codis/cmd/fe/assets/dashboard-fe.js

@@ -183,7 +183,7 @@ function renderSlotsCharts(slots_array) {
         },
         yAxis: {
             min: 0,
-            max: 1024,
+            max: n,
             tickInterval: 64,
             title: {
                 style: {
@@ -623,6 +623,7 @@ dashboard.controller('MainCodisCtrl', ['$scope', '$http', '$uibModal', '$timeout
                 $scope.codis_addr = overview.model.admin_addr;
                 $scope.codis_coord_name = "[" + overview.config.coordinator_name.charAt(0).toUpperCase() + overview.config.coordinator_name.slice(1) + "]";
                 $scope.codis_coord_addr = overview.config.coordinator_addr;
+                $scope.max_slot_num = overview.config.max_slot_num;
                 $scope.updateStats(overview.stats);
             });
         }

codis/cmd/fe/assets/index.html

@@ -70,6 +70,10 @@ <h4 style="padding-left:20px;"><b>Overview</b></h4>
                         <td>Redis Keys</td>
                         <td>[[redis_keys]]</td>
                     </tr>
+                    <tr>
+                        <td>Slot Nums</td>
+                        <td>[[max_slot_num]]</td>
+                    </tr>
                     <tr>
                         <td>Dashboard</td>
                         <td ng-switch="codis_addr">
@@ -128,6 +132,7 @@ <h4 style="padding-left:30px; padding-right:20px; display: inline;">Proxy</h4>
                         <th style="width: 60px;"/>
                         <th style="min-width: 100px;">Data Center</th>
                         <th style="width: 35px;"/>
+                        <th style="min-width: 60px;">Slot Nums</th>
                         <th style="min-width: 140px;">Sessions</th>
                         <th style="min-width: 200px;">Commands</th>
                         <th style="width: 35px;"></th>
@@ -181,6 +186,7 @@ <h4 style="padding-left:30px; padding-right:20px; display: inline;">Proxy</h4>
                                 <span class="glyphicon glyphicon-remove"></span>
                             </button>
                         </td>
+                        <td>[[proxy.max_slot_num]]</td>
                         <td>[[proxy.sessions]]</td>
                         <td ng-switch="proxy.status">
                             <span ng-switch-when="ERROR" class="status_label_error">[[proxy.status]]</span>
@@ -219,11 +225,11 @@ <h4 style="padding-left:30px; padding-right:20px; display: inline;">Slots</h4>
                         </button>
                         Slots-[
                         <input style="height: 28px; width: 140px; " type="text"
-                               class="form-control" placeholder="[0,1023]"
+                               class="form-control" placeholder=""
                                ng-model="migrate_slot_id_beg">
                         ~
                         <input style="height: 28px; width: 140px; " type="text"
-                               class="form-control" placeholder="[0,1023]"
+                               class="form-control" placeholder=""
                                ng-model="migrate_slot_id_end">
                         ] to Group
                         <input style="height: 28px; width: 140px;"

codis/cmd/proxy/main.go

@@ -113,6 +113,7 @@ Options:
 			log.PanicErrorf(err, "load config %s failed", s)
 		}
 	}
+	models.SetMaxSlotNum(config.MaxSlotNum)
 	if s, ok := utils.Argument(d, "--host-admin"); ok {
 		config.HostAdmin = s
 		log.Warnf("option --host-admin = %s", s)

codis/config/dashboard.toml

@@ -21,6 +21,9 @@ product_auth = ""
 # Set bind address for admin(rpc), tcp only.
 admin_addr = "0.0.0.0:18080"
 
+# Set slot num
+max_slot_num = 1024
+
 # Set arguments for data migration (only accept 'sync' & 'semi-async').
 migration_method = "semi-async"
 migration_parallel_slots = 100
@@ -33,7 +36,6 @@ migration_timeout = "30s"
 sentinel_check_server_state_interval = "5s"
 sentinel_check_master_failover_interval = "1s"
 sentinel_master_dead_check_times = 5
-
 sentinel_client_timeout = "10s"
 sentinel_quorum = 2
 sentinel_parallel_syncs = 1

codis/config/proxy.toml

@@ -71,11 +71,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 6
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/10.2.16.200_19000/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/10.2.16.200_19002/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/10.2.16.201_19000/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/10.2.16.201_19002/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/10.2.16.202_19000/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/10.2.16.202_19002/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/10.4.10.200_19000/proxy.toml

@@ -54,6 +54,9 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 

codis/deploy/root/opt/codis/etc/10.4.10.200_19002/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/10.4.10.201_19000/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/10.4.10.201_19002/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/127.0.0.1_19000/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/127.0.0.1_19001/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/127.0.0.1_19002/proxy.toml

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/templates/proxy.toml.template

@@ -54,11 +54,14 @@ backend_primary_only = false
 backend_primary_parallel = 1
 backend_replica_parallel = 1
 
+# Set slot num
+max_slot_num = 1024
+
 # Set backend tcp keepalive period. (0 to disable)
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 1
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/pkg/models/proxy.go

@@ -20,6 +20,8 @@ type Proxy struct {
 	Pwd string `json:"pwd"`
 	Sys string `json:"sys"`
 
+	MaxSlotNum int `toml:"max_slot_num" json:"max_slot_num"`
+
 	Hostname   string `json:"hostname"`
 	DataCenter string `json:"datacenter"`
 }