-
Notifications
You must be signed in to change notification settings - Fork 9
/
bootstrap.go
128 lines (118 loc) · 4.1 KB
/
bootstrap.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
// Copyright 2024 Matrix Origin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logset
import (
"fmt"
"github.com/go-errors/errors"
recon "github.com/matrixorigin/controller-runtime/pkg/reconciler"
"github.com/matrixorigin/matrixone-operator/api/core/v1alpha1"
"github.com/matrixorigin/matrixone-operator/pkg/controllers/common"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/json"
)
const (
bootstrapFile = "bootstrap.toml"
bootstrapAnnoKey = "logset.matrixorigin.io/bootstrap"
idRangeStart int = 131072
idRangeEnd int = 262144
)
type bootstrapReplica struct {
Ordinal int `json:"ordinal"`
ReplicaID int `json:"replicaId"`
}
// buildBootstrapConfig build the configmap that contains bootstrap information for log service
func buildBootstrapConfig(ctx *recon.Context[*v1alpha1.LogSet]) (*corev1.ConfigMap, error) {
ls := ctx.Obj
brs, err := bootstrap(ctx)
if err != nil {
return nil, err
}
m := map[string]interface{}{
"bootstrap-cluster": true,
"num-of-log-shards": ls.Spec.InitialConfig.LogShards,
"num-of-dn-shards": ls.Spec.InitialConfig.DNShards,
"num-of-log-shard-replicas": ls.Spec.InitialConfig.LogShardReplicas,
"init-hakeeper-members": encodeSeeds(brs),
}
if ls.Spec.InitialConfig.RestoreFrom != nil {
// if RestoreFrom is specified, read the backup data from the backup fileservice
m["restore"] = map[string]interface{}{
"file-path": fmt.Sprintf("%s://%s", common.BackupFileServiceName, *ls.Spec.InitialConfig.RestoreFrom),
}
}
t := v1alpha1.NewTomlConfig(map[string]interface{}{})
t.Set([]string{"logservice", "BootstrapConfig"}, m)
c, err := t.ToString()
if err != nil {
return nil, err
}
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: ls.Namespace,
Name: bootstrapConfigMapName(ls),
},
Data: map[string]string{
bootstrapFile: c,
},
}, nil
}
func bootstrap(ctx *recon.Context[*v1alpha1.LogSet]) ([]bootstrapReplica, error) {
var replicas []bootstrapReplica
previousDecision, hasBootstrapped := ctx.Obj.GetAnnotations()[bootstrapAnnoKey]
if hasBootstrapped {
if err := json.Unmarshal([]byte(previousDecision), &replicas); err != nil {
return nil, errors.WrapPrefix(err, "error deserialize boostrap replicas", 0)
}
return replicas, nil
}
// if the bootstrap decision has not yet been made,pick a bootstrap decision
n := *ctx.Obj.Spec.InitialConfig.LogShardReplicas
// pick first N pods as initial HAKeeperReplicas
for i := 0; i < n; i++ {
rid := idRangeStart + i
if rid > idRangeEnd {
return nil, errors.Errorf("ReplicaID %d exceed range, max allowed: %d", rid, idRangeEnd)
}
replicas = append(replicas, bootstrapReplica{
Ordinal: i,
ReplicaID: rid,
})
}
serialized, err := json.Marshal(replicas)
if err != nil {
return nil, errors.WrapPrefix(err, "error serialize bootstrap replicas", 0)
}
if ctx.Obj.Annotations == nil {
ctx.Obj.Annotations = map[string]string{}
}
ctx.Obj.Annotations[bootstrapAnnoKey] = string(serialized)
return replicas, ctx.Update(ctx.Obj)
}
// encodeSeeds encode the bootstrap replicas decision to the configuration format
// accepted by logservice
func encodeSeeds(brs []bootstrapReplica) []string {
var seeds []string
for _, r := range brs {
seeds = append(seeds, fmt.Sprintf("%d:%s", r.ReplicaID, encodeOrdinal(r.Ordinal)))
}
return seeds
}
// encodeOrdinal encode the pod ordinal to UUID
func encodeOrdinal(ordinal int) string {
return fmt.Sprintf("00000000-0000-0000-0000-%012x", ordinal)
}
func bootstrapConfigMapName(ls *v1alpha1.LogSet) string {
return resourceName(ls) + "-bootstrap"
}