-
Notifications
You must be signed in to change notification settings - Fork 2.8k
/
clustermesh.go
232 lines (185 loc) · 6.88 KB
/
clustermesh.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium
package clustermesh
import (
"context"
"fmt"
"github.com/cilium/cilium/api/v1/models"
"github.com/cilium/cilium/pkg/allocator"
"github.com/cilium/cilium/pkg/clustermesh/internal"
"github.com/cilium/cilium/pkg/clustermesh/types"
"github.com/cilium/cilium/pkg/hive"
"github.com/cilium/cilium/pkg/hive/cell"
"github.com/cilium/cilium/pkg/ipcache"
"github.com/cilium/cilium/pkg/k8s"
"github.com/cilium/cilium/pkg/kvstore"
"github.com/cilium/cilium/pkg/kvstore/store"
"github.com/cilium/cilium/pkg/lock"
"github.com/cilium/cilium/pkg/logging"
"github.com/cilium/cilium/pkg/logging/logfields"
nodeTypes "github.com/cilium/cilium/pkg/node/types"
serviceStore "github.com/cilium/cilium/pkg/service/store"
)
const subsystem = "clustermesh"
var log = logging.DefaultLogger.WithField(logfields.LogSubsys, subsystem)
// Configuration is the configuration that must be provided to
// NewClusterMesh()
type Configuration struct {
cell.In
internal.Config
// ClusterIDName is the id/name of the local cluster. This is used for logging and metrics
types.ClusterIDName
// NodeKeyCreator is the function used to create node instances as
// nodes are being discovered in remote clusters
NodeKeyCreator store.KeyCreator
// ServiceMerger is the interface responsible to merge service and
// endpoints into an existing cache
ServiceMerger ServiceMerger
// NodeObserver reacts to node events.
NodeObserver store.Observer
// RemoteIdentityWatcher provides identities that have been allocated on a
// remote cluster.
RemoteIdentityWatcher RemoteIdentityWatcher
IPCache ipcache.IPCacher
// ClusterSizeDependantInterval allows to calculate intervals based on cluster size.
ClusterSizeDependantInterval kvstore.ClusterSizeDependantIntervalFunc
// ServiceIPGetter, if not nil, is used to create a custom dialer for service resolution.
ServiceIPGetter k8s.ServiceIPGetter
Metrics Metrics
InternalMetrics internal.Metrics
}
// RemoteIdentityWatcher is any type which provides identities that have been
// allocated on a remote cluster.
type RemoteIdentityWatcher interface {
// WatchRemoteIdentities returns a RemoteCache instance which can be later
// started to watch identities in another kvstore and sync them to the local
// identity cache. remoteName should be unique unless replacing an existing
// remote's backend. When cachedPrefix is set, identities are assumed to be
// stored under the "cilium/cache" prefix, and the watcher is adapted accordingly.
WatchRemoteIdentities(remoteName string, backend kvstore.BackendOperations, cachedPrefix bool) (*allocator.RemoteCache, error)
// RemoveRemoteIdentities removes any reference to a remote identity source,
// emitting a deletion event for all previously known identities.
RemoveRemoteIdentities(name string)
}
// ClusterMesh is a cache of multiple remote clusters
type ClusterMesh struct {
// conf is the configuration, it is immutable after NewClusterMesh()
conf Configuration
// internal implements the common logic to connect to remote clusters.
internal internal.ClusterMesh
usedIDs *ClusterMeshUsedIDs
// globalServices is a list of all global services. The datastructure
// is protected by its own mutex inside the structure.
globalServices *globalServiceCache
// nodeName is the name of the local node. This is used for logging and metrics
nodeName string
}
type ClusterMeshUsedIDs struct {
usedClusterIDs map[uint32]struct{}
usedClusterIDsMutex lock.Mutex
}
func newClusterMeshUsedIDs() *ClusterMeshUsedIDs {
return &ClusterMeshUsedIDs{
usedClusterIDs: make(map[uint32]struct{}),
}
}
func (cm *ClusterMeshUsedIDs) reserveClusterID(clusterID uint32) error {
cm.usedClusterIDsMutex.Lock()
defer cm.usedClusterIDsMutex.Unlock()
if _, ok := cm.usedClusterIDs[clusterID]; ok {
// ClusterID already used
return fmt.Errorf("clusterID %d is already used", clusterID)
}
cm.usedClusterIDs[clusterID] = struct{}{}
return nil
}
func (cm *ClusterMeshUsedIDs) releaseClusterID(clusterID uint32) {
cm.usedClusterIDsMutex.Lock()
defer cm.usedClusterIDsMutex.Unlock()
delete(cm.usedClusterIDs, clusterID)
}
// NewClusterMesh creates a new remote cluster cache based on the
// provided configuration
func NewClusterMesh(lifecycle hive.Lifecycle, c Configuration) *ClusterMesh {
if c.ClusterID == 0 || c.ClusterMeshConfig == "" {
return nil
}
nodeName := nodeTypes.GetName()
cm := &ClusterMesh{
conf: c,
usedIDs: newClusterMeshUsedIDs(),
nodeName: nodeName,
globalServices: newGlobalServiceCache(
c.Metrics.TotalGlobalServices.WithLabelValues(c.ClusterName, nodeName),
),
}
cm.internal = internal.NewClusterMesh(internal.Configuration{
Config: c.Config,
ClusterIDName: c.ClusterIDName,
ClusterSizeDependantInterval: c.ClusterSizeDependantInterval,
ServiceIPGetter: c.ServiceIPGetter,
NewRemoteCluster: cm.newRemoteCluster,
NodeName: nodeName,
Metrics: c.InternalMetrics,
})
lifecycle.Append(&cm.internal)
return cm
}
func (cm *ClusterMesh) newRemoteCluster(name string, status internal.StatusFunc) internal.RemoteCluster {
rc := &remoteCluster{
name: name,
mesh: cm,
usedIDs: cm.usedIDs,
status: status,
swg: lock.NewStoppableWaitGroup(),
}
rc.remoteNodes = store.NewRestartableWatchStore(
name,
cm.conf.NodeKeyCreator,
cm.conf.NodeObserver,
store.RWSWithEntriesMetric(cm.conf.Metrics.TotalNodes.WithLabelValues(cm.conf.ClusterName, cm.nodeName, rc.name)),
)
rc.remoteServices = store.NewRestartableWatchStore(
name,
func() store.Key { return new(serviceStore.ClusterService) },
&remoteServiceObserver{remoteCluster: rc, swg: rc.swg},
store.RWSWithOnSyncCallback(func(ctx context.Context) { rc.swg.Stop() }),
)
rc.ipCacheWatcher = ipcache.NewIPIdentityWatcher(name, cm.conf.IPCache)
return rc
}
// NumReadyClusters returns the number of remote clusters to which a connection
// has been established
func (cm *ClusterMesh) NumReadyClusters() int {
return cm.internal.NumReadyClusters()
}
// ClustersSynced returns after all clusters were synchronized with the bpf
// datapath.
func (cm *ClusterMesh) ClustersSynced(ctx context.Context) error {
swgs := make([]*lock.StoppableWaitGroup, 0)
cm.internal.ForEachRemoteCluster(func(rci internal.RemoteCluster) error {
rc := rci.(*remoteCluster)
swgs = append(swgs, rc.swg)
return nil
})
for _, swg := range swgs {
select {
case <-swg.WaitChannel():
case <-ctx.Done():
return ctx.Err()
}
}
return nil
}
// Status returns the status of the ClusterMesh subsystem
func (cm *ClusterMesh) Status() (status *models.ClusterMeshStatus) {
status = &models.ClusterMeshStatus{
NumGlobalServices: int64(cm.globalServices.size()),
}
cm.internal.ForEachRemoteCluster(func(rci internal.RemoteCluster) error {
rc := rci.(*remoteCluster)
status.Clusters = append(status.Clusters, rc.Status())
return nil
})
return
}