-
Notifications
You must be signed in to change notification settings - Fork 332
/
node_annotations.go
1119 lines (982 loc) · 40.3 KB
/
node_annotations.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package util
import (
"encoding/json"
"fmt"
"math"
"net"
"net/netip"
"strconv"
"github.com/gaissmai/cidrtree"
corev1 "k8s.io/api/core/v1"
kapi "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
)
// This handles the annotations used by the node to pass information about its local
// network configuration to the master:
//
// annotations:
// k8s.ovn.org/l3-gateway-config: |
// {
// "default": {
// "mode": "local",
// "interface-id": "br-local_ip-10-0-129-64.us-east-2.compute.internal",
// "mac-address": "f2:20:a0:3c:26:4c",
// "ip-addresses": ["169.254.33.2/24"],
// "next-hops": ["169.254.33.1"],
// "node-port-enable": "true",
// "vlan-id": "0"
//
// # backward-compat
// "ip-address": "169.254.33.2/24",
// "next-hop": "169.254.33.1",
// }
// }
// k8s.ovn.org/node-chassis-id: b1f96182-2bdd-42b6-88f9-9a1fc1c85ece
// k8s.ovn.org/node-mgmt-port-mac-address: fa:f1:27:f5:54:69
//
// The "ip_address" and "next_hop" fields are deprecated and will eventually go away.
// (And they are not output when "ip_addresses" or "next_hops" contains multiple
// values.)
const (
// OvnNodeL3GatewayConfig is the constant string representing the l3 gateway annotation key
OvnNodeL3GatewayConfig = "k8s.ovn.org/l3-gateway-config"
// OvnNodeGatewayMtuSupport determines if option:gateway_mtu shall be set for GR router ports.
OvnNodeGatewayMtuSupport = "k8s.ovn.org/gateway-mtu-support"
// OvnDefaultNetworkGateway captures L3 gateway config for default OVN network interface
ovnDefaultNetworkGateway = "default"
// OvnNodeManagementPort is the constant string representing the annotation key
OvnNodeManagementPort = "k8s.ovn.org/node-mgmt-port"
// OvnNodeManagementPortMacAddress is the constant string representing the annotation key
OvnNodeManagementPortMacAddress = "k8s.ovn.org/node-mgmt-port-mac-address"
// OvnNodeChassisID is the systemID of the node needed for creating L3 gateway
OvnNodeChassisID = "k8s.ovn.org/node-chassis-id"
// OvnNodeIfAddr is the CIDR form representation of primary network interface's attached IP address (i.e: 192.168.126.31/24 or 0:0:0:0:0:feff:c0a8:8e0c/64)
OvnNodeIfAddr = "k8s.ovn.org/node-primary-ifaddr"
// ovnNodeGRLRPAddr is the CIDR form representation of Gate Router LRP IP address to join switch (i.e: 100.64.0.5/24)
ovnNodeGRLRPAddr = "k8s.ovn.org/node-gateway-router-lrp-ifaddr"
// OvnNodeEgressLabel is a user assigned node label indicating to ovn-kubernetes that the node is to be used for egress IP assignment
ovnNodeEgressLabel = "k8s.ovn.org/egress-assignable"
// OVNNodeHostCIDRs is used to track the different host IP addresses and subnet masks on the node
OVNNodeHostCIDRs = "k8s.ovn.org/host-cidrs"
// OVNNodeSecondaryHostEgressIPs contains EgressIP addresses that aren't managed by OVN. The EIP addresses are assigned to
// standard linux interfaces and not interfaces of type OVS.
OVNNodeSecondaryHostEgressIPs = "k8s.ovn.org/secondary-host-egress-ips"
// egressIPConfigAnnotationKey is used to indicate the cloud subnet and
// capacity for each node. It is set by
// openshift/cloud-network-config-controller
cloudEgressIPConfigAnnotationKey = "cloud.network.openshift.io/egress-ipconfig"
// OvnNodeZoneName is the zone to which the node belongs to. It is set by ovnkube-node.
// ovnkube-node gets the node's zone from the OVN Southbound database.
OvnNodeZoneName = "k8s.ovn.org/zone-name"
/** HACK BEGIN **/
// TODO(tssurya): Remove this annotation a few months from now (when one or two release jump
// upgrades are done). This has been added only to minimize disruption for upgrades when
// moving to interconnect=true.
// We want the legacy ovnkube-master to wait for remote ovnkube-node to
// signal it using "k8s.ovn.org/remote-zone-migrated" annotation before
// considering a node as remote when we upgrade from "global" (1 zone IC)
// zone to multi-zone. This is so that network disruption for the existing workloads
// is negligible and until the point where ovnkube-node flips the switch to connect
// to the new SBDB, it would continue talking to the legacy RAFT ovnkube-sbdb to ensure
// OVN/OVS flows are intact.
// OvnNodeMigratedZoneName is the zone to which the node belongs to. It is set by ovnkube-node.
// ovnkube-node gets the node's zone from the OVN Southbound database.
OvnNodeMigratedZoneName = "k8s.ovn.org/remote-zone-migrated"
/** HACK END **/
// ovnTransitSwitchPortAddr is the annotation to store the node Transit switch port ips.
// It is set by cluster manager.
ovnTransitSwitchPortAddr = "k8s.ovn.org/node-transit-switch-port-ifaddr"
// ovnNodeID is the id (of type integer) of a node. It is set by cluster-manager.
ovnNodeID = "k8s.ovn.org/node-id"
// InvalidNodeID indicates an invalid node id
InvalidNodeID = -1
// ovnNetworkIDs is the constant string representing the ids allocated for the
// default network and other layer3 secondary networks by cluster manager.
ovnNetworkIDs = "k8s.ovn.org/network-ids"
// invalidNetworkID signifies its an invalid network id
InvalidNetworkID = -1
)
type L3GatewayConfig struct {
Mode config.GatewayMode
ChassisID string
InterfaceID string
MACAddress net.HardwareAddr
IPAddresses []*net.IPNet
EgressGWInterfaceID string
EgressGWMACAddress net.HardwareAddr
EgressGWIPAddresses []*net.IPNet
NextHops []net.IP
NodePortEnable bool
VLANID *uint
}
type l3GatewayConfigJSON struct {
Mode config.GatewayMode `json:"mode"`
InterfaceID string `json:"interface-id,omitempty"`
MACAddress string `json:"mac-address,omitempty"`
IPAddresses []string `json:"ip-addresses,omitempty"`
IPAddress string `json:"ip-address,omitempty"`
EgressGWInterfaceID string `json:"exgw-interface-id,omitempty"`
EgressGWMACAddress string `json:"exgw-mac-address,omitempty"`
EgressGWIPAddresses []string `json:"exgw-ip-addresses,omitempty"`
EgressGWIPAddress string `json:"exgw-ip-address,omitempty"`
NextHops []string `json:"next-hops,omitempty"`
NextHop string `json:"next-hop,omitempty"`
NodePortEnable string `json:"node-port-enable,omitempty"`
VLANID string `json:"vlan-id,omitempty"`
}
func (cfg *L3GatewayConfig) MarshalJSON() ([]byte, error) {
cfgjson := l3GatewayConfigJSON{
Mode: cfg.Mode,
}
if cfg.Mode == config.GatewayModeDisabled {
return json.Marshal(&cfgjson)
}
cfgjson.InterfaceID = cfg.InterfaceID
cfgjson.MACAddress = cfg.MACAddress.String()
cfgjson.EgressGWInterfaceID = cfg.EgressGWInterfaceID
cfgjson.EgressGWMACAddress = cfg.EgressGWMACAddress.String()
cfgjson.NodePortEnable = fmt.Sprintf("%t", cfg.NodePortEnable)
if cfg.VLANID != nil {
cfgjson.VLANID = fmt.Sprintf("%d", *cfg.VLANID)
}
cfgjson.IPAddresses = make([]string, len(cfg.IPAddresses))
for i, ip := range cfg.IPAddresses {
cfgjson.IPAddresses[i] = ip.String()
}
if len(cfgjson.IPAddresses) == 1 {
cfgjson.IPAddress = cfgjson.IPAddresses[0]
}
cfgjson.EgressGWIPAddresses = make([]string, len(cfg.EgressGWIPAddresses))
for i, ip := range cfg.EgressGWIPAddresses {
cfgjson.EgressGWIPAddresses[i] = ip.String()
}
if len(cfgjson.EgressGWIPAddresses) == 1 {
cfgjson.EgressGWIPAddress = cfgjson.EgressGWIPAddresses[0]
}
cfgjson.NextHops = make([]string, len(cfg.NextHops))
for i, nh := range cfg.NextHops {
cfgjson.NextHops[i] = nh.String()
}
if len(cfgjson.NextHops) == 1 {
cfgjson.NextHop = cfgjson.NextHops[0]
}
return json.Marshal(&cfgjson)
}
func (cfg *L3GatewayConfig) UnmarshalJSON(bytes []byte) error {
cfgjson := l3GatewayConfigJSON{}
if err := json.Unmarshal(bytes, &cfgjson); err != nil {
return err
}
cfg.Mode = cfgjson.Mode
if cfg.Mode == config.GatewayModeDisabled {
return nil
} else if cfg.Mode != config.GatewayModeShared && cfg.Mode != config.GatewayModeLocal {
return fmt.Errorf("bad 'mode' value %q", cfgjson.Mode)
}
cfg.InterfaceID = cfgjson.InterfaceID
cfg.EgressGWInterfaceID = cfgjson.EgressGWInterfaceID
cfg.NodePortEnable = cfgjson.NodePortEnable == "true"
if cfgjson.VLANID != "" {
vlanID64, err := strconv.ParseUint(cfgjson.VLANID, 10, 0)
if err != nil {
return fmt.Errorf("bad 'vlan-id' value %q: %v", cfgjson.VLANID, err)
}
// VLANID is used for specifying TagRequest on the logical switch port
// connected to the external logical switch, NB DB specifies a maximum
// value on the TagRequest to 4095, hence validate this:
//https://github.com/ovn-org/ovn/blob/4b97d6fa88e36206213b9fdc8e1e1a9016cfc736/ovn-nb.ovsschema#L94-L98
if vlanID64 > 4095 {
return fmt.Errorf("vlan-id surpasses maximum supported value")
}
vlanID := uint(vlanID64)
cfg.VLANID = &vlanID
}
var err error
cfg.MACAddress, err = net.ParseMAC(cfgjson.MACAddress)
if err != nil {
return fmt.Errorf("bad 'mac-address' value %q: %v", cfgjson.MACAddress, err)
}
if cfg.EgressGWInterfaceID != "" {
cfg.EgressGWMACAddress, err = net.ParseMAC(cfgjson.EgressGWMACAddress)
if err != nil {
return fmt.Errorf("bad 'egress mac-address' value %q: %v", cfgjson.EgressGWMACAddress, err)
}
if len(cfgjson.EgressGWIPAddresses) == 0 {
cfg.EgressGWIPAddresses = make([]*net.IPNet, 1)
ip, ipnet, err := net.ParseCIDR(cfgjson.EgressGWIPAddress)
if err != nil {
return fmt.Errorf("bad 'ip-address' value %q: %v", cfgjson.EgressGWIPAddress, err)
}
cfg.EgressGWIPAddresses[0] = &net.IPNet{IP: ip, Mask: ipnet.Mask}
} else {
cfg.EgressGWIPAddresses = make([]*net.IPNet, len(cfgjson.EgressGWIPAddresses))
for i, ipStr := range cfgjson.EgressGWIPAddresses {
ip, ipnet, err := net.ParseCIDR(ipStr)
if err != nil {
return fmt.Errorf("bad 'ip-addresses' value %q: %v", ipStr, err)
}
cfg.EgressGWIPAddresses[i] = &net.IPNet{IP: ip, Mask: ipnet.Mask}
}
}
}
if len(cfgjson.IPAddresses) == 0 {
cfg.IPAddresses = make([]*net.IPNet, 1)
ip, ipnet, err := net.ParseCIDR(cfgjson.IPAddress)
if err != nil {
return fmt.Errorf("bad 'ip-address' value %q: %v", cfgjson.IPAddress, err)
}
cfg.IPAddresses[0] = &net.IPNet{IP: ip, Mask: ipnet.Mask}
} else {
cfg.IPAddresses = make([]*net.IPNet, len(cfgjson.IPAddresses))
for i, ipStr := range cfgjson.IPAddresses {
ip, ipnet, err := net.ParseCIDR(ipStr)
if err != nil {
return fmt.Errorf("bad 'ip-addresses' value %q: %v", ipStr, err)
}
cfg.IPAddresses[i] = &net.IPNet{IP: ip, Mask: ipnet.Mask}
}
}
cfg.NextHops = make([]net.IP, len(cfgjson.NextHops))
for i, nextHopStr := range cfgjson.NextHops {
cfg.NextHops[i] = net.ParseIP(nextHopStr)
if cfg.NextHops[i] == nil {
return fmt.Errorf("bad 'next-hops' value %q", nextHopStr)
}
}
return nil
}
func SetL3GatewayConfig(nodeAnnotator kube.Annotator, cfg *L3GatewayConfig) error {
gatewayAnnotation := map[string]*L3GatewayConfig{ovnDefaultNetworkGateway: cfg}
if err := nodeAnnotator.Set(OvnNodeL3GatewayConfig, gatewayAnnotation); err != nil {
return err
}
if cfg.ChassisID != "" {
if err := nodeAnnotator.Set(OvnNodeChassisID, cfg.ChassisID); err != nil {
return err
}
}
return nil
}
// SetGatewayMTUSupport sets annotation "k8s.ovn.org/gateway-mtu-support" to "false" or removes the annotation from
// this node.
func SetGatewayMTUSupport(nodeAnnotator kube.Annotator, set bool) error {
if set {
nodeAnnotator.Delete(OvnNodeGatewayMtuSupport)
return nil
}
return nodeAnnotator.Set(OvnNodeGatewayMtuSupport, "false")
}
// ParseNodeGatewayMTUSupport parses annotation "k8s.ovn.org/gateway-mtu-support". The default behavior should be true,
// therefore only an explicit string of "false" will make this function return false.
func ParseNodeGatewayMTUSupport(node *kapi.Node) bool {
return node.Annotations[OvnNodeGatewayMtuSupport] != "false"
}
// ParseNodeL3GatewayAnnotation returns the parsed l3-gateway-config annotation
func ParseNodeL3GatewayAnnotation(node *kapi.Node) (*L3GatewayConfig, error) {
l3GatewayAnnotation, ok := node.Annotations[OvnNodeL3GatewayConfig]
if !ok {
return nil, newAnnotationNotSetError("%s annotation not found for node %q", OvnNodeL3GatewayConfig, node.Name)
}
var cfgs map[string]*L3GatewayConfig
if err := json.Unmarshal([]byte(l3GatewayAnnotation), &cfgs); err != nil {
return nil, fmt.Errorf("failed to unmarshal l3 gateway config annotation %s for node %q: %v", l3GatewayAnnotation, node.Name, err)
}
cfg, ok := cfgs[ovnDefaultNetworkGateway]
if !ok {
return nil, fmt.Errorf("%s annotation for %s network not found", OvnNodeL3GatewayConfig, ovnDefaultNetworkGateway)
}
if cfg.Mode != config.GatewayModeDisabled {
cfg.ChassisID, ok = node.Annotations[OvnNodeChassisID]
if !ok {
return nil, fmt.Errorf("%s annotation not found", OvnNodeChassisID)
}
}
return cfg, nil
}
func NodeL3GatewayAnnotationChanged(oldNode, newNode *kapi.Node) bool {
return oldNode.Annotations[OvnNodeL3GatewayConfig] != newNode.Annotations[OvnNodeL3GatewayConfig]
}
// ParseNodeChassisIDAnnotation returns the node's ovnNodeChassisID annotation
func ParseNodeChassisIDAnnotation(node *kapi.Node) (string, error) {
chassisID, ok := node.Annotations[OvnNodeChassisID]
if !ok {
return "", newAnnotationNotSetError("%s annotation not found for node %s", OvnNodeChassisID, node.Name)
}
return chassisID, nil
}
func NodeChassisIDAnnotationChanged(oldNode, newNode *kapi.Node) bool {
return oldNode.Annotations[OvnNodeChassisID] != newNode.Annotations[OvnNodeChassisID]
}
type ManagementPortDetails struct {
PfId int `json:"PfId"`
FuncId int `json:"FuncId"`
}
func SetNodeManagementPortAnnotation(nodeAnnotator kube.Annotator, PfId int, FuncId int) error {
mgmtPortDetails := ManagementPortDetails{
PfId: PfId,
FuncId: FuncId,
}
bytes, err := json.Marshal(mgmtPortDetails)
if err != nil {
return fmt.Errorf("failed to marshal mgmtPortDetails with PfId '%v', FuncId '%v'", PfId, FuncId)
}
return nodeAnnotator.Set(OvnNodeManagementPort, string(bytes))
}
// ParseNodeManagementPortAnnotation returns the parsed host addresses living on a node
func ParseNodeManagementPortAnnotation(node *kapi.Node) (int, int, error) {
mgmtPortAnnotation, ok := node.Annotations[OvnNodeManagementPort]
if !ok {
return -1, -1, newAnnotationNotSetError("%s annotation not found for node %q", OvnNodeManagementPort, node.Name)
}
cfg := ManagementPortDetails{}
if err := json.Unmarshal([]byte(mgmtPortAnnotation), &cfg); err != nil {
return -1, -1, fmt.Errorf("failed to unmarshal management port annotation %s for node %q: %v",
mgmtPortAnnotation, node.Name, err)
}
return cfg.PfId, cfg.FuncId, nil
}
func SetNodeManagementPortMACAddress(nodeAnnotator kube.Annotator, macAddress net.HardwareAddr) error {
return nodeAnnotator.Set(OvnNodeManagementPortMacAddress, macAddress.String())
}
func ParseNodeManagementPortMACAddress(node *kapi.Node) (net.HardwareAddr, error) {
macAddress, ok := node.Annotations[OvnNodeManagementPortMacAddress]
if !ok {
return nil, newAnnotationNotSetError("macAddress annotation not found for node %q ", node.Name)
}
return net.ParseMAC(macAddress)
}
type primaryIfAddrAnnotation struct {
IPv4 string `json:"ipv4,omitempty"`
IPv6 string `json:"ipv6,omitempty"`
}
// SetNodePrimaryIfAddr sets the IPv4 / IPv6 values of the node's primary network interface
func SetNodePrimaryIfAddrs(nodeAnnotator kube.Annotator, ifAddrs []*net.IPNet) (err error) {
nodeIPNetv4, _ := MatchFirstIPNetFamily(false, ifAddrs)
nodeIPNetv6, _ := MatchFirstIPNetFamily(true, ifAddrs)
primaryIfAddrAnnotation := primaryIfAddrAnnotation{}
if nodeIPNetv4 != nil {
primaryIfAddrAnnotation.IPv4 = nodeIPNetv4.String()
}
if nodeIPNetv6 != nil {
primaryIfAddrAnnotation.IPv6 = nodeIPNetv6.String()
}
return nodeAnnotator.Set(OvnNodeIfAddr, primaryIfAddrAnnotation)
}
// createPrimaryIfAddrAnnotation marshals the IPv4 / IPv6 values in the
// primaryIfAddrAnnotation format and stores it in the nodeAnnotation
// map with the provided 'annotationName' as key
func createPrimaryIfAddrAnnotation(annotationName string, nodeAnnotation map[string]interface{}, nodeIPNetv4,
nodeIPNetv6 *net.IPNet) (map[string]interface{}, error) {
if nodeAnnotation == nil {
nodeAnnotation = make(map[string]interface{})
}
primaryIfAddrAnnotation := primaryIfAddrAnnotation{}
if nodeIPNetv4 != nil {
primaryIfAddrAnnotation.IPv4 = nodeIPNetv4.String()
}
if nodeIPNetv6 != nil {
primaryIfAddrAnnotation.IPv6 = nodeIPNetv6.String()
}
bytes, err := json.Marshal(primaryIfAddrAnnotation)
if err != nil {
return nil, err
}
nodeAnnotation[annotationName] = string(bytes)
return nodeAnnotation, nil
}
// CreateNodeGatewayRouterLRPAddrAnnotation sets the IPv4 / IPv6 values of the node's Gateway Router LRP to join switch.
func CreateNodeGatewayRouterLRPAddrAnnotation(nodeAnnotation map[string]interface{}, nodeIPNetv4,
nodeIPNetv6 *net.IPNet) (map[string]interface{}, error) {
return createPrimaryIfAddrAnnotation(ovnNodeGRLRPAddr, nodeAnnotation, nodeIPNetv4, nodeIPNetv6)
}
func NodeGatewayRouterLRPAddrAnnotationChanged(oldNode, newNode *corev1.Node) bool {
return oldNode.Annotations[ovnNodeGRLRPAddr] != newNode.Annotations[ovnNodeGRLRPAddr]
}
// CreateNodeTransitSwitchPortAddrAnnotation creates the node annotation for the node's Transit switch port addresses.
func CreateNodeTransitSwitchPortAddrAnnotation(nodeAnnotation map[string]interface{}, nodeIPNetv4,
nodeIPNetv6 *net.IPNet) (map[string]interface{}, error) {
return createPrimaryIfAddrAnnotation(ovnTransitSwitchPortAddr, nodeAnnotation, nodeIPNetv4, nodeIPNetv6)
}
func NodeTransitSwitchPortAddrAnnotationChanged(oldNode, newNode *corev1.Node) bool {
return oldNode.Annotations[ovnTransitSwitchPortAddr] != newNode.Annotations[ovnTransitSwitchPortAddr]
}
const UnlimitedNodeCapacity = math.MaxInt32
type ifAddr struct {
IPv4 string `json:"ipv4,omitempty"`
IPv6 string `json:"ipv6,omitempty"`
}
type Capacity struct {
IPv4 int `json:"ipv4,omitempty"`
IPv6 int `json:"ipv6,omitempty"`
IP int `json:"ip,omitempty"`
}
type nodeEgressIPConfiguration struct {
Interface string `json:"interface"`
IFAddr ifAddr `json:"ifaddr"`
Capacity Capacity `json:"capacity"`
}
type ParsedIFAddr struct {
IP net.IP
Net *net.IPNet
}
type ParsedNodeEgressIPConfiguration struct {
V4 ParsedIFAddr
V6 ParsedIFAddr
Capacity Capacity
}
func getNodeIfAddrAnnotation(node *kapi.Node) (*primaryIfAddrAnnotation, error) {
nodeIfAddrAnnotation, ok := node.Annotations[OvnNodeIfAddr]
if !ok {
return nil, newAnnotationNotSetError("%s annotation not found for node %q", OvnNodeIfAddr, node.Name)
}
nodeIfAddr := &primaryIfAddrAnnotation{}
if err := json.Unmarshal([]byte(nodeIfAddrAnnotation), nodeIfAddr); err != nil {
return nil, fmt.Errorf("failed to unmarshal annotation: %s for node %q, err: %v", OvnNodeIfAddr, node.Name, err)
}
if nodeIfAddr.IPv4 == "" && nodeIfAddr.IPv6 == "" {
return nil, fmt.Errorf("node: %q does not have any IP information set", node.Name)
}
return nodeIfAddr, nil
}
// ParseNodePrimaryIfAddr returns the IPv4 / IPv6 values for the node's primary network interface
func ParseNodePrimaryIfAddr(node *kapi.Node) (*ParsedNodeEgressIPConfiguration, error) {
nodeIfAddr, err := getNodeIfAddrAnnotation(node)
if err != nil {
return nil, err
}
nodeEgressIPConfig := nodeEgressIPConfiguration{
IFAddr: ifAddr(*nodeIfAddr),
Capacity: Capacity{
IP: UnlimitedNodeCapacity,
IPv4: UnlimitedNodeCapacity,
IPv6: UnlimitedNodeCapacity,
},
}
parsedEgressIPConfig, err := parseNodeEgressIPConfig(&nodeEgressIPConfig)
if err != nil {
return nil, err
}
return parsedEgressIPConfig, nil
}
// ParseNodeGatewayRouterLRPAddr returns the IPv4 / IPv6 values for the node's gateway router
func ParseNodeGatewayRouterLRPAddr(node *kapi.Node) (net.IP, error) {
nodeIfAddrAnnotation, ok := node.Annotations[ovnNodeGRLRPAddr]
if !ok {
return nil, newAnnotationNotSetError("%s annotation not found for node %q", ovnNodeGRLRPAddr, node.Name)
}
nodeIfAddr := primaryIfAddrAnnotation{}
if err := json.Unmarshal([]byte(nodeIfAddrAnnotation), &nodeIfAddr); err != nil {
return nil, fmt.Errorf("failed to unmarshal annotation: %s for node %q, err: %v", ovnNodeGRLRPAddr, node.Name, err)
}
if nodeIfAddr.IPv4 == "" && nodeIfAddr.IPv6 == "" {
return nil, fmt.Errorf("node: %q does not have any IP information set", node.Name)
}
ip, _, err := net.ParseCIDR(nodeIfAddr.IPv4)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation: %s for node %q, err: %v", ovnNodeGRLRPAddr, node.Name, err)
}
return ip, nil
}
// parsePrimaryIfAddrAnnotation unmarshals the IPv4 / IPv6 values in the
// primaryIfAddrAnnotation format from the nodeAnnotation map with the
// provided 'annotationName' as key and returns the addresses.
func parsePrimaryIfAddrAnnotation(node *kapi.Node, annotationName string) ([]*net.IPNet, error) {
nodeIfAddrAnnotation, ok := node.Annotations[annotationName]
if !ok {
return nil, newAnnotationNotSetError("%s annotation not found for node %q", annotationName, node.Name)
}
nodeIfAddr := primaryIfAddrAnnotation{}
if err := json.Unmarshal([]byte(nodeIfAddrAnnotation), &nodeIfAddr); err != nil {
return nil, fmt.Errorf("failed to unmarshal annotation: %s for node %q, err: %w", annotationName, node.Name, err)
}
if nodeIfAddr.IPv4 == "" && nodeIfAddr.IPv6 == "" {
return nil, fmt.Errorf("node: %q does not have any IP information set", node.Name)
}
var ipAddrs []*net.IPNet
if nodeIfAddr.IPv4 != "" {
ip, ipNet, err := net.ParseCIDR(nodeIfAddr.IPv4)
if err != nil {
return nil, fmt.Errorf("failed to parse IPv4 address %s from annotation: %s for node %q, err: %w", nodeIfAddr.IPv4, annotationName, node.Name, err)
}
ipAddrs = append(ipAddrs, &net.IPNet{IP: ip, Mask: ipNet.Mask})
}
if nodeIfAddr.IPv6 != "" {
ip, ipNet, err := net.ParseCIDR(nodeIfAddr.IPv6)
if err != nil {
return nil, fmt.Errorf("failed to parse IPv6 address %s from annotation: %s for node %q, err: %w", nodeIfAddr.IPv6, annotationName, node.Name, err)
}
ipAddrs = append(ipAddrs, &net.IPNet{IP: ip, Mask: ipNet.Mask})
}
return ipAddrs, nil
}
// ParseNodeGatewayRouterLRPAddrs returns the IPv4 and/or IPv6 addresses for the node's gateway router port
// stored in the 'ovnNodeGRLRPAddr' annotation
func ParseNodeGatewayRouterLRPAddrs(node *kapi.Node) ([]*net.IPNet, error) {
return parsePrimaryIfAddrAnnotation(node, ovnNodeGRLRPAddr)
}
// ParseNodeTransitSwitchPortAddrs returns the IPv4 and/or IPv6 addresses for the node's transit switch port
// stored in the 'ovnTransitSwitchPortAddr' annotation
func ParseNodeTransitSwitchPortAddrs(node *kapi.Node) ([]*net.IPNet, error) {
return parsePrimaryIfAddrAnnotation(node, ovnTransitSwitchPortAddr)
}
// GetNodeEIPConfig attempts to generate EIP configuration from a nodes annotations.
// If the platform is running in the cloud, retrieve config info from node obj annotation added by Cloud Network Config
// Controller (CNCC). If not on a cloud platform (i.e. baremetal), retrieve from the node obj primary interface annotation.
func GetNodeEIPConfig(node *kapi.Node) (*ParsedNodeEgressIPConfiguration, error) {
var parsedEgressIPConfig *ParsedNodeEgressIPConfiguration
var err error
if PlatformTypeIsEgressIPCloudProvider() {
parsedEgressIPConfig, err = ParseCloudEgressIPConfig(node)
} else {
parsedEgressIPConfig, err = ParseNodePrimaryIfAddr(node)
}
if err != nil {
return nil, fmt.Errorf("unable to generate egress IP config for node %s: %w", node.Name, err)
}
return parsedEgressIPConfig, nil
}
// ParseCloudEgressIPConfig returns the cloud's information concerning the node's primary network interface
func ParseCloudEgressIPConfig(node *kapi.Node) (*ParsedNodeEgressIPConfiguration, error) {
egressIPConfigAnnotation, ok := node.Annotations[cloudEgressIPConfigAnnotationKey]
if !ok {
return nil, newAnnotationNotSetError("%s annotation not found for node %q", cloudEgressIPConfigAnnotationKey, node.Name)
}
nodeEgressIPConfig := []nodeEgressIPConfiguration{
{
Capacity: Capacity{
IP: UnlimitedNodeCapacity,
IPv4: UnlimitedNodeCapacity,
IPv6: UnlimitedNodeCapacity,
},
},
}
if err := json.Unmarshal([]byte(egressIPConfigAnnotation), &nodeEgressIPConfig); err != nil {
return nil, fmt.Errorf("failed to unmarshal annotation: %s for node %q, err: %v", OvnNodeIfAddr, node.Name, err)
}
if len(nodeEgressIPConfig) == 0 {
return nil, fmt.Errorf("empty annotation: %s for node: %q", cloudEgressIPConfigAnnotationKey, node.Name)
}
parsedEgressIPConfig, err := parseNodeEgressIPConfig(&nodeEgressIPConfig[0])
if err != nil {
return nil, err
}
// ParsedNodeEgressIPConfiguration.V[4|6].IP is used to verify if an egress IP matches node IP to disable its creation
// use node IP instead of the value assigned from cloud egress CIDR config
nodeIfAddr, err := getNodeIfAddrAnnotation(node)
if err != nil {
return nil, err
}
if nodeIfAddr.IPv4 != "" {
ipv4, _, err := net.ParseCIDR(nodeIfAddr.IPv4)
if err != nil {
return nil, err
}
parsedEgressIPConfig.V4.IP = ipv4
}
if nodeIfAddr.IPv6 != "" {
ipv6, _, err := net.ParseCIDR(nodeIfAddr.IPv6)
if err != nil {
return nil, err
}
parsedEgressIPConfig.V6.IP = ipv6
}
return parsedEgressIPConfig, nil
}
func parseNodeEgressIPConfig(egressIPConfig *nodeEgressIPConfiguration) (*ParsedNodeEgressIPConfiguration, error) {
parsedEgressIPConfig := &ParsedNodeEgressIPConfiguration{
Capacity: egressIPConfig.Capacity,
}
if egressIPConfig.IFAddr.IPv4 != "" {
ipv4, v4Subnet, err := net.ParseCIDR(egressIPConfig.IFAddr.IPv4)
if err != nil {
return nil, err
}
parsedEgressIPConfig.V4 = ParsedIFAddr{
IP: ipv4,
Net: v4Subnet,
}
}
if egressIPConfig.IFAddr.IPv6 != "" {
ipv6, v6Subnet, err := net.ParseCIDR(egressIPConfig.IFAddr.IPv6)
if err != nil {
return nil, err
}
parsedEgressIPConfig.V6 = ParsedIFAddr{
IP: ipv6,
Net: v6Subnet,
}
}
return parsedEgressIPConfig, nil
}
// GetNodeEgressLabel returns label annotation needed for marking nodes as egress assignable
func GetNodeEgressLabel() string {
return ovnNodeEgressLabel
}
func SetNodeHostCIDRs(nodeAnnotator kube.Annotator, cidrs sets.Set[string]) error {
return nodeAnnotator.Set(OVNNodeHostCIDRs, sets.List(cidrs))
}
func NodeHostCIDRsAnnotationChanged(oldNode, newNode *v1.Node) bool {
return oldNode.Annotations[OVNNodeHostCIDRs] != newNode.Annotations[OVNNodeHostCIDRs]
}
// ParseNodeHostCIDRs returns the parsed host CIDRS living on a node
func ParseNodeHostCIDRs(node *kapi.Node) (sets.Set[string], error) {
addrAnnotation, ok := node.Annotations[OVNNodeHostCIDRs]
if !ok {
return nil, newAnnotationNotSetError("%s annotation not found for node %q", OVNNodeHostCIDRs, node.Name)
}
var cfg []string
if err := json.Unmarshal([]byte(addrAnnotation), &cfg); err != nil {
return nil, fmt.Errorf("failed to unmarshal host cidrs annotation %s for node %q: %v",
addrAnnotation, node.Name, err)
}
return sets.New(cfg...), nil
}
// ParseNodeHostCIDRsDropNetMask returns the parsed host IP addresses found on a node's host CIDR annotation. Removes the mask.
func ParseNodeHostCIDRsDropNetMask(node *kapi.Node) (sets.Set[string], error) {
addrAnnotation, ok := node.Annotations[OVNNodeHostCIDRs]
if !ok {
return nil, newAnnotationNotSetError("%s annotation not found for node %q", OVNNodeHostCIDRs, node.Name)
}
var cfg []string
if err := json.Unmarshal([]byte(addrAnnotation), &cfg); err != nil {
return nil, fmt.Errorf("failed to unmarshal host cidrs annotation %s for node %q: %v",
addrAnnotation, node.Name, err)
}
for i, cidr := range cfg {
ip, _, err := net.ParseCIDR(cidr)
if err != nil || ip == nil {
return nil, fmt.Errorf("failed to parse node host cidr: %v", err)
}
cfg[i] = ip.String()
}
return sets.New(cfg...), nil
}
// GetNodeHostAddrs returns the parsed Host CIDR annotation of the given node
// as an array of strings. If the annotation is not set, then we return empty list.
func GetNodeHostAddrs(node *kapi.Node) ([]string, error) {
hostAddresses, err := ParseNodeHostCIDRsDropNetMask(node)
if err != nil && !IsAnnotationNotSetError(err) {
return nil, fmt.Errorf("failed to get node host CIDRs for %s: %s", node.Name, err.Error())
}
return hostAddresses.UnsortedList(), nil
}
func ParseNodeHostCIDRsExcludeOVNNetworks(node *kapi.Node) ([]string, error) {
networks, err := ParseNodeHostCIDRsList(node)
if err != nil {
return nil, err
}
ovnNetworks, err := getNodeIfAddrAnnotation(node)
if err != nil {
return nil, err
}
if ovnNetworks.IPv4 != "" {
networks = RemoveItemFromSliceUnstable(networks, ovnNetworks.IPv4)
}
if ovnNetworks.IPv6 != "" {
networks = RemoveItemFromSliceUnstable(networks, ovnNetworks.IPv6)
}
return networks, nil
}
func ParseNodeHostCIDRsList(node *kapi.Node) ([]string, error) {
addrAnnotation, ok := node.Annotations[OVNNodeHostCIDRs]
if !ok {
return nil, newAnnotationNotSetError("%s annotation not found for node %q", OVNNodeHostCIDRs, node.Name)
}
var cfg []string
if err := json.Unmarshal([]byte(addrAnnotation), &cfg); err != nil {
return nil, fmt.Errorf("failed to unmarshal host cidrs annotation %s for node %q: %v",
addrAnnotation, node.Name, err)
}
return cfg, nil
}
// IsNodeSecondaryHostEgressIPsAnnotationSet returns true if an annotation that tracks assigned of egress IPs to interfaces OVN doesn't manage
// is set
func IsNodeSecondaryHostEgressIPsAnnotationSet(node *kapi.Node) bool {
_, ok := node.Annotations[OVNNodeSecondaryHostEgressIPs]
return ok
}
// ParseNodeSecondaryHostEgressIPsAnnotation returns secondary host egress IPs addresses for a node
func ParseNodeSecondaryHostEgressIPsAnnotation(node *kapi.Node) (sets.Set[string], error) {
addrAnnotation, ok := node.Annotations[OVNNodeSecondaryHostEgressIPs]
if !ok {
return nil, newAnnotationNotSetError("%s annotation not found for node %q", OVNNodeSecondaryHostEgressIPs, node.Name)
}
var cfg []string
if err := json.Unmarshal([]byte(addrAnnotation), &cfg); err != nil {
return nil, fmt.Errorf("failed to unmarshal %s annotation %s for node %q: %v", OVNNodeSecondaryHostEgressIPs, addrAnnotation, node.Name, err)
}
return sets.New(cfg...), nil
}
// IsSecondaryHostNetworkContainingIP attempts to find a secondary host network that will host the argument IP. If no network is
// found, false is returned
func IsSecondaryHostNetworkContainingIP(node *v1.Node, ip net.IP) (bool, error) {
if ip == nil {
return false, fmt.Errorf("empty IP is not valid")
}
if node == nil {
return false, fmt.Errorf("unable to determine if IP %s is a secondary host network because node argument is nil", ip.String())
}
network, err := GetSecondaryHostNetworkContainingIP(node, ip)
if err != nil {
return false, fmt.Errorf("failed to determine if IP %s is hosted by a secondary host network for node %s: %v",
ip.String(), node.Name, err)
}
if network == "" {
return false, nil
}
return true, nil
}
// GetEgressIPNetwork attempts to retrieve a network that contains EgressIP. Check the OVN network first as
// represented by parameter eIPConfig, and if no match is found, and if not in a cloud environment, check secondary host networks.
func GetEgressIPNetwork(node *v1.Node, eIPConfig *ParsedNodeEgressIPConfiguration, eIP net.IP) (string, error) {
if eIPConfig.V4.Net != nil && eIPConfig.V4.Net.Contains(eIP) {
return eIPConfig.V4.Net.String(), nil
}
if eIPConfig.V6.Net != nil && eIPConfig.V6.Net.Contains(eIP) {
return eIPConfig.V6.Net.String(), nil
}
// Do not attempt to check if a secondary host network may host an EIP if we are in a cloud environment
if PlatformTypeIsEgressIPCloudProvider() {
return "", nil
}
network, err := GetSecondaryHostNetworkContainingIP(node, eIP)
if err != nil {
return "", fmt.Errorf("failed to get Egress IP %s network for node %s: %v", eIP.String(), node.Name, err)
}
return network, nil
}
// IsOVNNetwork attempts to detect if the argument IP can be hosted by a network managed by OVN. Currently, this is
// only the primary OVN network
func IsOVNNetwork(eIPConfig *ParsedNodeEgressIPConfiguration, ip net.IP) bool {
if eIPConfig.V4.Net != nil && eIPConfig.V4.Net.Contains(ip) {
return true
}
if eIPConfig.V6.Net != nil && eIPConfig.V6.Net.Contains(ip) {
return true
}
return false
}
// GetSecondaryHostNetworkContainingIP attempts to find a secondary host network to host the argument IP
// and includes only global unicast addresses.
func GetSecondaryHostNetworkContainingIP(node *v1.Node, ip net.IP) (string, error) {
networks, err := ParseNodeHostCIDRsExcludeOVNNetworks(node)
if err != nil {
return "", fmt.Errorf("failed to get host-cidrs annotation excluding OVN networks for node %s: %v",
node.Name, err)
}
cidrs, err := makeCIDRs(networks...)
if err != nil {
return "", err
}
if len(cidrs) == 0 {
return "", nil
}
isIPv6 := ip.To4() == nil
cidrs = filterIPVersion(cidrs, isIPv6)
lpmTree := cidrtree.New(cidrs...)
for _, prefix := range cidrs {
if !prefix.Addr().IsGlobalUnicast() {
lpmTree.Delete(prefix)
}
}
addr, err := netip.ParseAddr(ip.String())
if err != nil {
return "", fmt.Errorf("failed to convert IP %s to netip address: %v", ip.String(), err)
}
match, found := lpmTree.Lookup(addr)
if !found {
return "", nil
}
return match.String(), nil
}
// UpdateNodeIDAnnotation updates the ovnNodeID annotation with the node id in the annotations map
// and returns it.
func UpdateNodeIDAnnotation(annotations map[string]interface{}, nodeID int) map[string]interface{} {
if annotations == nil {
annotations = make(map[string]interface{})
}
annotations[ovnNodeID] = strconv.Itoa(nodeID)
return annotations
}
// GetNodeID returns the id of the node set in the 'ovnNodeID' node annotation.
// Returns InvalidNodeID (-1) if the 'ovnNodeID' node annotation is not set or if the value is
// not an integer value.
func GetNodeID(node *kapi.Node) int {
nodeID, ok := node.Annotations[ovnNodeID]
if !ok {
return InvalidNodeID
}
id, err := strconv.Atoi(nodeID)
if err != nil {
return InvalidNodeID
}
return id
}
// NodeIDAnnotationChanged returns true if the ovnNodeID in the corev1.Nodes doesn't match
func NodeIDAnnotationChanged(oldNode, newNode *corev1.Node) bool {
return oldNode.Annotations[ovnNodeID] != newNode.Annotations[ovnNodeID]
}
// SetNodeZone sets the node's zone in the 'ovnNodeZoneName' node annotation.
func SetNodeZone(nodeAnnotator kube.Annotator, zoneName string) error {
return nodeAnnotator.Set(OvnNodeZoneName, zoneName)
}
/** HACK BEGIN **/
// TODO(tssurya): Remove this a few months from now
// SetNodeZoneMigrated sets the node's zone in the 'ovnNodeMigratedZoneName' node annotation.
func SetNodeZoneMigrated(nodeAnnotator kube.Annotator, zoneName string) error {
return nodeAnnotator.Set(OvnNodeMigratedZoneName, zoneName)
}
// HasNodeMigratedZone returns true if node has its ovnNodeMigratedZoneName set already
func HasNodeMigratedZone(node *kapi.Node) bool {
_, ok := node.Annotations[OvnNodeMigratedZoneName]
return ok
}
// NodeMigratedZoneAnnotationChanged returns true if the ovnNodeMigratedZoneName annotation changed for the node
func NodeMigratedZoneAnnotationChanged(oldNode, newNode *corev1.Node) bool {
return oldNode.Annotations[OvnNodeMigratedZoneName] != newNode.Annotations[OvnNodeMigratedZoneName]
}
/** HACK END **/
// GetNodeZone returns the zone of the node set in the 'ovnNodeZoneName' node annotation.
// If the annotation is not set, it returns the 'default' zone name.
func GetNodeZone(node *kapi.Node) string {
zoneName, ok := node.Annotations[OvnNodeZoneName]
if !ok {
return types.OvnDefaultZone
}
return zoneName
}
// NodeZoneAnnotationChanged returns true if the ovnNodeZoneName in the corev1.Nodes doesn't match
func NodeZoneAnnotationChanged(oldNode, newNode *corev1.Node) bool {
return oldNode.Annotations[OvnNodeZoneName] != newNode.Annotations[OvnNodeZoneName]
}
func parseNetworkIDsAnnotation(nodeAnnotations map[string]string, annotationName string) (map[string]string, error) {
annotation, ok := nodeAnnotations[annotationName]
if !ok {
return nil, newAnnotationNotSetError("could not find %q annotation", annotationName)
}
networkIdsStrMap := map[string]string{}
networkIds := make(map[string]string)
if err := json.Unmarshal([]byte(annotation), &networkIds); err != nil {
return nil, fmt.Errorf("could not parse %q annotation %q : %v",
annotationName, annotation, err)
}
for netName, v := range networkIds {
networkIdsStrMap[netName] = v
}
if len(networkIdsStrMap) == 0 {
return nil, fmt.Errorf("unexpected empty %s annotation", annotationName)
}
return networkIdsStrMap, nil
}
// ParseNetworkIDAnnotation parses the 'ovnNetworkIDs' annotation for the specified
// network in 'netName' and returns the network id.
func ParseNetworkIDAnnotation(node *kapi.Node, netName string) (int, error) {