-
Notifications
You must be signed in to change notification settings - Fork 0
/
director.go
290 lines (254 loc) · 13 KB
/
director.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
package director
import (
"errors"
"fmt"
"io/ioutil"
"net"
"os"
"os/exec"
"strings"
"github.com/coreos/go-iptables/iptables"
"github.com/golang/glog"
"github.com/janeczku/go-ipset/ipset"
utils "github.com/nirmata/kube-static-egress-ip/pkg/utils"
"k8s.io/client-go/kubernetes"
)
const (
customStaticEgressIPRouteTableID = "99"
customStaticEgressIPRouteTableName = "kube-static-egress-ip"
staticEgressIPFWMARK = "1000"
bypassCNIMasquradeChainName = "STATIC-EGRESS-BYPASS-CNI"
)
// EgressDirector manages routing rules needed on a node to redirect egress traffic from the pods that need
// a static egress IP to a node acting as egress gateway based on the `staticegressip` CRD object
type EgressDirector struct {
ipt *iptables.IPTables
nodeIP string
}
// NewEgressDirector is a constructor for EgressDirector
func NewEgressDirector(clientset kubernetes.Interface) (*EgressDirector, error) {
ipt, err := iptables.New()
if err != nil {
return nil, fmt.Errorf("failed to locate iptables: %v", err)
}
nodeObject, err := utils.GetNodeObject(clientset, "")
if err != nil {
return nil, err
}
nodeIP, err := utils.GetNodeIP(nodeObject)
if err != nil {
return nil, errors.New("Failed to get node IP due to " + err.Error())
}
return &EgressDirector{nodeIP: nodeIP.String(), ipt: ipt}, nil
}
// Setup sets up the node with one-time basic settings needed for director functionality
func (d *EgressDirector) Setup() error {
// create custom routing table for directing the traffic from director nodes to the gateway node
b, err := ioutil.ReadFile("/etc/iproute2/rt_tables")
if err != nil {
return errors.New("Failed to add custom routing table in /etc/iproute2/rt_tables needed for policy routing for directing traffing to egress gateway" + err.Error())
}
if !strings.Contains(string(b), customStaticEgressIPRouteTableName) {
f, err := os.OpenFile("/etc/iproute2/rt_tables", os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
return errors.New("Failed to open /etc/iproute2/rt_tables to verify custom routing table " + customStaticEgressIPRouteTableName + " required for static egress IP functionality due to " + err.Error())
}
defer f.Close()
if _, err = f.WriteString(customStaticEgressIPRouteTableID + " " + customStaticEgressIPRouteTableName + "\n"); err != nil {
return errors.New("Failed to add custom routing table " + customStaticEgressIPRouteTableName + " in /etc/iproute2/rt_tables needed for policy routing due to " + err.Error())
}
}
// create policy based routing (ip rule) to lookup the custom routing table for FWMARK packets
out, err := exec.Command("ip", "rule", "list").Output()
if err != nil {
return errors.New("Failed to verify if `ip rule` exists due to: " + err.Error())
}
if !strings.Contains(string(out), customStaticEgressIPRouteTableName) {
err = exec.Command("ip", "rule", "add", "prio", "32764", "fwmark", staticEgressIPFWMARK, "table", customStaticEgressIPRouteTableName).Run()
if err != nil {
return errors.New("Failed to add policy rule to lookup traffic marked with fwmark " + staticEgressIPFWMARK + " to the custom " + " routing table due to " + err.Error())
}
}
// setup a chain in nat table to bypass the CNI masqurade for the traffic bound to egress gateway
err = d.ipt.NewChain("nat", bypassCNIMasquradeChainName)
if err != nil && err.(*iptables.Error).ExitStatus() != 1 {
return errors.New("Failed to add a " + bypassCNIMasquradeChainName + " chain in NAT table required to bypass CNI masqurading due to" + err.Error())
}
ruleSpec := []string{"-j", bypassCNIMasquradeChainName}
hasRule, err := d.ipt.Exists("nat", "POSTROUTING", ruleSpec...)
if err != nil {
return errors.New("Failed to verify bypass CNI masqurade rule exists in POSTROUTING chain of nat table due to " + err.Error())
}
if !hasRule {
err = d.ipt.Insert("nat", "POSTROUTING", 1, ruleSpec...)
if err != nil {
return errors.New("Failed to run iptables command to add a rule to jump to STATIC_EGRESSIP_BYPASS_CNI_MASQURADE chain due to " + err.Error())
}
}
glog.Infof("Node has been setup for static egress IP director functionality successfully.")
return nil
}
// AddRouteToGateway adds a routes on the director node to redirect traffic from a set of pod IP's
// (selected by service name in the rule of staticegressip CRD object) to a specific
// destination CIDR to be directed to egress gateway node
func (d *EgressDirector) AddRouteToGateway(setName string, sourceIPs []string, destinationIP, egressGateway string) error {
// create IPset for the set of sourceIP's
set, err := ipset.New(setName, "hash:ip", &ipset.Params{})
if err != nil {
return errors.New("Failed to create ipset with name " + setName + " due to %" + err.Error())
}
glog.Infof("Created ipset name: %s", setName)
// add IP's that need to be part of the ipset
for _, ip := range sourceIPs {
err = set.Add(ip, 0)
if err != nil {
return errors.New("Failed to add an ip " + ip + " into ipset with name " + setName + " due to %" + err.Error())
}
}
glog.Infof("Added ips %v to the ipset name: %s", sourceIPs, setName)
// create iptables rule in mangle table PREROUTING chain to match src to ipset created and destination
// matching destinationIP then fwmark the packets
ruleSpec := []string{"-m", "set", "--match-set", setName, "src", "-d", destinationIP, "-j", "MARK", "--set-mark", staticEgressIPFWMARK}
hasRule, err := d.ipt.Exists("mangle", "PREROUTING", ruleSpec...)
if err != nil {
return errors.New("Failed to verify rule exists in PREROUTING chain of mangle table to fwmark egress traffic that needs static egress IP" + err.Error())
}
if !hasRule {
err = d.ipt.Insert("mangle", "PREROUTING", 1, ruleSpec...)
if err != nil {
return errors.New("Failed to add rule in PREROUTING chain of mangle table to fwmark egress traffic that needs static egress IP" + err.Error())
}
glog.Infof("added rule in PREROUTING chain of mangle table to fwmark egress traffic that needs static egress IP")
}
glog.Infof("iptables rule in mangle table PREROUTING chain to match src to ipset")
ruleSpec = []string{"-m", "set", "--match-set", setName, "src", "-d", destinationIP, "-j", "ACCEPT"}
hasRule, err = d.ipt.Exists("nat", bypassCNIMasquradeChainName, ruleSpec...)
if err != nil {
return errors.New("Failed to verify rule exists in BYPASS_CNI_MASQURADE chain of nat table to bypass the CNI masqurade" + err.Error())
}
if !hasRule {
err = d.ipt.Append("nat", bypassCNIMasquradeChainName, ruleSpec...)
if err != nil {
return errors.New("Failed to run iptables command to add a rule to ACCEPT traffic in BYPASS_CNI_MASQURADE chain" + err.Error())
}
}
// create a tunnel interface to gateway node if does not exist
tunnelName := "tun" + strings.Replace(egressGateway, ".", "", -1)
out, err := exec.Command("ip", "link", "list").Output()
if err != nil {
return errors.New("Failed to verify required tunnel to gatewat exists. " + err.Error())
}
if !strings.Contains(string(out), tunnelName) {
// ip tunnel add seip mode gre remote 192.168.1.102 local 192.168.1.101
if err = exec.Command("ip", "tunnel", "add", tunnelName, "mode", "gre", "remote", egressGateway, "local", d.nodeIP).Run(); err != nil {
return errors.New("Failed to tunnel interface to gateway node due to: " + err.Error())
}
}
if err = exec.Command("ip", "link", "set", "up", tunnelName).Run(); err != nil {
return errors.New("Failed to set tunnel interface to up due to: " + err.Error())
}
// add routing entry in custom routing table to forward destinationIP to egressGateway
out, err = exec.Command("ip", "route", "list", "table", customStaticEgressIPRouteTableName).Output()
if err != nil {
return errors.New("Failed to verify required default route to gatewat exists. " + err.Error())
}
destAddr, _, err := net.ParseCIDR(destinationIP)
if err != nil {
if net.ParseIP(destinationIP) == nil {
}
}
if !strings.Contains(string(out), destinationIP) && !strings.Contains(string(out), destAddr.String()) {
if err = exec.Command("ip", "route", "add", destinationIP, "dev", tunnelName, "table", customStaticEgressIPRouteTableName).Run(); err != nil {
return errors.New("Failed to add route in custom route table due to: " + err.Error())
}
}
glog.Infof("added routing entry in custom routing table to forward destinationIP to egressGateway")
return nil
}
// DeleteRouteToGateway removes the route routes on the director node to redirect traffic to gateway node
func (d *EgressDirector) DeleteRouteToGateway(setName string, destinationIP, egressGateway string) error {
set, err := ipset.New(setName, "hash:ip", &ipset.Params{})
if err != nil {
return errors.New("Failed to get ipset with name " + setName + " due to %" + err.Error())
}
// create iptables rule in mangle table PREROUTING chain to match src to ipset created and destination
// matching destinationIP then fwmark the packets
ruleSpec := []string{"-m", "set", "--match-set", setName, "src", "-d", destinationIP, "-j", "MARK", "--set-mark", staticEgressIPFWMARK}
hasRule, err := d.ipt.Exists("mangle", "PREROUTING", ruleSpec...)
if err != nil {
return errors.New("Failed to verify rule exists in PREROUTING chain of mangle table to fwmark egress traffic that needs static egress IP" + err.Error())
}
if hasRule {
err = d.ipt.Delete("mangle", "PREROUTING", ruleSpec...)
if err != nil {
return errors.New("Failed to delete rule in PREROUTING chain of mangle table to fwmark egress traffic that needs static egress IP" + err.Error())
}
glog.Infof("deleted rule in PREROUTING chain of mangle table to fwmark egress traffic that needs static egress IP")
}
ruleSpec = []string{"-m", "set", "--match-set", setName, "src", "-d", destinationIP, "-j", "ACCEPT"}
hasRule, err = d.ipt.Exists("nat", bypassCNIMasquradeChainName, ruleSpec...)
if err != nil {
return errors.New("Failed to verify rule exists in BYPASS_CNI_MASQURADE chain of nat table to bypass the CNI masqurade" + err.Error())
}
if hasRule {
err = d.ipt.Delete("nat", bypassCNIMasquradeChainName, ruleSpec...)
if err != nil {
return errors.New("Failed to delete iptables command to add a rule to ACCEPT traffic in BYPASS_CNI_MASQURADE chain" + err.Error())
}
}
// add routing entry in custom routing table to forward destinationIP to egressGateway
out, err := exec.Command("ip", "route", "list", "table", customStaticEgressIPRouteTableName).Output()
if err != nil {
return errors.New("Failed to verify required default route to gatewat exists. " + err.Error())
}
if !strings.Contains(string(out), destinationIP) {
if err = exec.Command("ip", "route", "delete", destinationIP, "via", egressGateway, "table", customStaticEgressIPRouteTableName).Run(); err != nil {
return errors.New("Failed to delete route in custom route table due to: " + err.Error())
}
glog.Infof("deleted route")
}
err = set.Destroy()
if err != nil {
return errors.New("Failed to delete ipset due to " + err.Error())
}
return nil
}
func (d *EgressDirector) ClearStaleRouteToGateway(setName string, destinationIP, egressGateway string) error {
// create iptables rule in mangle table PREROUTING chain to match src to ipset created and destination
// matching destinationIP then fwmark the packets
ruleSpec := []string{"-m", "set", "--match-set", setName, "src", "-d", destinationIP, "-j", "MARK", "--set-mark", staticEgressIPFWMARK}
hasRule, err := d.ipt.Exists("mangle", "PREROUTING", ruleSpec...)
if err != nil {
return errors.New("Failed to verify rule exists in PREROUTING chain of mangle table to fwmark egress traffic that needs static egress IP" + err.Error())
}
if hasRule {
err = d.ipt.Delete("mangle", "PREROUTING", ruleSpec...)
if err != nil {
return errors.New("Failed to delete rule in PREROUTING chain of mangle table to fwmark egress traffic that needs static egress IP" + err.Error())
}
glog.Infof("deleted rule in PREROUTING chain of mangle table to fwmark egress traffic that needs static egress IP")
}
ruleSpec = []string{"-m", "set", "--match-set", setName, "src", "-d", destinationIP, "-j", "ACCEPT"}
hasRule, err = d.ipt.Exists("nat", bypassCNIMasquradeChainName, ruleSpec...)
if err != nil {
return errors.New("Failed to verify rule exists in BYPASS_CNI_MASQURADE chain of nat table to bypass the CNI masqurade" + err.Error())
}
if hasRule {
err = d.ipt.Delete("nat", bypassCNIMasquradeChainName, ruleSpec...)
if err != nil {
return errors.New("Failed to delete iptables command to add a rule to ACCEPT traffic in BYPASS_CNI_MASQURADE chain" + err.Error())
}
}
// add routing entry in custom routing table to forward destinationIP to egressGateway
out, err := exec.Command("ip", "route", "list", "table", customStaticEgressIPRouteTableName).Output()
if err != nil {
return errors.New("Failed to verify required default route to gatewat exists. " + err.Error())
}
if !strings.Contains(string(out), destinationIP) {
if err = exec.Command("ip", "route", "delete", destinationIP, "via", egressGateway, "table", customStaticEgressIPRouteTableName).Run(); err != nil {
return errors.New("Failed to delete route in custom route table due to: " + err.Error())
}
glog.Infof("deleted route")
}
return nil
}