This repository has been archived by the owner on Mar 3, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 27
/
secondary_vnic_all_configure.sh
1431 lines (1305 loc) · 54.3 KB
/
secondary_vnic_all_configure.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
# Copyright (c) 2018, Oracle and/or its affiliates.
# The Universal Permissive License (UPL), Version 1.0
#
# Oracle OCI Virtual Cloud Networks IP configuration script
#
# 2017-10-24 initial release
# 2017-11-21 filter out VLANs if VM
# 2017-11-21 inhibit namespaces for ubuntu 16
# 2018-02-12 fix sshd typo in help
# 2018-02-20 update copyright notice
# 2018-03-21 fix to add src routing on primary VNIC when not using namespaces
# 2018-04-05 add note on multiple VCNs
# 2018-05-10 fix to delete config by deleting default route before deleting table, added -v option
# 2018-05-22 fix to ignore interfaces enslaved to a master device
# 2018-09-14 allow namespaces for Ubuntu 16 since problems with DHCP are fixed, and inhibit for all OL/Centos 6
# 2018-09-19 fix os name and version setting for centos 6 images
# 2018-09-19 fix ip addr parsing to remove MAC VLAN interfaces
# 2018-10-25 fix secondary private ip src routing
# 2018-10-31 fix misc routing setup, fix multiple ifaces on same subnet
declare -r THIS=$(basename "$0")
declare -r MD_URL='http://169.254.169.254/opc/v1/vnics/'
declare -r NA='-'
declare -r RTS_FILE='/etc/iproute2/rt_tables'
declare -ir RT_ID_MIN=10 # in case lower ones are reserved
declare -ir RT_ID_MAX=255
declare -r RT_FORMAT_BM='ort${nic}vl${vltag}'
declare -r RT_FORMAT_VM='ort${nic}'
declare -r DEF_NS_FORMAT_BM='ons${nic}vl${vltag}'
declare -r DEF_NS_FORMAT_VM='ons${nic}'
declare -r MACVLAN_FORMAT='${iface}.${vltag}' # note awk script looks for this (max 15 chars)
declare -r VLAN_FORMAT='${iface}v${vltag}' # (max 15 chars)
declare -ir MTU=9000
declare -r ADD='ADD'
declare -r DELETE='DELETE'
declare -r YES='YES'
declare -r CURL=$(which curl)
declare -r IP=$(which ip)
declare -r SSHD=$(which sshd)
declare -r MODPROBE=$(which modprobe)
declare -r OS_RELEASE='/etc/os-release'
if [ -f "$OS_RELEASE" ]; then
declare -r OS_ID=$(grep -ws ID $OS_RELEASE | cut -f 2 -d '=' | tr -d '"' | tr '[:upper:]' '[:lower:]')
declare -r OS_VERSION=$(grep -ws VERSION_ID $OS_RELEASE | cut -f 2 -d '=' | tr -d '"')
else
declare -r OS_RELEASE_alt='/etc/redhat-release'
if [ -f "$OS_RELEASE_alt" ]; then
declare -r OS_ID=$(cat $OS_RELEASE_alt|cut -f 1 -d ' ' | tr '[:upper:]' '[:lower:]')
declare -r OS_VERSION=$(cat $OS_RELEASE_alt | cut -f 3 -d ' ')
fi
fi
if [ -n "$OS_VERSION" ]; then
declare -r OS_MAJ_VERSION=$(echo $OS_VERSION | cut -f 1 -d '.')
fi
declare -r SYS_CLASS_NET='/sys/class/net'
declare -A VIRTUAL_IFACES
declare IS_VM=''
declare -a MACS # all (unique) macs
declare -A MD_I_BY_MAC # index into arrays by MAC
declare -a MD_MACS
declare -a MD_ADDRS
declare -a MD_VLTAGS
declare -a MD_SCIDRS
declare -a MD_SPREFIXS
declare -a MD_SBITSS
declare -a MD_VIRTRTS
declare -a MD_VNICS
declare -a MD_NIC_IS # not set at all if vm
declare -a MD_CONFIGS # $ADD if vnic added
declare -A DUP_ADDRS # hash of addrs that appear more than once
declare -A DUP_SADDRS # hash of subnet addrs that appear more than once
# items use $NA to mean null
declare -A IP_I_BY_MAC # index into arrays by MAC
declare -a IP_MACS # runs of dups if sec addrs
declare -a IP_NSS
declare -a IP_IFACES
declare -a IP_ADDRS
declare -a IP_SADDRS
declare -a IP_SBITSS
declare -a IP_VIRTRTS
declare -a IP_STATES
declare -a IP_VLANS
declare -a IP_VLTAGS # vltag (0 if phys iface)
declare -a IP_SECADS # set to $YES if secondary addr
declare -a IP_SRCS # set to $YES if src hint
declare -a IP_NIC_IS # nic index of iface
declare -a IP_CONFIGS # $DELETE if vnic deleted
declare -a NIC_IP_IS # index of physical iface for nic index
declare -A NIC_I_BY_PHYS_IP_I # nic index for physical iface ip_i
# be sure to clear any IP_ arrays above in the read function
# options:
declare QUIET=''
declare DEBUG=''
declare START_SSHD=''
declare USE_NS=''
declare NS_FORMAT=''
declare -a SEC_ADDRS
declare -a SEC_VNICS
declare -r IFACE_AWK_SCRIPT='/tmp/oci_vcn_iface.awk'
cat >$IFACE_AWK_SCRIPT <<'EOF'
function prtiface(mac, iface, addr, sbits, state, vlan, vltag, secad) {
if (iface != "") printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", mac, iface, addr, sbits, state, vlan, vltag, secad
}
BEGIN { iface = ""; addr = "-" }
/^[0-9]/ {
# if not the first and prev was not a mac vlan then print previous (note print at end too)
if (addr == "-") prtiface(mac, iface, addr, sbits, state, vlan, vltag, secad)
addr = "-"
sbits = "-"
state = "-"
macvlan = "-"
vlan = "-"
vltag = "-"
secad = "-"
if ($0 ~ /BROADCAST/ && $0 !~ /UNKNOWN/ && $0 !~ /NO-CARRIER/ && $0 !~ /master /) {
i = index($2, "@")
if (1 < i) {
j = index($2, ".")
if (j < i) { # mac vlan (not used, no addrs)
macvlan = substr($2, 1, i - 1)
iface = substr($2, i + 1, length($2) - i - 1) # skip : at end
addr = "" # skip the mac vlan iface
} else { # vlan
vlan = substr($2, 1, i - 1)
# extract iface/vltag from macvlan
iface = substr($2, i + 1, j - i - 1)
vltag = substr($2, j + 1, length($2) - j - 1) # skip : at end
}
} else {
i = index($2, ":")
if (i <= 1) { print "cannot find interface name"; exit 1 }
iface = substr($2, 1, i - 1)
}
if ($0 ~ /LOWER_UP/) state = "UP"
else state = "DOWN"
} else iface = ""
next
}
/ link\/ether / { mac = tolower($2) }
/ inet [0-9]/ {
i = index($2, "/")
if (i <= 1) { print "cannot find interface inet address"; exit 1 }
if (addr != "-") secad = "YES"
addr = substr($2, 0, i - 1)
sbits = substr($2, i + 1, length($2) - i)
prtiface(mac, iface, addr, sbits, state, vlan, vltag, secad)
}
END { if (addr == "-") prtiface(mac, iface, addr, sbits, state, vlan, vltag, secad) }
EOF
oci_vcn_err() {
echo "Error: $1" >&2
exit 1
}
oci_vcn_warn() {
echo "Warning: $1" >&2
}
oci_vcn_info() {
[ -n "$QUIET" ] || echo "Info: $1" >&2
}
oci_vcn_debug() {
[ -z "$DEBUG" ] || echo "Debug: $1" >&2
}
oci_vcn_virtual_ifaces_read() {
VIRTUAL_IFACES=()
local iface
for iface in $(ls $SYS_CLASS_NET); do
if ls -l $SYS_CLASS_NET/$iface | grep -wq virtual; then
VIRTUAL_IFACES[$iface]='t'
fi
done
}
oci_vcn_md_read() {
# sets all MD data arrays and their index I_BY_MAC
local -r tmpfile=$(mktemp /tmp/oci_vcn_md.XXXXX)
# MD notes:
# vnic order: primary first, then time created (and therefore vltag/nic)
# BM notes: (1) may be interleaved wrt nic index (i.e. all nic 0 not guaranteed before all nic 1)
# (2) nicIndex was supported starting around 8/23/17, but will not appear on previously
# launched instances unless refreshed by a vnic attach or detach after that date
# parse: force json fields on separate lines
# WARNING: assumes no string values with commas or double quotes
# WARNING: assumes no sub-objects with identical field names
[ -n "$CURL" ] || oci_vcn_err "cannot find curl command"
$CURL -s $MD_URL | tr , '\n' >"$tmpfile" || oci_vcn_err "cannot read metadata"
MD_MACS=($(grep -w macAddr "$tmpfile" | cut -f 4 -d '"')) || exit $? # string
local -i i
for i in $(seq 0 $((${#MD_MACS[@]} - 1))); do
MD_MACS[$i]="${MD_MACS[$i],,}"
done
MD_ADDRS=($(grep -w privateIp "$tmpfile" | cut -f 4 -d '"')) # string
MD_VLTAGS=($(grep -w vlanTag "$tmpfile" | cut -f 2 -d ':' | tr -d ' ')) # integer
MD_VIRTRTS=($(grep -w virtualRouterIp "$tmpfile" | cut -f 4 -d '"')) # string
local s
for s in $(grep -w subnetCidrBlock "$tmpfile" | cut -f 4 -d '"'); do # string
MD_SCIDRS+=(${s})
MD_SPREFIXS+=(${s%/*})
MD_SBITSS+=(${s#*/})
done
MD_VNICS=($(grep -w vnicId "$tmpfile" | cut -f 4 -d '"'))
MD_NIC_IS=($(grep -w nicIndex "$tmpfile" | cut -f 2 -d ':' | tr -d ' '))
# do some validity checks on md data
[ ${#MD_MACS[@]} -eq ${#MD_ADDRS[@]} ] || oci_vcn_err "invalid metadata: MAC or IP addresses are missing"
[ ${#MD_MACS[@]} -eq ${#MD_VLTAGS[@]} ] || oci_vcn_err "invalid metadata: MAC or VLAN tags are missing"
[ ${#MD_MACS[@]} -eq ${#MD_VIRTRTS[@]} ] || oci_vcn_err "invalid metadata: MAC or virtual router addresses are missing"
[ ${#MD_MACS[@]} -eq ${#MD_SPREFIXS[@]} ] || oci_vcn_err "invalid metadata: MAC or subnets are missing"
[ ${#MD_MACS[@]} -eq ${#MD_VNICS[@]} ] || oci_vcn_err "invalid metadata: MAC or VNIC ids are missing"
for i in $(seq 0 $((${#MD_MACS[@]} - 1))); do
[[ ${MD_ADDRS[$i]} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || oci_vcn_err "invalid metadata: address IP format incorrect: ${MD_ADDRS[$i]}"
[[ ${MD_VLTAGS[$i]} =~ ^[0-9]+$ ]] || oci_vcn_err "invalid metadata: VLAN tag incorrect: ${MD_VLTAGS[$i]}"
[[ ${MD_VIRTRTS[$i]} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || oci_vcn_err "invalid metadata: virtual router address format incorrect: ${MD_VIRTRTS[$i]}"
[[ ${MD_SPREFIXS[$i]} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || oci_vcn_err "invalid metadata: subnet prefix format incorrect: ${MD_SPREFIXS[$i]}"
done
# set vm flag based on existence of nic index (see override in oci_vcn_ip_read)
# get the virtual interfaces if VM
if [ ${#MD_NIC_IS[@]} -eq 0 ]; then
IS_VM='t'
oci_vcn_virtual_ifaces_read
fi
# create reverse lookup
for i in "${!MD_MACS[@]}"; do
MD_I_BY_MAC[${MD_MACS[$i]}]=$i
done
rm "$tmpfile"
# find the duplicate addrs, if any
local -A addrs=()
local addr
for addr in "${MD_ADDRS[@]}"; do
if [ -n "${addrs[$addr]}" ]; then DUP_ADDRS[$addr]='t'; fi
addrs[$addr]='t'
done
# find the duplicate subnet addrs, if any
local -A saddrs=()
for addr in "${MD_SPREFIXS[@]}"; do
if [ -n "${saddrs[$addr]}" ]; then DUP_SADDRS[addr]='t'; fi
saddrs[addr]='t'
done
}
oci_vcn_ip_route_table_name() {
local -ir nic=$1 # format looks for "${nic}", note this is really $nic_i
local -ir vltag=$2
local format="$RT_FORMAT_VM"
[ -n "$IS_VM" ] || format="$RT_FORMAT_BM"
eval echo "$format"
}
oci_vcn_ip_route_table_name_ip_i() {
# use only when ip already setup
local -ir ip_i=$1
local -ir nic_i=${IP_NIC_IS[$ip_i]}
local -ir vltag=${IP_VLTAGS[$ip_i]}
oci_vcn_ip_route_table_name $nic_i $vltag
}
oci_vcn_ip_route_table_exists() {
local -r rt_name=$1
if grep -qsw $rt_name $RTS_FILE; then
echo "$rt_name"
fi
}
oci_vcn_ip_route_table_find_unused_id() {
# read all the current route table id/name pairs
# mapfile will create array with each line an element
local lines
mapfile -t lines < <(cat $RTS_FILE | grep -E '^[0-9]' | tr '\t' ' ' | tr -s ' ' ' ') || oci_vcn_err "cannot read route tables file $RTS_FILE"
local line
local -A rt_by_id
for line in "${lines[@]}"; do
local -a pair=($line)
local id=${pair[0]}
local name=${pair[1]}
rt_by_id[$id]=$name
done
# find first id not used
local -i unused=-1
local -i i
for i in $(seq $RT_ID_MIN $RT_ID_MAX); do
if [ -z "${rt_by_id[$i]}" ]; then
unused=$i
break
fi
done
[ $unused -ne -1 ] || oci_vcn_err "cannot find unused id in route tables file $RTS_FILE"
echo $unused
}
oci_vcn_ip_route_table_create() {
local -ir nic_i=$1
local -ir vltag=$2
local -r skip_if_exists=$3
local -r rt_name=$(oci_vcn_ip_route_table_name $nic_i $vltag)
# Check if the route table exists
local rt_exists
rt_exists=$(oci_vcn_ip_route_table_exists $rt_name) || exit $?
if [ -n "$rt_exists" ]; then # already exists
if [ -n "$skip_if_exists" ]; then
oci_vcn_debug "create route table $rt_name already exists, skipping"
return
fi
oci_vcn_warn "route table $rt_name already exists, reusing"
else # create
local -i rt_id
rt_id=$(oci_vcn_ip_route_table_find_unused_id) || exit $?
oci_vcn_debug "create route table $rt_name ($rt_id)"
echo "$rt_id $rt_name" >> $RTS_FILE
fi
echo "$rt_name"
}
oci_vcn_ip_route_table_del() {
local -r rt_name=$1
local -r tmpfile=$(mktemp /tmp/oci_vcn_rt_tables.XXXXX)
if ! grep -qsw $rt_name $RTS_FILE; then
oci_vcn_debug "rt $rt_name not in $RTS_FILE"
return
fi
oci_vcn_debug "rt $rt_name delete"
cp -p $RTS_FILE "$tmpfile" # in case grep fails
grep -vw $rt_name $RTS_FILE > "$tmpfile"
mv "$tmpfile" $RTS_FILE
echo 't'
}
oci_vcn_ip_routing_add() {
local -ir md_i=$1
local -ir nic_i=$2
local -r iface=$3
local -r ns=$4
local -r skip_if_exists=$5
local -ir vltag="${MD_VLTAGS[$md_i]}"
local -r addr="${MD_ADDRS[$md_i]}"
local -r sprefix="${MD_SPREFIXS[$md_i]}"
local -r virtrt="${MD_VIRTRTS[$md_i]}"
local -r scidr="${MD_SCIDRS[$md_i]}"
# set a default route to the gateway/virtual router
if [ -n "$ns" ]; then
local -r nscmd="netns exec $ns $IP"
# using a namespace: iface is assumed only one in namespace, so just set a
# default route to the gateway in the main route table
oci_vcn_debug "default route add"
$IP $nscmd route add default via $virtrt || oci_vcn_err "cannot add namespace $ns default gateway $virtrt"
oci_vcn_info "added namespace $ns default route to gateway $virtrt"
else
# the default route (in main route table) already defined through primary vnic.
# this adds a rule to lookup a route table for any packets sourced from addr
# and then the route table has the default route for that addr.
# this allows packets from protocol services that reply with src addr
# set to route back out through the iface (prevents asymmetric routing).
# check for dup addrs and subnet addrs
if [ -n "${DUP_ADDRS[$addr]}" ]; then
oci_vcn_warn "IP address $addr is a duplicate, skipping creating source route rule"
return
fi
[ -z "${DUP_SADDRS[$sprefix]}" ] || oci_vcn_warn "duplicate subnet prefix $sprefix"
# create route table and add a default route via the gateway, and subnet route
local rt_name
rt_name=$(oci_vcn_ip_route_table_create $nic_i $vltag $skip_if_exists) || exit $?
[ -n "$rt_name" ] || return
oci_vcn_debug "route table add rules"
$IP route add default via $virtrt dev $iface table $rt_name || oci_vcn_err "cannot add default route via $virtrt on $iface to table $rt_name"
$IP route add $scidr dev $iface table $rt_name || oci_vcn_err "cannot add subnet $scidr route on $iface to table $rt_name"
# create source-based rule to use table
oci_vcn_debug "src rule add"
$IP rule add from $addr lookup $rt_name || oci_vcn_err "cannot add rule from $addr use table $rt_name"
oci_vcn_info "added rule for routing from $addr lookup $rt_name with default via $virtrt"
fi
}
oci_vcn_ip_routing_del() {
local -ir ip_i=$1
local -r ns="${IP_NSS[$ip_i]#$NA}"
local -r iface="${IP_IFACES[$ip_i]}"
# for namespaces the subnet and default routes will be auto deleted with the namespace
if [ -z "$ns" ]; then
# delete rule
local -r rt_name=$(oci_vcn_ip_route_table_name_ip_i $ip_i)
local had_rule=''
if $IP rule | grep -qsw $rt_name; then # rule(s) exists
# note that there may be secondary private ips rules using the same table
oci_vcn_debug "rules to table $rt_name delete"
while $IP rule del lookup $rt_name 2>/dev/null; do true; done
had_rule='t'
fi
# delete route table
local had_table
had_table=$(oci_vcn_ip_route_table_del $rt_name) || exit $?
([ -z "$had_rule" ] && [ -z "$had_table" ]) || oci_vcn_info "removed routing on interface $iface"
fi
}
oci_vcn_ip_routing_sec_addr_add() {
local -ir iface_ip_i=$1
local -r addr=$2
local -r ns="${IP_NSS[$iface_ip_i]#$NA}"
# need to add source rule so reuse the routing table from the primary private ip
# not needed if in namespace
# not needed if on pri iface (uses main route table for dest and default route)
if [ -z "$ns" ] && [ $iface_ip_i -ne 0 ]; then
# create source-based rule to use iface's routing table
local -r rt_name=$(oci_vcn_ip_route_table_name_ip_i $iface_ip_i)
oci_vcn_debug "sec addr src rule add"
$IP rule add from $addr lookup $rt_name || oci_vcn_err "cannot add rule from secondary $addr use table $rt_name"
oci_vcn_info "added rule for routing from secondary IP address $addr lookup $rt_name"
fi
}
oci_vcn_ip_routing_sec_addr_del() {
local -ir ip_i=$1
local -r addr=$2
local -r ns="${IP_NSS[$ip_i]#$NA}"
# deconfig the routing (see comments in add)
if [ -z "$ns" ] && [ $ip_i -ne 0 ]; then
$IP rule del from $addr 2>/dev/null # ok if already deleted
oci_vcn_info "deleted rule for routing from secondary IP address $addr"
fi
}
oci_vcn_ip_routes_read() {
#ip r list 192.168.1.0/24
local -ir ip_i=$1
local -r ns="${IP_NSS[$ip_i]#$NA}"
local -r iface="${IP_IFACES[$ip_i]}"
local nscmd=''
local virtrt="$NA"
if [ -n "$ns" ]; then
nscmd="netns exec $ns $IP"
else
# no namespace: check for route table
local -r rt_name=$(oci_vcn_ip_route_table_name_ip_i $ip_i)
local rt_exists
rt_exists=$(oci_vcn_ip_route_table_exists $rt_name) || exit $?
if [ -n "$rt_exists" ]; then # exists
# check for rule
if $IP rule | grep -qsw $rt_name; then # rule exists that uses route table
# look for default route in table, note table may exist but be empty
# "default via 10.0.0.1 dev ens3"
local -a def_entry
def_entry=($($IP route show table $rt_name | grep -sw ^default))
if [ -n "${def_entry[2]}" ]; then
virtrt="${def_entry[2]}"
oci_vcn_debug "default route: $virtrt"
else # emtpy table: delete rule and table
virtrt="$NA"
oci_vcn_ip_routing_del $ip_i
fi
else # clean up route table since no rule uses it
$(oci_vcn_ip_route_table_del $rt_name)
fi
fi
fi
# read the routes
local sprefix="$NA"
local sbits="$NA"
local src="$NA"
# mapfile will create array with each line an element
mapfile -t routes < <($IP $nscmd route | grep -w $iface) || oci_vcn_err "cannot read IP routes for interface $iface"
local line
for line in "${routes[@]}"; do
local -a route=($line)
if [ "${route[0]}" = 'default' ]; then
# "default via 10.0.0.1 dev ens3"
virtrt="${route[2]}"
oci_vcn_debug "default route via: $virtrt"
elif [ "${route[0]#169.}" = "${route[0]}" ]; then # not cavium route
# "10.0.0.0/24 dev ens3 proto kernel scope link src 10.0.0.2"
local -i i
for i in $(seq 0 $((${#route[@]} - 1))); do
local x="${route[$i]}"
if [ $i -eq 0 ]; then
sprefix=${x%/*}
sbits=${x#*/}
if [ "$sprefix" = "$sbits" ]; then # not valid line
sprefix="$NA"
sbits="$NA"
break
else
oci_vcn_debug "subnet route: $sprefix/$sbits"
fi
elif [ "$x" = 'src' ]; then src="$YES"
fi
done
fi
done
IP_VIRTRTS[$ip_i]="$virtrt"
IP_SADDRS[$ip_i]="$sprefix"
IP_SBITSS[$ip_i]="$sbits"
IP_SRCS[$ip_i]="$src"
}
oci_vcn_macvlan_name() {
local -r iface=$1
local -r vltag=$2
eval echo "$MACVLAN_FORMAT"
}
oci_vcn_vlan_name() {
local -r iface=$1
local -r vltag=$2
eval echo "$VLAN_FORMAT"
}
oci_vcn_ip_ns_name() {
local -ir nic=$1 # format looks for "${nic}"
local -ir vltag=$2
if [ -n "$USE_NS" ] && [ -z "$NS_FORMAT" ]; then
if [ -n "$IS_VM" ]; then NS_FORMAT="$DEF_NS_FORMAT_VM"
else NS_FORMAT="$DEF_NS_FORMAT_BM"; fi
fi
eval echo "$NS_FORMAT"
}
oci_vcn_ip_ns_svcs_stop() {
local -r ns=$1
local pids
pids=$($IP netns pids $ns) || oci_vcn_err "cannot get ids for processes in namespace $ns"
if [ -n "$pids" ]; then
kill -TERM $pids || oci_vcn_err "cannot terminate namespace $ns processes: $pids"
oci_vcn_info "terminated namespace $ns processes: $pids"
fi
}
oci_vcn_ip_ns_svcs_start() {
local -r ns=$1
if [ -n "$START_SSHD" ]; then # start SSH daemon
$IP netns exec $ns $SSHD || oci_vcn_err "cannot start ssh daemon"
oci_vcn_info "started namespace $ns ssh daemon"
fi
}
oci_vcn_ip_ns_del() {
local -r ns=$1
# note also deletes vlans and routes
$IP netns del $ns || oci_vcn_err "cannot delete namespace $ns"
oci_vcn_info "deleted namespace $ns"
}
oci_vcn_ip_ns_create() {
local -ir nic_i=$1
local -ir vltag=$2
[ -n "$MODPROBE" ] || oci_vcn_err "cannot find modprobe command"
$MODPROBE 8021q || oci_vcn_err "failed to load 8021q module"
local ns
ns=$(oci_vcn_ip_ns_name $nic_i $vltag) || exit $?
$IP netns add $ns || oci_vcn_err "cannot create namespace $ns"
oci_vcn_info "created namespace $ns"
echo "$ns"
}
oci_vcn_ip_addr_add_iface() {
local -ir md_i=$1
local -ir ip_i=$2 # index of physical iface/nic
local -r ns=$3
local iface="${IP_IFACES[$ip_i]}"
local -r physns="${IP_NSS[$ip_i]#$NA}"
local -r mac="${MD_MACS[$md_i]}"
local -ir vltag="${MD_VLTAGS[$md_i]}"
local -r addr="${MD_ADDRS[$md_i]}"
local -r sbits="${MD_SBITSS[$md_i]}"
local vlan=''
# must be adding to physical iface/nic
[ -z "${IP_VLANS[$ip_i]#$NA}" ] || oci_vcn_err "cannot add IP address $addr to virtual interface ${IP_VLANS[$ip_i]}"
# create virtual interface if needed (bm cases)
local macvlan=''
if [ -z "$IS_VM" ] && [ $vltag -ne 0 ]; then
# bm vnics need a virtual iface except for vltag=0
# if physical iface/nic is in a namespace we must go there to create
local physnscmd=''
if [ -n "$physns" ]; then
physnscmd="netns exec $physns $IP"
fi
# create a mac vlan from physical iface/nic
oci_vcn_debug "macvlan link add $macvlan"
macvlan=$(oci_vcn_macvlan_name $iface $vltag) || exit $?
$IP $physnscmd link add link $iface name $macvlan address $mac type macvlan \
|| oci_vcn_err "cannot create MAC VLAN interface $macvlan for MAC address $mac"
# if physical iface/nic is in a namespace pull out the created mac vlan
if [ -n "$physns" ]; then
$IP $physnscmd link set $macvlan netns 1
fi
# create an ip vlan on top of the mac vlan
oci_vcn_debug "vlan link add"
vlan=$(oci_vcn_vlan_name $iface $vltag) || exit $?
$IP link add link $macvlan name $vlan type vlan id $vltag \
|| oci_vcn_err "cannot create VLAN $vlan on MAC VLAN $macvlan"
fi
# use namespace, if option
local nscmd=''
local -r dev="${vlan:-$iface}"
if [ -n "$ns" ]; then
nscmd="netns exec $ns $IP"
# move the iface(s) to the target namespace
if [ -n "$macvlan" ]; then
oci_vcn_debug "macvlan link move $ns"
$IP link set dev $macvlan netns $ns || oci_vcn_err "cannot move MAC VLAN $macvlan into namespace $ns"
fi
oci_vcn_debug "$dev link move $ns"
$IP link set dev $dev netns $ns || oci_vcn_err "cannot move interface $dev into namespace $ns"
fi
# add IP address to iface:
# the netmask is specified to create a route in the main route table
# if this iface is on the same subnet as another iface then the route will not have an effect
# (until the other iface is removed)
oci_vcn_debug "addr $addr/$sbits add on $dev ns '$ns'"
$IP $nscmd addr add $addr/$sbits dev $dev || oci_vcn_err "cannot add IP address $addr/$sbits on interface $dev"
if [ -n "$macvlan" ]; then # set vlans up
oci_vcn_debug "vlans set up"
$IP $nscmd link set dev $macvlan mtu $MTU up || oci_vcn_err "cannot set MAC VLAN $macvlan up"
$IP $nscmd link set dev $vlan mtu $MTU up || oci_vcn_err "cannot set VLAN $vlan up"
else
oci_vcn_debug "$iface set up"
$IP $nscmd link set dev $iface mtu $MTU up || oci_vcn_err "cannot set interface $iface MTU"
fi
oci_vcn_info "added IP address $addr on interface $dev with MTU $MTU"
echo "$dev"
}
oci_vcn_ip_addr_del_iface() {
local -ir ip_i=$1
local -r iface="${IP_IFACES[$ip_i]}"
local -r ns="${IP_NSS[$ip_i]#$NA}"
local -r vlan="${IP_VLANS[$ip_i]#$NA}"
local -r secad="${IP_SECADS[$ip_i]#$NA}"
local nscmd=''
if [ -n "$ns" ]; then
nscmd="netns exec $ns $IP"
fi
if [ "$secad" != "$YES" ] && [ -n "$vlan" ]; then
# delete vlan and macvlan, removes the addrs (pri and sec) as well
oci_vcn_debug "$ns link delete"
local -ir vltag="${IP_VLTAGS[$ip_i]}"
local macvlan
macvlan=$(oci_vcn_macvlan_name $iface $vltag) || exit $?
$IP $nscmd link del link $vlan dev $macvlan || oci_vcn_err "cannot remove VLAN $vlan"
oci_vcn_info "removed VLAN $vlan"
else
# delete addr from phys iface
# deleting namespace will move phys iface back to main
# note that we may be deleting sec addr from a vlan here
local -r addr="${IP_ADDRS[$ip_i]#$NA}"
local -r dev="${vlan:-$iface}"
local bits="${IP_SBITSS[$ip_i]#$NA}"
[ "$secad" != "$YES" ] || bits=32
oci_vcn_debug "addr $addr del ns '$ns' dev $dev"
$IP $nscmd addr del $addr/$bits dev $dev || oci_vcn_err "cannot remove IP address $addr/$bits from interface $dev"
oci_vcn_info "removed IP address $addr from interface $dev"
fi
}
oci_vcn_ip_addr_add() {
local -ir md_i=$1
local -r mac="${MD_MACS[$md_i]}"
local -r addr="${MD_ADDRS[$md_i]}"
local ns=''
local iface=''
local -i nic_i
local -i vltag
# note that when adding an addr to a physical iface ip_i will be the index of the
# addr, but when creating a vlan for addr ip_i will not be the vlan iface but its phys one
local -i ip_i
if [ -z "$IS_VM" ]; then
# bm vnics' physical ifaces are identified by nic index
nic_i=${MD_NIC_IS[$md_i]}
[ $nic_i -lt ${#NIC_IP_IS[@]} ] || oci_vcn_err "cannot find interface for NIC $nic_i"
ip_i=${NIC_IP_IS[$nic_i]}
iface="${IP_IFACES[$ip_i]}"
vltag=${MD_VLTAGS[$md_i]}
else
# vm vnics' physical ifaces are identified by matching mac
local found=''
ip_i=0
local ip_mac
for ip_mac in "${IP_MACS[@]}"; do
if [ "$ip_mac" = "$mac" ]; then
found='t'
break
fi
ip_i+=1
done
[ -n "$found" ] || oci_vcn_err "cannot find interface matching VNIC MAC $mac"
iface="${IP_IFACES[$ip_i]}"
nic_i=${IP_NIC_IS[$ip_i]}
vltag=0
fi
# check that there is no current addr on iface
if [ -n "$IS_VM" ] || [ "${MD_VLTAGS[$md_i]}" = "0" ]; then # putting addr directly on iface
local -r ip_addr="${IP_ADDRS[$ip_i]#$NA}"
[ -z "$ip_addr" ] || oci_vcn_err "IP address $ip_addr already added on interface $iface"
fi
# make sure physical iface/nic is up
if [ "${IP_STATES[$ip_i]}" != "UP" ]; then
$IP link set dev $iface up || oci_vcn_err "cannot set interface $iface up"
fi
# create namespace if requested
local ns=''
if [ -n "$USE_NS" ]; then
ns=$(oci_vcn_ip_ns_create $nic_i $vltag) || exit $?
# if working on a physical iface we need to set its namespace so that any
# vlan created subsequently will know how to find it
[ $vltag -ne 0 ] || IP_NSS[$ip_i]="$ns"
fi
# add addr to iface (possibly creating vlan)
local dev
dev=$(oci_vcn_ip_addr_add_iface $md_i $ip_i $ns) || exit $?
# setup routes
oci_vcn_ip_routing_add $md_i $nic_i $dev $ns
# if namespace then wait for changes and start services
if [ -n "$ns" ]; then
sleep 1 # namespace changes seem to take time
oci_vcn_ip_ns_svcs_start $ns
fi
}
oci_vcn_ip_sec_addr_add() {
local -ir iface_ip_i=$1
local -r addr=$2
local -r iface="${IP_IFACES[$iface_ip_i]}"
local -r vlan="${IP_VLANS[$iface_ip_i]#$NA}"
local -r dev="${vlan:-$iface}"
local -r ns="${IP_NSS[$iface_ip_i]#$NA}"
local nscmd=''
local nsinfo=''
# config routing
oci_vcn_ip_routing_sec_addr_add $iface_ip_i $addr
# config the addr on the iface
if [ -n "$ns" ]; then
nscmd="netns exec $ns $IP"
nsinfo=" in namespace $ns"
fi
oci_vcn_info "adding secondary IP address $addr to interface (or VLAN) $dev$nsinfo"
$IP $nscmd addr add $addr/32 dev $dev || oci_vcn_err "cannot add secondary IP address $addr on interface $dev$nsinfo"
}
oci_vcn_ip_sec_addr_del() {
local -ir ip_i=$1
local -r deconfig_all=$2
local -r addr=${IP_ADDRS[$ip_i]}
local -r iface=${IP_IFACES[$ip_i]}
local -r vlan="${IP_VLANS[$ip_i]#$NA}"
local -r dev="${vlan:-$iface}"
local -r ns="${IP_NSS[$ip_i]#$NA}"
local nscmd=''
local nsinfo=''
[ "${IP_SECADS[$ip_i]}" = "$YES" ] || oci_vcn_err "not a secondary IP address: $addr"
# remove the addr on the iface (see comments in add)
# no need if deconfig all and on a vlan or in namespace
if [ -z "$deconfig_all" ] || ([ -z "$vlan" ] && [ -z "$ns" ]); then
if [ -z "$deconfig_all" ] && [ -n "$ns" ]; then
nscmd="netns exec $ns $IP"
nsinfo=" in namespace $ns"
fi
oci_vcn_info "removing secondary IP address $addr from interface (or VLAN) $dev$nsinfo"
$IP $nscmd addr del $addr/32 dev $dev || oci_vcn_err "cannot remove IP address $addr on interface $dev"
fi
# remove the routing (see comments in add)
oci_vcn_ip_routing_sec_addr_del $ip_i $addr
}
oci_vcn_sec_addr_is_provisioned() {
local -r find_addr=$1
local -r find_vnic=$2
local -i i
local found=''
for i in $(seq 0 $((${#SEC_ADDRS[@]} - 1))); do
local addr=${SEC_ADDRS[$i]}
local vnic=${SEC_VNICS[$i]}
if [ "$find_addr" = "$addr" ] && [ "$find_vnic" = "$vnic" ]; then
found='t'
break
fi
done
echo "$found"
}
oci_vcn_ip_addr_del() {
local -ir ip_i=$1
local -r ns="${IP_NSS[$ip_i]#$NA}"
local -r secad="${IP_SECADS[$ip_i]#$NA}"
[ $ip_i -ne 0 ] || oci_vcn_err "cannot remove primary VNIC"
if [ "$secad" != "$YES" ]; then
if [ -n "$ns" ]; then
# stop services in namespace
oci_vcn_ip_ns_svcs_stop $ns
fi
# remove routing
oci_vcn_ip_routing_del $ip_i
fi
# remove addr
oci_vcn_ip_addr_del_iface $ip_i
if [ "$secad" != "$YES" ] && [ -n "$ns" ]; then
# delete namespace
oci_vcn_ip_ns_del $ns
sleep 1 # namespace changes seem to take time
fi
}
oci_vcn_ip_ifaces_read() {
local -r ns="$1"
local nscmd=''
local -a iface_datas
if [ -n "$ns" ]; then # change ip command to use namespace
nscmd="netns exec $ns $IP"
fi
# read the interfaces in namespace (if any)
# mapfile will create array with each line an element
mapfile -t iface_datas < <($IP $nscmd addr show | awk -f $IFACE_AWK_SCRIPT) || oci_vcn_err "cannot read IP addresses"
if [ ${#iface_datas[@]} -eq 0 ]; then
# if reading physical ifaces, must be at least 1
[ -n "$ns" ] || oci_vcn_err "cannot locate interfaces"
# empty namespace: probably result of a VM VNIC delete
# note empty namespaces do not survive reboot
$IP netns del $ns || oci_vcn_err "cannot delete empty namespace $ns"
oci_vcn_warn "deleted empty namespace $ns"
else
local -r nsna="${ns:-$NA}"
local -i ip_i=${#IP_MACS[@]} # continue from previous ns (if any)
local line
for line in "${iface_datas[@]}"; do
# line items are in order printed by awk script print
# note that $NA is used to mean null (i.e. not set)
oci_vcn_debug "iface line: $line"
local -a iface_data=($line)
local iface="${iface_data[1]}"
# filter out virtual interfaces if VM (assume created by user for other purpose)
if [ -z "$IS_VM" ] || [ -z "${VIRTUAL_IFACES[$iface]}" ]; then
local mac="${iface_data[0]}"
IP_MACS+=("$mac")
IP_NSS+=("$nsna")
IP_IFACES+=("$iface")
IP_ADDRS+=("${iface_data[2]}")
IP_SBITSS+=("${iface_data[3]}")
IP_STATES+=("${iface_data[4]}")
IP_VLANS+=("${iface_data[5]}")
IP_VLTAGS+=("${iface_data[6]}")
local secad="${iface_data[7]}"
IP_SECADS+=("$secad")
[ "$secad" = "$YES" ] || IP_I_BY_MAC["$mac"]=$ip_i # primary addrs only
ip_i+=1
fi
done
fi
}
oci_vcn_ip_read() {
IP_I_BY_MAC=()
IP_MACS=()
IP_NSS=()
IP_IFACES=()
IP_ADDRS=()
IP_SADDRS=()
IP_SBITSS=()
IP_VIRTRTS=()
IP_STATES=()
IP_VLANS=()
IP_VLTAGS=()
IP_SECADS=()
IP_SRCS=()
IP_NIC_IS=()
NIC_IP_IS=()
NIC_I_BY_PHYS_IP_I=()
# read the non-namespace ifaces and any addrs on them
oci_vcn_ip_ifaces_read
# read ifaces in all the namespaces (if any), note some os's don't support netns
mapfile -t nss < <($IP netns 2>/dev/null)
local ns
for ns in "${nss[@]}"; do
oci_vcn_ip_ifaces_read $ns
done
# set vltag 0 (for phys ifaces) and read os routes for all ifaces
local -i ip_i
for ip_i in $(seq 0 $((${#IP_MACS[@]} - 1))); do
# any iface w/o vltag has tag 0
[ -n "${IP_VLTAGS[$ip_i]#$NA}" ] || IP_VLTAGS[$ip_i]=0
oci_vcn_ip_routes_read $ip_i
done
# find the "physical" iface indices: vltag 0 and not secondary addr
# note vm ifaces are considered physical
# these may not be in nic index order due to inclusion in namespaces (see next)
local -r tmpfile=$(mktemp /tmp/oci_vcn_ifaces.XXXXX)
local -A ip_i_by_phys_iface
for ip_i in $(seq 0 $((${#IP_MACS[@]} - 1))); do
if [ ${IP_VLTAGS[$ip_i]} -eq 0 ] && [ "${IP_SECADS[$ip_i]}" != "$YES" ]; then
# physical iface/nic
local iface=${IP_IFACES[$ip_i]}
ip_i_by_phys_iface[$iface]=$ip_i
echo "$iface" >> "$tmpfile"
fi
NIC_I_BY_PHYS_IP_I[$ip_i]=-1
done
# sort physical ifaces by first number (either at end or in middle) in iface name
# this will provide the nic index
local -i nic_i=0
local iface
for iface in $(cat "$tmpfile" | awk -- '{ match($1, /[0-9]+/); print substr($1, RSTART, RLENGTH), $1 }' | sort -n | cut -f 2 -d ' '); do
local ip_i=${ip_i_by_phys_iface[$iface]}
NIC_IP_IS+=($ip_i)
NIC_I_BY_PHYS_IP_I[$ip_i]=$nic_i
nic_i+=1
done
rm "$tmpfile"
# for each iface (phys or vlan) get nic index
for ip_i in $(seq 0 $((${#IP_MACS[@]} - 1))); do
local iface=${IP_IFACES[$ip_i]}
local -i phys_ip_i=${ip_i_by_phys_iface[$iface]}
IP_NIC_IS[$ip_i]=${NIC_I_BY_PHYS_IP_I[$phys_ip_i]}
done
# fix up missing nic index for bms (see also oci_vcn_md_read): bms will be missing nic index metadata
# if they were created before and have not had a vnic attached or detached since 8/23/17.
# if there is a secondary vnic created and configured (before 8/23/17) we can tell it is a bm
# because there will either be configured vlans or more vnics than physical ifaces.
# note that gen 2 shapes are post 8/23/17 and, hence, will have nic indices already set.
# a missing nic index for an old bm will not matter if there are no vnics to configure.
if [ ${#MD_NIC_IS[@]} -eq 0 ] && [ ${#NIC_IP_IS[@]} -eq 1 ] && \
([ ${#IP_MACS[@]} -gt 1 ] || [ ${#MD_MACS[@]} -gt 1 ]); then # configured or new secondaries
local -i md_i
for md_i in $(seq 0 $((${#MD_MACS[@]} - 1))); do
MD_NIC_IS+=(0)
done
oci_vcn_info "legacy BM instance detected"
IS_VM=''
fi
}
oci_vcn_read() {
# assumes md info is already read
# reads ip configs and creates single array of all macs
# (fixes vm interfaces if random mac)
[ -n "$IP" ] || oci_vcn_err "cannot find ip command"