Skip to content

Commit

Permalink
refactor
Browse files Browse the repository at this point in the history 
Signed-off-by: forrestchen <forrestchen@tencent.com>
  • Loading branch information
@ChenLingPeng @chenchun
ChenLingPeng authored and chenchun committed Oct 22, 2019
1 parent 2421eb6 commit 2a501e2
Show file tree
Hide file tree
Showing 11 changed files with 268 additions and 217 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Expand Up @@ -2,6 +2,7 @@
/.idea
*.iml
/.vscode
/vendor
/go
/tools/route_monitor/route_monitor
/vendor/github.com/docker/distribution/contrib
Expand Down
125 changes: 65 additions & 60 deletions cni/k8s-sriov/k8s_sriov.go
View file
Expand Up @@ -17,8 +17,8 @@ import (
t020 "github.com/containernetworking/cni/pkg/types/020"
"github.com/containernetworking/cni/pkg/version"
"github.com/containernetworking/plugins/pkg/ns"
glog "k8s.io/klog"
"github.com/vishvananda/netlink"
glog "k8s.io/klog"
)

type NetConf struct {
Expand Down Expand Up @@ -107,112 +107,117 @@ func main() {
func setupVF(conf *NetConf, result *t020.Result, podifName string, vlan int, netns ns.NetNS) error {
cpus := runtime.NumCPU()
ifName := conf.Device
var vfIdx int
var infos []os.FileInfo

m, err := netlink.LinkByName(ifName)
if err != nil {
return fmt.Errorf("failed to lookup master %q: %v", conf.Device, err)
}

// get the ifname sriov vf num
vfTotal, err := getSriovNumVfs(ifName, kindTotalVfs)
minVfNum, err := getSriovVfNum(conf)
if err != nil {
return err
}

if vfTotal <= 0 {
return fmt.Errorf("no virtual function in the device %q", ifName)
}

vfNums, err := getSriovNumVfs(ifName, kindNumVfs)
infos, vfIdx, vfDev, vfName, err := findAvailableVf(minVfNum, conf)
if err != nil {
return err
}

minVfNum := min(vfTotal, conf.VFNum)
// only set vf when `sriov_numvfs` is 0
if vfNums == 0 {
if err := setSriovNumVfs(ifName, minVfNum); err != nil {
return err
if err = netlink.LinkSetVfVlan(m, vfIdx, vlan); err != nil {
return fmt.Errorf("failed to set vf %d vlan: %v", vfIdx, err)
}
if err = netlink.LinkSetUp(vfDev); err != nil {
return fmt.Errorf("failed to setup vf %d device: %v", vfIdx, err)
}
// move VF device to ns
if err = netlink.LinkSetNsFd(vfDev, int(netns.Fd())); err != nil {
return fmt.Errorf("failed to move vf %d to netns: %v", vfIdx, err)
}
if err = netns.Do(func(_ ns.NetNS) error {
err := renameLink(vfName, podifName)
if err != nil {
return fmt.Errorf("failed to rename %d vf of the device %q to %q: %v", vfIdx, vfName, ifName, err)
}
} else if vfNums < minVfNum {
glog.Warning("sriov_numvfs is set but small")
return cniutil.ConfigureIface(podifName, result)
}); err != nil {
return err
}
hiDir := fmt.Sprintf("/sys/class/net/%s/device/virtfn%d/msi_irqs", ifName, vfIdx)
if infos, err = ioutil.ReadDir(hiDir); err == nil {
for i := range infos {
hiNum, err := strconv.Atoi(infos[i].Name())
if err == nil {
selectedCPU := fmt.Sprintf("%d", hiNum%cpus)
irqFile := fmt.Sprintf("/proc/irq/%d/smp_affinity_list", hiNum)
if err = ioutil.WriteFile(irqFile, []byte(selectedCPU), 0644); err != nil {
return fmt.Errorf("failed set irq smp affinity: %v", err)
}
}
}
}
return nil
}

func findAvailableVf(minVfNum int, conf *NetConf) ([]os.FileInfo, int, netlink.Link, string, error) {
ifName := conf.Device
var infos []os.FileInfo
var err error
var vfIdx int
for vf := 0; vf <= (minVfNum - 1); vf++ {
vfDir := fmt.Sprintf("/sys/class/net/%s/device/virtfn%d/net", ifName, vf)
if _, err := os.Lstat(vfDir); err != nil {
if vf == (minVfNum - 1) {
return fmt.Errorf("failed to open the virtfn%d dir of the device %q: %v", vf, ifName, err)
return nil, 0, nil, "", fmt.Errorf("failed to open the virtfn%d dir of the device %q: %v", vf, ifName, err)
}
continue
}

infos, err = ioutil.ReadDir(vfDir)
if err != nil {
return fmt.Errorf("failed to read the virtfn%d dir of the device %q: %v", vf, ifName, err)
return nil, 0, nil, "", fmt.Errorf("failed to read the virtfn%d dir of the device %q: %v", vf, ifName, err)
}

if (len(infos) == 0) && (vf == (minVfNum - 1)) {
return fmt.Errorf("no Virtual function exist in directory %s, last vf is virtfn%d", vfDir, vf)
return nil, 0, nil, "", fmt.Errorf("no Virtual function exist in directory %s, last vf is virtfn%d", vfDir, vf)
}

if (len(infos) == 0) && (vf != (minVfNum - 1)) {
continue
}
vfIdx = vf
break

}

// VF NIC name
if len(infos) != 1 {
return fmt.Errorf("no virtual network resources available for the %q", conf.Device)
return nil, 0, nil, "", fmt.Errorf("no virtual network resources available for the %q", ifName)
}
vfName := infos[0].Name()

vfDev, err := netlink.LinkByName(vfName)
if err != nil {
return fmt.Errorf("failed to lookup vf device %q: %v", vfName, err)
}

if err = netlink.LinkSetVfVlan(m, vfIdx, vlan); err != nil {
return fmt.Errorf("failed to set vf %d vlan: %v", vfIdx, err)
return nil, 0, nil, "", fmt.Errorf("failed to lookup vf device %q: %v", vfName, err)
}
return infos, vfIdx, vfDev, vfName, nil
}

if err = netlink.LinkSetUp(vfDev); err != nil {
return fmt.Errorf("failed to setup vf %d device: %v", vfIdx, err)
func getSriovVfNum(conf *NetConf) (int, error) {
ifName := conf.Device
vfTotal, err := getSriovNumVfs(ifName, kindTotalVfs)
if err != nil {
return 0, err
}

// move VF device to ns
if err = netlink.LinkSetNsFd(vfDev, int(netns.Fd())); err != nil {
return fmt.Errorf("failed to move vf %d to netns: %v", vfIdx, err)
if vfTotal <= 0 {
return 0, fmt.Errorf("no virtual function in the device %q", ifName)
}

if err = netns.Do(func(_ ns.NetNS) error {
err := renameLink(vfName, podifName)
if err != nil {
return fmt.Errorf("failed to rename %d vf of the device %q to %q: %v", vfIdx, vfName, ifName, err)
}
return cniutil.ConfigureIface(podifName, result)
}); err != nil {
return err
vfNums, err := getSriovNumVfs(ifName, kindNumVfs)
if err != nil {
return 0, err
}
hiDir := fmt.Sprintf("/sys/class/net/%s/device/virtfn%d/msi_irqs", ifName, vfIdx)
if infos, err = ioutil.ReadDir(hiDir); err == nil {
for i := range infos {
hiNum, err := strconv.Atoi(infos[i].Name())
if err == nil {
selectedCPU := fmt.Sprintf("%d", hiNum%cpus)
irqFile := fmt.Sprintf("/proc/irq/%d/smp_affinity_list", hiNum)
if err = ioutil.WriteFile(irqFile, []byte(selectedCPU), 0644); err != nil {
return fmt.Errorf("failed set irq smp affinity: %v", err)
}
}
minVfNum := min(vfTotal, conf.VFNum)
// only set vf when `sriov_numvfs` is 0
if vfNums == 0 {
if err := setSriovNumVfs(ifName, minVfNum); err != nil {
return 0, err
}
} else if vfNums < minVfNum {
glog.Warning("sriov_numvfs is set but small")
}
return nil
return minVfNum, nil
}

func releaseVF(podifName string, netns ns.NetNS) error {
Expand Down
96 changes: 56 additions & 40 deletions cni/k8s-vlan/k8s_vlan.go
View file
Expand Up @@ -50,24 +50,9 @@ func cmdAdd(args *skel.CmdArgs) error {
if err := d.Init(); err != nil {
return fmt.Errorf("failed to setup bridge %v", err)
}
var result020s []*t020.Result
for i := 0; i < len(results); i++ {
result020, err := t020.GetResult(results[i])
if err != nil {
return err
}
result020s = append(result020s, result020)
}
if len(result020s) == 2 {
routes := result020s[0].IP4.Routes
for i := 0; i < len(routes); i++ {
if routes[i].Dst.String() == "0.0.0.0/0" {
routes = append(routes[:i], routes[i+1:]...)
break
}
}
routes = append(routes, types.Route{Dst: *pANet}, types.Route{Dst: *pBNet}, types.Route{Dst: *pCNet})
result020s[0].IP4.Routes = routes
result020s, err := resultConvert(results)
if err != nil {
return err
}

if d.MacVlanMode() {
Expand All @@ -88,28 +73,8 @@ func cmdAdd(args *skel.CmdArgs) error {
_ = utils.SendGratuitousARP(args.IfName, result020s[0].IP4.IP.IP.String(), args.Netns)
} else {
ifName := args.IfName
ifIndex := 0
for i := 0; i < len(result020s); i++ {
vlanId := vlanIds[i]
result020 := result020s[i]
bridgeName, err := d.CreateBridgeAndVlanDevice(vlanId)
if err != nil {
return err
}
suffix := ""
if i != 0 {
suffix = fmt.Sprintf("-%d", i+1)
ifIndex++
args.IfName = fmt.Sprintf("eth%d", ifIndex)
if args.IfName == ifName {
ifIndex++
args.IfName = fmt.Sprintf("eth%d", ifIndex)
}
}
if err := utils.VethConnectsHostWithContainer(result020, args, bridgeName, suffix); err != nil {
return err
}
_ = utils.SendGratuitousARP(args.IfName, result020s[0].IP4.IP.IP.String(), args.Netns)
if err := setupVlanDevice(result020s, vlanIds, args); err != nil {
return err
}
args.IfName = ifName
}
Expand All @@ -122,6 +87,57 @@ func cmdAdd(args *skel.CmdArgs) error {
return result020s[0].Print()
}

func setupVlanDevice(result020s []*t020.Result, vlanIds []uint16, args *skel.CmdArgs) error {
ifName := args.IfName
ifIndex := 0
for i := 0; i < len(result020s); i++ {
vlanId := vlanIds[i]
result020 := result020s[i]
bridgeName, err := d.CreateBridgeAndVlanDevice(vlanId)
if err != nil {
return err
}
suffix := ""
if i != 0 {
suffix = fmt.Sprintf("-%d", i+1)
ifIndex++
args.IfName = fmt.Sprintf("eth%d", ifIndex)
if args.IfName == ifName {
ifIndex++
args.IfName = fmt.Sprintf("eth%d", ifIndex)
}
}
if err := utils.VethConnectsHostWithContainer(result020, args, bridgeName, suffix); err != nil {
return err
}
_ = utils.SendGratuitousARP(args.IfName, result020s[0].IP4.IP.IP.String(), args.Netns)
}
return nil
}

func resultConvert(results []types.Result) ([]*t020.Result, error) {
var result020s []*t020.Result
for i := 0; i < len(results); i++ {
result020, err := t020.GetResult(results[i])
if err != nil {
return nil, err
}
result020s = append(result020s, result020)
}
if len(result020s) == 2 {
routes := result020s[0].IP4.Routes
for i := 0; i < len(routes); i++ {
if routes[i].Dst.String() == "0.0.0.0/0" {
routes = append(routes[:i], routes[i+1:]...)
break
}
}
routes = append(routes, types.Route{Dst: *pANet}, types.Route{Dst: *pBNet}, types.Route{Dst: *pCNet})
result020s[0].IP4.Routes = routes
}
return result020s, nil
}

func cmdDel(args *skel.CmdArgs) error {
if err := utils.DeleteAllVeth(args.Netns); err != nil {
return err
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Expand Up @@ -3,7 +3,6 @@ module git.code.oa.com/tkestack/galaxy
go 1.13

require (
git.tencent.com/tke/tapp-controller v0.0.0-20190903090859-2b03d3f3bbca
github.com/containernetworking/cni v0.6.0
github.com/containernetworking/plugins v0.6.0
github.com/coreos/go-iptables v0.4.3
Expand Down Expand Up @@ -41,6 +40,7 @@ require (
k8s.io/kube-openapi v0.0.0-20190918143330-0270cf2f1c1d // indirect
k8s.io/kubernetes v1.16.0-alpha.0
k8s.io/utils v0.0.0-20191010214722-8d271d903fe4
tkestack.io/tapp-controller v0.0.0-20191017074115-bdf9b4742398
)

replace (
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Expand Up @@ -3,8 +3,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU=
cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw=
git.tencent.com/tke/tapp-controller v0.0.0-20190903090859-2b03d3f3bbca h1:pTgOqCAdxIYRUqQZLlDrHjIh+HWZV+xvxvQhgPFmb5k=
git.tencent.com/tke/tapp-controller v0.0.0-20190903090859-2b03d3f3bbca/go.mod h1:/fPdei8YS80Q+zX6USM08f5JzP3QKkihr4DLPkgKxxM=
github.com/Azure/azure-sdk-for-go v21.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
Expand Down Expand Up @@ -530,4 +528,6 @@ sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:w
sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
tkestack.io/tapp-controller v0.0.0-20191017074115-bdf9b4742398 h1:vFEijJFTblJHxG9AMVgVtkt8YifkLzQ9Grxuox/+Y7c=
tkestack.io/tapp-controller v0.0.0-20191017074115-bdf9b4742398/go.mod h1:+wwXNBMe6xDAe6geS2UxNvrimW9fMeuh68o9JiA2JOQ=
vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI=
4 changes: 2 additions & 2 deletions pkg/ipam/schedulerplugin/floatingip_plugin_test.go
View file
Expand Up @@ -18,8 +18,6 @@ import (
"git.code.oa.com/tkestack/galaxy/pkg/ipam/floatingip"
"git.code.oa.com/tkestack/galaxy/pkg/ipam/schedulerplugin/util"
"git.code.oa.com/tkestack/galaxy/pkg/utils/database"
fakeTAppCli "git.tencent.com/tke/tapp-controller/pkg/client/clientset/versioned/fake"
tappInformer "git.tencent.com/tke/tapp-controller/pkg/client/informers/externalversions"
"github.com/jinzhu/gorm"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand All @@ -31,6 +29,8 @@ import (
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
fakeV1 "k8s.io/client-go/kubernetes/typed/core/v1/fake"
fakeTAppCli "tkestack.io/tapp-controller/pkg/client/clientset/versioned/fake"
tappInformer "tkestack.io/tapp-controller/pkg/client/informers/externalversions"
)

const (
Expand Down
2 changes: 1 addition & 1 deletion pkg/ipam/schedulerplugin/resync.go
View file
Expand Up @@ -13,13 +13,13 @@ import (
"git.code.oa.com/tkestack/galaxy/pkg/ipam/schedulerplugin/util"
"git.code.oa.com/tkestack/galaxy/pkg/utils/database"
"git.code.oa.com/tkestack/galaxy/pkg/utils/nets"
tappv1 "git.tencent.com/tke/tapp-controller/pkg/apis/tappcontroller/v1"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metaErrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
glog "k8s.io/klog"
tappv1 "tkestack.io/tapp-controller/pkg/apis/tappcontroller/v1"
)

func (p *FloatingIPPlugin) storeReady() bool {
Expand Down
4 changes: 2 additions & 2 deletions pkg/ipam/schedulerplugin/types.go
View file
Expand Up @@ -3,12 +3,12 @@ package schedulerplugin
import (
crd_clientset "git.code.oa.com/tkestack/galaxy/pkg/ipam/client/clientset/versioned"
list "git.code.oa.com/tkestack/galaxy/pkg/ipam/client/listers/galaxy/v1alpha1"
"git.tencent.com/tke/tapp-controller/pkg/client/clientset/versioned"
"git.tencent.com/tke/tapp-controller/pkg/client/listers/tappcontroller/v1"
extensionClient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/client-go/kubernetes"
appv1 "k8s.io/client-go/listers/apps/v1"
corev1lister "k8s.io/client-go/listers/core/v1"
"tkestack.io/tapp-controller/pkg/client/clientset/versioned"
"tkestack.io/tapp-controller/pkg/client/listers/tappcontroller/v1"
)

type PluginFactoryArgs struct {
Expand Down

0 comments on commit 2a501e2

Please sign in to comment.