diff --git a/cloud-controller-manager/alicloud/alicloud.go b/cloud-controller-manager/alicloud/alicloud.go index 81da45cac..cb75605bd 100644 --- a/cloud-controller-manager/alicloud/alicloud.go +++ b/cloud-controller-manager/alicloud/alicloud.go @@ -65,13 +65,15 @@ type CloudConfig struct { VpcID string `json:"vpcid"` Region string `json:"region"` ZoneID string `json:"zoneid"` - + VswitchID string `json:"vswitchid"` AccessKeyID string `json:"accessKeyID"` AccessKeySecret string `json:"accessKeySecret"` } } +var cfg CloudConfig + func init() { cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { @@ -80,7 +82,6 @@ func init() { keysecret = "" ) if config != nil { - var cfg CloudConfig if err := json.NewDecoder(config).Decode(&cfg); err != nil { return nil, err } @@ -153,7 +154,7 @@ func (c *Cloud) GetLoadBalancer(clusterName string, service *v1.Service) (status } return &v1.LoadBalancerStatus{ - Ingress: []v1.LoadBalancerIngress{{IP: lb.Address, Hostname: domain(service, lb)}}}, true, nil + Ingress: []v1.LoadBalancerIngress{{IP: lb.Address}}}, true, nil } // EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer @@ -210,7 +211,6 @@ func (c *Cloud) EnsureLoadBalancer(clusterName string, service *v1.Service, node Ingress: []v1.LoadBalancerIngress{ { IP: lb.Address, - Hostname: domain(service, lb), }, }, }, nil @@ -241,7 +241,6 @@ func (c *Cloud) UpdateLoadBalancer(clusterName string, service *v1.Service, node func (c *Cloud) EnsureLoadBalancerDeleted(clusterName string, service *v1.Service) error { glog.V(2).Infof("Alicloud.EnsureLoadBalancerDeleted(%v, %v, %v, %v, %v, %v)", clusterName, service.Namespace, service.Name, c.region, service.Spec.LoadBalancerIP, service.Spec.Ports) - glog.V(4).Infof("SERVICE: %+v\n",service) return c.climgr.LoadBalancers().EnsureLoadBalanceDeleted(service) } @@ -387,6 +386,12 @@ func (c *Cloud) DeleteRoute(clusterName string, route *cloudprovider.Route) erro // GetZone returns the Zone containing the current failure zone and locality region that the program is running in func (c *Cloud) GetZone() (cloudprovider.Zone, error) { + if cfg.Global.ZoneID != "" && cfg.Global.Region != "" { + return cloudprovider.Zone{ + Region: cfg.Global.Region, + FailureDomain: cfg.Global.ZoneID, + }, nil + } host, err := c.climgr.MetaData().InstanceID() if err != nil { return cloudprovider.Zone{}, errors.New(fmt.Sprintf("Alicloud.GetZone(): error execute c.meta.InstanceID(). message=[%s]", err.Error())) diff --git a/cloud-controller-manager/alicloud/alicloud_test.go b/cloud-controller-manager/alicloud/alicloud_test.go index ffe321718..37a673dfd 100644 --- a/cloud-controller-manager/alicloud/alicloud_test.go +++ b/cloud-controller-manager/alicloud/alicloud_test.go @@ -31,6 +31,7 @@ import ( "errors" "github.com/denverdino/aliyungo/ecs" "github.com/denverdino/aliyungo/common" + "k8s.io/kubernetes/pkg/cloudprovider" ) var keyid string @@ -52,54 +53,6 @@ var ( nodeName = "iZuf694l8lw6xvdx6gh7tkZ" ) -func TestLoadBalancerDomain(t *testing.T) { - domain := loadBalancerDomain("", "idd",string(DEFAULT_REGION)) - t.Log(domain) - if domain != "idd.cn-hangzhou.alicontainer.com" { - t.Fatal("TestLoadBalancerDomain fail") - } - domain = loadBalancerDomain("user", "idd",string(DEFAULT_REGION)) - t.Log(domain) - if domain != "user.idd.cn-hangzhou.alicontainer.com" { - t.Fatal("TestLoadBalancerDomain with user fail") - } -} - -func TestFromLoadBalancerDomain(t *testing.T) { - service := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-service", - UID: types.UID(serviceUID), - }, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeLoadBalancer, - SessionAffinity: v1.ServiceAffinityNone, - }, - Status: v1.ServiceStatus{ - LoadBalancer: v1.LoadBalancerStatus{ - Ingress: []v1.LoadBalancerIngress{ - { - IP: "1.1.1.1", - // indicate user defined loadbalancer - Hostname: loadBalancerDomain("my-service","lbid",string(DEFAULT_REGION)), - }, - }, - }, - }, - } - - lbid := fromLoadBalancerStatus(service) - if lbid != "lbid" { - t.Fatal("must equal to lbid") - } - t.Log(lbid) - service.Status.LoadBalancer.Ingress[0].Hostname = loadBalancerDomain("","lbid",string(DEFAULT_REGION)) - t.Log(lbid) - if lbid != "lbid" { - t.Fatal("short must equal to lbid") - } -} - func TestBase64(t *testing.T) { data := "YWJjCg==" key, err := b64.StdEncoding.DecodeString(data) @@ -188,7 +141,10 @@ func TestEnsureLoadBalancerBasic(t *testing.T) { if args.LoadBalancerName != "" { base[0].LoadBalancerName = args.LoadBalancerName return base, nil - } else { + } + if len(args.Tags)>0{ + base[0].LoadBalancerName = cloudprovider.GetLoadBalancerName(service) + }else { return nil, errors.New("loadbalancerid or loadbanancername must be specified.\n") } return base, nil @@ -371,7 +327,10 @@ func TestEnsureLoadBalancerHTTPS(t *testing.T) { if args.LoadBalancerName != "" { base[0].LoadBalancerName = args.LoadBalancerName return base, nil - } else { + } + if len(args.Tags)>0{ + base[0].LoadBalancerName = cloudprovider.GetLoadBalancerName(service) + }else { return nil, errors.New("loadbalancerid or loadbanancername must be specified.\n") } return base, nil @@ -552,7 +511,10 @@ func TestEnsureLoadBalancerWithPortChange(t *testing.T) { if args.LoadBalancerName != "" { base[0].LoadBalancerName = args.LoadBalancerName return base, nil - } else { + } + if len(args.Tags)>0{ + base[0].LoadBalancerName = cloudprovider.GetLoadBalancerName(service) + }else { return nil, errors.New("loadbalancerid or loadbanancername must be specified.\n") } return base, nil @@ -761,7 +723,10 @@ func TestEnsureLoadbalancerDeleted(t *testing.T) { if args.LoadBalancerName != "" { base[0].LoadBalancerName = args.LoadBalancerName return base, nil - } else { + } + if len(args.Tags)>0{ + base[0].LoadBalancerName = cloudprovider.GetLoadBalancerName(service) + }else { return nil, errors.New("loadbalancerid or loadbanancername must be specified.\n") } return base, nil @@ -832,6 +797,9 @@ func TestEnsureLoadBalancerDeleteWithUserDefined(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "my-service", UID: types.UID(serviceUID), + Annotations: map[string]string{ + //ServiceAnnotationLoadBalancerId: "idbllll", + }, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ @@ -845,8 +813,6 @@ func TestEnsureLoadBalancerDeleteWithUserDefined(t *testing.T) { Ingress: []v1.LoadBalancerIngress{ { IP: "1.1.1.1", - // indicate user defined loadbalancer - Hostname: loadBalancerDomain("my-service",base[0].LoadBalancerId,string(DEFAULT_REGION)), }, }, }, @@ -864,7 +830,10 @@ func TestEnsureLoadBalancerDeleteWithUserDefined(t *testing.T) { if args.LoadBalancerName != "" { base[0].LoadBalancerName = args.LoadBalancerName return base, nil - } else { + } + if len(args.Tags)>0{ + base[0].LoadBalancerName = cloudprovider.GetLoadBalancerName(service) + }else { return nil, errors.New("loadbalancerid or loadbanancername must be specified.\n") } return base, nil @@ -873,7 +842,7 @@ func TestEnsureLoadBalancerDeleteWithUserDefined(t *testing.T) { t.Logf("findloadbalancer, [%s]", loadBalancerId) return loadbalancerAttrib(&base[0]), nil }, - deleteLoadBalancer: func(loadBalancerId string) (err error) { + deleteLoadBalancer: func(loadBalancerId string) (err error) { base = []slb.LoadBalancerType{} return nil }, @@ -889,15 +858,8 @@ func TestEnsureLoadBalancerDeleteWithUserDefined(t *testing.T) { t.Errorf("TestEnsureLoadBalancerDeleteWithUserDefined error: %s\n", e.Error()) } t.Log(PrettyJson(base)) - if len(base) != 1 { - t.Fatal("TestEnsureLoadBalancerDeleteWithUserDefined error, expected not to be deleted") - } - - base = newBaseLoadbalancer() - service.Status.LoadBalancer.Ingress[0].Hostname = loadBalancerDomain("",base[0].LoadBalancerId,string(DEFAULT_REGION)) - e = cloud.EnsureLoadBalancerDeleted(clusterName, service) if len(base) != 0 { - t.Fatal("TestEnsureLoadBalancerDeleteWithUserDefined error, expected to be deleted") + t.Fatal("TestEnsureLoadBalancerDeleteWithUserDefined error, expected not to be deleted") } } diff --git a/cloud-controller-manager/alicloud/clientmgr.go b/cloud-controller-manager/alicloud/clientmgr.go index fb5082936..c2b621b4a 100644 --- a/cloud-controller-manager/alicloud/clientmgr.go +++ b/cloud-controller-manager/alicloud/clientmgr.go @@ -26,6 +26,8 @@ import ( "github.com/golang/glog" "github.com/patrickmn/go-cache" "k8s.io/apimachinery/pkg/util/wait" + "fmt" + "strings" ) var ROLE_NAME = "KubernetesMasterRole" @@ -52,7 +54,7 @@ type ClientMgr struct { token *TokenAuth - meta *metadata.MetaData + meta IMetaData routes *RoutesClient loadbalancer *LoadBalancerClient instance *InstanceClient @@ -66,7 +68,7 @@ func NewClientMgr(key, secret string) (*ClientMgr, error) { }, active: false, } - m := metadata.NewMetaData(nil) + m := NewMetaData() if key == "" || secret == "" { if rolename, err := m.RoleName(); err != nil { @@ -142,7 +144,169 @@ func (c *ClientMgr) LoadBalancers() *LoadBalancerClient { return c.loadbalancer } -func (c *ClientMgr) MetaData() *metadata.MetaData { +func (c *ClientMgr) MetaData() IMetaData { return c.meta } + +type IMetaData interface { + HostName()(string, error) + ImageID() (string, error) + InstanceID() (string, error) + Mac() (string, error) + NetworkType() (string, error) + OwnerAccountID() (string, error) + PrivateIPv4() (string, error) + Region() (string, error) + SerialNumber() (string, error) + SourceAddress() (string, error) + VpcCIDRBlock() (string, error) + VpcID() (string, error) + VswitchCIDRBlock() (string, error) + Zone() (string, error) + NTPConfigServers() ([]string, error) + RoleName() (string, error) + RamRoleToken(role string) (metadata.RoleAuth, error) + VswitchID() (string, error) +} + +func NewMetaData() IMetaData{ + if cfg.Global.Region != "" && + cfg.Global.VpcID != "" && + cfg.Global.VswitchID != "" && + cfg.Global.ZoneID != "" { + glog.V(2).Infof("use mocked metadata server.") + return &fakeMetaData{base: metadata.NewMetaData(nil)} + } + return metadata.NewMetaData(nil) +} + +type fakeMetaData struct { + base IMetaData +} + +func (m *fakeMetaData) HostName() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) ImageID() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) InstanceID() (string, error) { + + return "fakedInstanceid",nil +} + +func (m *fakeMetaData) Mac() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) NetworkType() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) OwnerAccountID() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) PrivateIPv4() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) Region() (string, error) { + if cfg.Global.Region != "" { + return cfg.Global.Region, nil + } + return m.base.Region() +} + +func (m *fakeMetaData) SerialNumber() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) SourceAddress() (string, error) { + + return "",fmt.Errorf("unimplemented") + +} + +func (m *fakeMetaData) VpcCIDRBlock() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) VpcID() (string, error) { + + return cfg.Global.VpcID,nil +} + +func (m *fakeMetaData) VswitchCIDRBlock() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +// zone1:vswitchid1,zone2:vswitch2 +func (m *fakeMetaData) VswitchID() (string, error) { + + zlist := strings.Split(cfg.Global.VswitchID,",") + if len(zlist) == 1 { + glog.Infof("simple vswitchid mode, %s",cfg.Global.VswitchID) + return cfg.Global.VswitchID,nil + } + zone, err := m.Zone() + if err != nil { + return "",fmt.Errorf("retrieve vswitchid error for %s",err.Error()) + } + for _, zone := range zlist { + vs := strings.Split(zone,":") + if len(vs) != 2 { + return "", fmt.Errorf("cloud-config vswitch format error: %s",cfg.Global.VswitchID) + } + if vs[0] == zone { + return vs[1], nil + } + } + glog.Infof("zone[%s] match failed, fallback with simple vswitch id mode, [%s]",zone,cfg.Global.VswitchID) + return cfg.Global.VswitchID, nil +} + +func (m *fakeMetaData) EIPv4() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) DNSNameServers() ([]string, error) { + + return []string{""},fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) NTPConfigServers() ([]string, error) { + + return []string{""},fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) Zone() (string, error) { + if cfg.Global.ZoneID != "" { + return cfg.Global.ZoneID, nil + } + return m.base.Zone() +} + +func (m *fakeMetaData) RoleName() (string, error) { + + return "",fmt.Errorf("unimplemented") +} + +func (m *fakeMetaData) RamRoleToken(role string) (metadata.RoleAuth, error) { + + return metadata.RoleAuth{},fmt.Errorf("unimplemented") +} + diff --git a/cloud-controller-manager/alicloud/loadbalancer.go b/cloud-controller-manager/alicloud/loadbalancer.go index be71c9c4b..c29b97ac5 100644 --- a/cloud-controller-manager/alicloud/loadbalancer.go +++ b/cloud-controller-manager/alicloud/loadbalancer.go @@ -27,6 +27,8 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" + "github.com/denverdino/aliyungo/common" + "encoding/json" ) type AnnotationRequest struct { @@ -62,6 +64,7 @@ type AnnotationRequest struct { CookieTimeout int PersistenceTimeout int } +const TAGKEY = "kubernetes.do.not.delete" type ClientSLBSDK interface { DescribeLoadBalancers(args *slb.DescribeLoadBalancersArgs) (loadBalancers []slb.LoadBalancerType, err error) @@ -88,6 +91,10 @@ type ClientSLBSDK interface { SetLoadBalancerHTTPSListenerAttribute(args *slb.SetLoadBalancerHTTPSListenerAttributeArgs) (err error) SetLoadBalancerTCPListenerAttribute(args *slb.SetLoadBalancerTCPListenerAttributeArgs) (err error) SetLoadBalancerUDPListenerAttribute(args *slb.SetLoadBalancerUDPListenerAttributeArgs) (err error) + + RemoveTags(args *slb.RemoveTagsArgs) error + DescribeTags(args *slb.DescribeTagsArgs) (tags []slb.TagItemType, pagination *common.PaginationResult, err error) + AddTags(args *slb.AddTagsArgs) error } type LoadBalancerClient struct { @@ -98,30 +105,28 @@ type LoadBalancerClient struct { func (s *LoadBalancerClient) findLoadBalancer(service *v1.Service) (bool, *slb.LoadBalancerType, error) { def,_ := ExtractAnnotationRequest(service) - loadbalancer := service.Status.LoadBalancer - glog.V(4).Infof("alicloud: find loadbalancer [%v] with user defined annotations [%v]\n",loadbalancer,def) - if len(loadbalancer.Ingress) > 0 { - if lbid := fromLoadBalancerStatus(service); lbid != "" { - if def.Loadbalancerid != "" && def.Loadbalancerid != lbid { - glog.Errorf("alicloud: changing loadbalancer id was not allowed after loadbalancer"+ - " was already created ! please remove service annotation: [%s]\n", ServiceAnnotationLoadBalancerId) - return false, nil, errors.New(fmt.Sprintf("alicloud: change loadbalancer id after service " + - "has been created is not supported. remove service annotation[%s] and retry!", ServiceAnnotationLoadBalancerId)) - } - // found loadbalancer id in service ingress status. - // this id was set previously when loadbalancer was created. - return s.findLoadBalancerByID(lbid) - } - } - // if service ingress status was not initialized with loadbalancer, then - // we check annotation to see if user had assigned a loadbalancer manually. - // if so , we use user defined loadbalancer + //loadbalancer := service.Status.LoadBalancer + //glog.V(4).Infof("alicloud: find loadbalancer [%v] with user defined annotations [%v]\n",loadbalancer,def) + //if len(loadbalancer.Ingress) > 0 { + // if lbid := fromLoadBalancerStatus(service); lbid != "" { + // if def.Loadbalancerid != "" && def.Loadbalancerid != lbid { + // glog.Errorf("alicloud: changing loadbalancer id was not allowed after loadbalancer"+ + // " was already created ! please remove service annotation: [%s]\n", ServiceAnnotationLoadBalancerId) + // return false, nil, errors.New(fmt.Sprintf("alicloud: change loadbalancer id after service " + + // "has been created is not supported. remove service annotation[%s] and retry!", ServiceAnnotationLoadBalancerId)) + // } + // // found loadbalancer id in service ingress status. + // // this id was set previously when loadbalancer was created. + // return s.findLoadBalancerByID(lbid) + // } + //} + + // User assigned lobadbalancer id go first. if def.Loadbalancerid != "" { return s.findLoadBalancerByID(def.Loadbalancerid) } - - // finally , fallback to find by name to compatible with old version - return s.findLoadBalancerByName(cloudprovider.GetLoadBalancerName(service)) + // if not, find by slb tags + return s.findLoadBalancerByTags(service); } func (s *LoadBalancerClient) findLoadBalancerByID(lbid string) (bool, *slb.LoadBalancerType, error) { @@ -148,14 +153,51 @@ func (s *LoadBalancerClient) findLoadBalancerByID(lbid string) (bool, *slb.LoadB return err == nil, lb, err } -func (s *LoadBalancerClient) findLoadBalancerByName(lbn string) (bool, *slb.LoadBalancerType, error) { +func (s *LoadBalancerClient) findLoadBalancerByTags(service *v1.Service) (bool, *slb.LoadBalancerType, error) { + lbn := cloudprovider.GetLoadBalancerName(service) + items, err := json.Marshal( + []slb.TagItem{ + { + TagKey: TAGKEY, + TagValue: lbn, + }, + }, + ) + if err != nil { + return false, nil, err + } + lbs, err := s.c.DescribeLoadBalancers( + &slb.DescribeLoadBalancersArgs{ + RegionId: DEFAULT_REGION, + Tags: string(items), + }, + ) + glog.V(2).Infof("alicloud: fallback to find loadbalancer by tags [%s]\n", string(items)) + if err != nil { + return false, nil, err + } + + if lbs == nil || len(lbs) == 0 { + // here we need to fallback on finding by name for compatible reason + // the old service slb may not have a tag. + return s.findLoadBalancerByName(lbn) + } + if len(lbs) > 1 { + glog.Warningf("alicloud: multiple loadbalancer returned with tags [%s], "+ + "using the first one with IP=%s", string(items), lbs[0].Address) + } + lb, err := s.c.DescribeLoadBalancerAttribute(lbs[0].LoadBalancerId) + return err == nil, lb, err +} + +func (s *LoadBalancerClient) findLoadBalancerByName(name string) (bool, *slb.LoadBalancerType, error) { lbs, err := s.c.DescribeLoadBalancers( &slb.DescribeLoadBalancersArgs{ - RegionId: DEFAULT_REGION, - LoadBalancerName: lbn, + RegionId: DEFAULT_REGION, + LoadBalancerName: name, }, ) - glog.V(2).Infof("alicloud: fallback to find loadbalancer by name [%s]\n", lbn) + glog.V(2).Infof("alicloud: fallback to find loadbalancer by name [%s]\n", name) if err != nil { return false, nil, err } @@ -165,7 +207,7 @@ func (s *LoadBalancerClient) findLoadBalancerByName(lbn string) (bool, *slb.Load } if len(lbs) > 1 { glog.Warningf("alicloud: multiple loadbalancer returned with name [%s], "+ - "using the first one with IP=%s", lbn, lbs[0].Address) + "using the first one with IP=%s", name, lbs[0].Address) } lb, err := s.c.DescribeLoadBalancerAttribute(lbs[0].LoadBalancerId) return err == nil, lb, err @@ -179,44 +221,60 @@ func (s *LoadBalancerClient) EnsureLoadBalancer(service *v1.Service, nodes []*v1 return nil, err } glog.V(4).Infof("alicloud: find loadbalancer with result, exist=%v\n %s\n", exists, PrettyJson(origined)) + _,request := ExtractAnnotationRequest(service) // this is a workaround for issue: https://github.com/kubernetes/kubernetes/issues/59084 if !exists { // If need created, double check if the resource id has been deleted - err = s.EnsureSVCNotDeleted(service) - if err != nil { - glog.V(2).Infof("alicloud: EnsureSVCNotDeleted func can't work properly due to %+v, exit", err) + if isServiceDeleted(service) { + glog.V(2).Infof("alicloud: isServiceDeleted report that this service has been " + + "deleted before. see issue: https://github.com/kubernetes/kubernetes/issues/59084") os.Exit(1) } - } - - if !exists && checkIfSLBExistInService(service) { - return nil, errors.New(fmt.Sprintf("alicloud: not able to find loadbalancer "+ - "named [%s] in openapi, but it's defined in service.loaderbalancer.ingress.\n", service.Name)) - } - - _,request := ExtractAnnotationRequest(service) - - if !exists && request.Loadbalancerid != "" { - return nil, errors.New(fmt.Sprintf("alicloud: user specified " + - "loadbalancer[%s] does not exist. pls check!", request.Loadbalancerid)) - } - - opts := s.getLoadBalancerOpts(service, vswitchid) + if isLoadbalancerOwnIngress(service) { + return nil, errors.New(fmt.Sprintf("alicloud: not able to find loadbalancer "+ + "named [%s] in openapi, but it's defined in service.loaderbalancer.ingress. " + + "this may happend when you removed loadbalancerid annotation.\n", service.Name)) + } + if request.Loadbalancerid != "" { + return nil, errors.New(fmt.Sprintf("alicloud: user specified " + + "loadbalancer[%s] does not exist. pls check!", request.Loadbalancerid)) + } - if !exists { + // From here, we need to create a new loadbalancer glog.V(5).Infof("alicloud: can not find a loadbalancer with service name [%s/%s], creating a new one\n",service.Namespace,service.Name) + opts := s.getLoadBalancerOpts(service, vswitchid) lbr, err := s.c.CreateLoadBalancer(opts) if err != nil { return nil, err } + + // Tag the loadbalancer. + items, err := json.Marshal( + []slb.TagItem{ + { + TagKey:TAGKEY, + TagValue:opts.LoadBalancerName, + }, + }, + ) + if err != nil { + return nil, err + } + if err := s.c.AddTags(&slb.AddTagsArgs{ + RegionId:opts.RegionId, + LoadBalancerID:lbr.LoadBalancerId, + Tags: string(items), + }); err != nil { + return nil, err + } origined, err = s.c.DescribeLoadBalancerAttribute(lbr.LoadBalancerId) } else { glog.V(5).Infof("alicloud: found an exist loadbalancer[%s], check to see whether update is needed.", origined.LoadBalancerId) if request.ChargeType != "" && request.ChargeType != origined.InternetChargeType { // Todo: here we need to compare loadbalance glog.Infof("alicloud: internet charge type changed. [%s] -> [%s], update loadbalancer [%s]\n", - string(origined.InternetChargeType), string(request.ChargeType), opts.LoadBalancerName) + string(origined.InternetChargeType), string(request.ChargeType), origined.LoadBalancerName) if err := s.c.ModifyLoadBalancerInternetSpec( &slb.ModifyLoadBalancerInternetSpecArgs{ @@ -230,7 +288,7 @@ func (s *LoadBalancerClient) EnsureLoadBalancer(service *v1.Service, nodes []*v1 if request.AddressType != "" && request.AddressType != origined.AddressType { glog.Errorf("alicloud: warning! can not change "+ "loadbalancer address type after it has been created! please "+ - "recreate the service.[%s]->[%s],[%s]\n", origined.AddressType, request.AddressType, opts.LoadBalancerName) + "recreate the service.[%s]->[%s],[%s]\n", origined.AddressType, request.AddressType, origined.LoadBalancerName) return nil, errors.New("alicloud: change loadbalancer address type after service has been created is not supported. delete and retry") } origined, err = s.c.DescribeLoadBalancerAttribute(origined.LoadBalancerId) @@ -240,7 +298,7 @@ func (s *LoadBalancerClient) EnsureLoadBalancer(service *v1.Service, nodes []*v1 return nil, err } // we should apply listener update only if user does not assign loadbalancer id by themselves. - if ! isUserDefinedLoadBalancer(service, request) { + if ! isUserDefinedLoadBalancer(request) { glog.V(5).Infof("alicloud: not user defined loadbalancer[%s], start to apply listener.\n",origined.LoadBalancerId) err = NewListenerManager(s.c, service, origined).Apply() if err != nil { @@ -263,51 +321,51 @@ func (s *LoadBalancerClient) UpdateLoadBalancer(service *v1.Service, nodes []*v1 return s.UpdateBackendServers(nodes, lb) } -//create , play actual create. -func (s *LoadBalancerClient) create(loadbalancer *slb.CreateLoadBalancerArgs) (*slb.LoadBalancerType, error) { - lbr, err := s.c.CreateLoadBalancer(loadbalancer) - if err != nil { - return nil, err - } - return s.c.DescribeLoadBalancerAttribute(lbr.LoadBalancerId) -} +////create , play actual create. +//func (s *LoadBalancerClient) create(loadbalancer *slb.CreateLoadBalancerArgs) (*slb.LoadBalancerType, error) { +// lbr, err := s.c.CreateLoadBalancer(loadbalancer) +// if err != nil { +// return nil, err +// } +// return s.c.DescribeLoadBalancerAttribute(lbr.LoadBalancerId) +//} //update, play loadbalancer update -func (s *LoadBalancerClient) update(old *slb.LoadBalancerType, new *slb.CreateLoadBalancerArgs) (*slb.LoadBalancerType, error) { - // Todo: here we need to compare loadbalance - if new.InternetChargeType != old.InternetChargeType { - glog.Infof("alicloud: internet charge type changed. [%s] -> [%s], update loadbalancer [%s]\n", - string(old.InternetChargeType), string(new.InternetChargeType), new.LoadBalancerName) - - if err := s.c.ModifyLoadBalancerInternetSpec( - &slb.ModifyLoadBalancerInternetSpecArgs{ - LoadBalancerId: old.LoadBalancerId, - InternetChargeType: new.InternetChargeType, - //Bandwidth: opts.Bandwidth, - }); err != nil { - return nil, err - } - } - if new.AddressType != old.AddressType { - glog.Errorf("alicloud: warning! can not change "+ - "loadbalancer address type after it has been created! please "+ - "recreate the service.[%s]->[%s],[%s]\n", old.AddressType, new.AddressType, new.LoadBalancerName) - return nil, errors.New("alicloud: change loadbalancer address type after created is not support.") - } - return s.c.DescribeLoadBalancerAttribute(old.LoadBalancerId) -} +//func (s *LoadBalancerClient) update(old *slb.LoadBalancerType, new *slb.CreateLoadBalancerArgs) (*slb.LoadBalancerType, error) { +// // Todo: here we need to compare loadbalance +// if new.InternetChargeType != old.InternetChargeType { +// glog.Infof("alicloud: internet charge type changed. [%s] -> [%s], update loadbalancer [%s]\n", +// string(old.InternetChargeType), string(new.InternetChargeType), new.LoadBalancerName) +// +// if err := s.c.ModifyLoadBalancerInternetSpec( +// &slb.ModifyLoadBalancerInternetSpecArgs{ +// LoadBalancerId: old.LoadBalancerId, +// InternetChargeType: new.InternetChargeType, +// //Bandwidth: opts.Bandwidth, +// }); err != nil { +// return nil, err +// } +// } +// if new.AddressType != old.AddressType { +// glog.Errorf("alicloud: warning! can not change "+ +// "loadbalancer address type after it has been created! please "+ +// "recreate the service.[%s]->[%s],[%s]\n", old.AddressType, new.AddressType, new.LoadBalancerName) +// return nil, errors.New("alicloud: change loadbalancer address type after created is not support.") +// } +// return s.c.DescribeLoadBalancerAttribute(old.LoadBalancerId) +//} func (s *LoadBalancerClient) EnsureLoadBalanceDeleted(service *v1.Service) error { // need to save the resource version when deleted event - err := s.SaveDeletedSVCResourceVersion(service) + err := keepResourceVesion(service) if err != nil { glog.Warningf("alicloud: failed to save " + "deleted service resourceVersion, [%s] due to [%s] ", service.Name, err.Error()) } _, request := ExtractAnnotationRequest(service) // skip delete user defined loadbalancer - if isUserDefinedLoadBalancer(service,request) { + if isUserDefinedLoadBalancer(request) { glog.V(2).Infof("alicloud: user created loadbalancer " + "will not be deleted by cloudprovider. service [%s]", service.Name) return nil @@ -408,128 +466,47 @@ func (s *LoadBalancerClient) UpdateBackendServers(nodes []*v1.Node, lb *slb.Load return nil } -// domain retrun slb hostname. -// this is intended to keep track of user define slb. -// user defined slb has the hostname format of ${serviceName}.${slbid}.${regionid}.alicontainer.com -// auto generated slb has the hostname format of ${slbid}.${regionid}.alicontainer.com -func domain(service *v1.Service, lb *slb.LoadBalancerType) (string) { - _, request := ExtractAnnotationRequest(service) - - hostname := func ()string { - if request.Loadbalancerid != "" { - return loadBalancerDomain(service.Name,lb.LoadBalancerId, string(DEFAULT_REGION)) - } - return loadBalancerDomain("",lb.LoadBalancerId, string(DEFAULT_REGION)) - } - if service.Status.LoadBalancer.Ingress == nil || - len(service.Status.LoadBalancer.Ingress) == 0 { - - return hostname() - }else { - ingress := service.Status.LoadBalancer.Ingress[0] - if ingress.IP == "" { - // This is not expected scenario. we just keep ingress status unchanged. - return ingress.Hostname - } - if ingress.Hostname == "" { - // That is the case we should fix. Just to ensure hostname is eventually set. - return hostname() - } - } - return service.Status.LoadBalancer.Ingress[0].Hostname - -} - -func loadBalancerDomain(name, id, region string) string { - domain := []string{ - id, - region, - "alicontainer", - "com", - } - if name != "" { - domain = append([]string{name}, domain...) - } - return strings.Join(domain, ".") -} -// isUserDefinedLoadBalancer -// 1. has ingress hostname length equal to 5. -// 2. with id annotation -func isUserDefinedLoadBalancer(service *v1.Service, request * AnnotationRequest) bool { - - ing := service.Status.LoadBalancer.Ingress - if ing == nil || len(ing) <= 0 { - return request.Loadbalancerid != "" - } - domain := ing[0].Hostname - if domain == "" { - return request.Loadbalancerid != "" - } - if len(strings.Split(domain, ".")) != 5 { - return false - } - return true -} +// check to see if user has assigned any loadbalancer +func isUserDefinedLoadBalancer(request * AnnotationRequest) bool { -// fromLoadBalancerStatus split loadbalancer id from service`s loadbalancer status. -// see func domain() for format information -func fromLoadBalancerStatus(service *v1.Service) string { - ing := service.Status.LoadBalancer.Ingress - if ing == nil || len(ing) <= 0 { - return "" - } - domain := ing[0].Hostname - if domain == "" { - return "" - } - secs := strings.Split(domain, ".") - if len(secs) < 4 { - glog.Warningf("alicloud: can not split loadbalancer hostname domain. [%s]\n",domain) - return "" - } - glog.V(4).Infof("alicloud: extract loadbalancer id from service.Spec.LoadBalancerStatus[%s]",secs[len(secs) - 4]) - return secs[len(secs) - 4] + return request.Loadbalancerid != "" } // check if the service exists in service definition -func checkIfSLBExistInService(service *v1.Service) (exists bool) { +func isLoadbalancerOwnIngress(service *v1.Service) (bool) { if service == nil || len(service.Status.LoadBalancer.Ingress) == 0 { - glog.V(2).Infof("alicloud: service %s doesn't have ingresses\n", service.Name) - exists = false - } else { - glog.V(2).Infof("alicloud: service %s has ingresses=%v\n", service.Name, service.Status.LoadBalancer.Ingress) - exists = true + glog.V(2).Infof("alicloud: service " + + "%s doesn't have ingresses\n", service.Name) + return false } - - return exists - + glog.V(2).Infof("alicloud: service %s has" + + " ingresses=%v\n", service.Name, service.Status.LoadBalancer.Ingress) + return true } -// ensure service resource version properly and update last known resource version to the largest one, for now only keep create and delete behavior -func (s *LoadBalancerClient) EnsureSVCNotDeleted(service *v1.Service) error { - if service == nil { - return fmt.Errorf("alicloud:service is nil") - } - +// ensure service resource version properly and update last known resource +// version to the largest one, for now only keep create and delete behavior +func isServiceDeleted(service *v1.Service) bool { + //if service == nil { + // return fmt.Errorf("alicloud:service is nil") + //} serviceUID := string(service.GetUID()) keeper := GetLocalService() deleted := keeper.get(serviceUID) if deleted { glog.V(2).Infof("alicloud: service "+ "%s's uid %v has been deleted, shouldn't be created again.\n", service.Name, serviceUID) - return fmt.Errorf("alicloud: service "+ - "%s's uid %v has been deleted, shouldn't be created again.\n", service.Name, serviceUID) + return true } else { glog.V(2).Infof("alicloud: service %s's uid %v "+ "hasn't been deleted, first time to process, as expected.\n", service.Name, serviceUID) } - - return nil + return false } // save the deleted service's uid -func (s *LoadBalancerClient) SaveDeletedSVCResourceVersion(service *v1.Service) error { +func keepResourceVesion(service *v1.Service) error { if service == nil { return fmt.Errorf("alicloud: failed to save deleted service resource version , for service is nil") } diff --git a/cloud-controller-manager/alicloud/loadbalancer_test.go b/cloud-controller-manager/alicloud/loadbalancer_test.go index 3d0f8c4b9..849dd429f 100644 --- a/cloud-controller-manager/alicloud/loadbalancer_test.go +++ b/cloud-controller-manager/alicloud/loadbalancer_test.go @@ -93,7 +93,10 @@ func TestFindLoadBalancer(t *testing.T) { if args.LoadBalancerName != "" { base[0].LoadBalancerName = args.LoadBalancerName return base, nil - } else { + } + if len(args.Tags)>0{ + base[0].LoadBalancerName = cloudprovider.GetLoadBalancerName(service) + }else { return nil, errors.New("loadbalancerid or loadbanancername must be specified.\n") } return base, nil @@ -128,23 +131,6 @@ func TestFindLoadBalancer(t *testing.T) { if lb.LoadBalancerId != LOADBALANCER_ID+"-new" { t.Fatal("find loadbalancer fail. suppose to find by exist loadbalancerid.") } - - // 3. - // user has already create a loadbalancer. use ingress status`s id instead. - delete(service.Annotations, ServiceAnnotationLoadBalancerId) - ingress := v1.LoadBalancerIngress{ - IP: LOADBALANCER_ADDRESS, - Hostname: loadBalancerDomain("my-service",LOADBALANCER_ID+"-ingress",string(DEFAULT_REGION)), - } - service.Status.LoadBalancer.Ingress = append(service.Status.LoadBalancer.Ingress, ingress) - exist, lb, err = mgr.loadbalancer.findLoadBalancer(service) - if err != nil || !exist { - t.Logf("3.user has already create a loadbalancer. use ingress status`s id instead") - t.Fatal("Test findLoadBalancer fail.") - } - if lb.LoadBalancerId != LOADBALANCER_ID+"-ingress" { - t.Fatal("find loadbalancer fail. suppose to find by exist loadbalancerid.") - } } func realSlbClient(keyid, keysec string) { @@ -202,6 +188,9 @@ type mockClientSLB struct { setLoadBalancerHTTPSListenerAttribute func(args *slb.SetLoadBalancerHTTPSListenerAttributeArgs) (err error) setLoadBalancerTCPListenerAttribute func(args *slb.SetLoadBalancerTCPListenerAttributeArgs) (err error) setLoadBalancerUDPListenerAttribute func(args *slb.SetLoadBalancerUDPListenerAttributeArgs) (err error) + removeTags func(args *slb.RemoveTagsArgs) error + describeTags func(args *slb.DescribeTagsArgs) (tags []slb.TagItemType, pagination *common.PaginationResult, err error) + addTags func(args *slb.AddTagsArgs) error } var ( @@ -375,4 +364,23 @@ func (c *mockClientSLB) SetLoadBalancerUDPListenerAttribute(args *slb.SetLoadBal return c.setLoadBalancerUDPListenerAttribute(args) } return nil +} + +func (c *mockClientSLB) RemoveTags(args *slb.RemoveTagsArgs) error { + if c.removeTags != nil { + return c.removeTags(args) + } + return nil +} +func (c *mockClientSLB) DescribeTags(args *slb.DescribeTagsArgs) (tags []slb.TagItemType, pagination *common.PaginationResult, err error){ + if c.describeTags != nil { + return c.describeTags(args) + } + return []slb.TagItemType{}, nil, nil +} +func (c *mockClientSLB) AddTags(args *slb.AddTagsArgs) error{ + if c.addTags != nil { + return c.addTags(args) + } + return nil } \ No newline at end of file diff --git a/cloud-controller-manager/controller/service/service_controller.go b/cloud-controller-manager/controller/service/service_controller.go index 722396a42..20fab5351 100644 --- a/cloud-controller-manager/controller/service/service_controller.go +++ b/cloud-controller-manager/controller/service/service_controller.go @@ -201,11 +201,11 @@ clusterName string, serviceInformer.Informer().AddEventHandlerWithResyncPeriod( cache.ResourceEventHandlerFuncs{ AddFunc: func(cur interface{}) { - if !needLoadBalancer(cur.(*v1.Service)) { - key, _ := controller.KeyFunc(cur) - glog.Infof("controller: do not need loadbalancer ,skip. %s\n",key) - return - } + //if !needLoadBalancer(cur.(*v1.Service)) { + // key, _ := controller.KeyFunc(cur) + // glog.Infof("controller: do not need loadbalancer ,skip. %s\n",key) + // return + //} svc := cur.(*v1.Service) glog.V(5).Infof("controller: Add event, service [%s/%s]\n",svc.Namespace, svc.Name) s.enqueueService(cur) @@ -578,7 +578,12 @@ func (s *ServiceController) syncBackend(service *cachedService) (error) { if len(nodes) == 0 { s.eventRecorder.Eventf(service.state, v1.EventTypeWarning, "UnAvailableLoadBalancer", "There are no available nodes for LoadBalancer service %s/%s", service.state.Namespace, service.state.Name) } - + // service holds the latest service info from apiserver + _, err = s.serviceLister.Services(service.state.Namespace).Get(service.state.Name) + if errors.IsNotFound(err) { + glog.Infof("SyncBackend: Service has been deleted %v", key) + return nil + } // here we should not check for the neediness of updating loadbalancer. // Because, the provider may need to filter by label again. /* diff --git a/docs/getting-started.md b/docs/getting-started.md index c2fc1b6bb..02c20dd61 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -52,7 +52,7 @@ If you are not sure how to find your ECS instance's ID and region id, try to run 1. Prepare `cloud-controller-manager` daemonset yaml -Mare sure container image, `--cluster-cidr` field match what your needs. +Mare sure container image, `--cluster-cidr` field match what your needs. replace image with your version. ``` apiVersion: extensions/v1beta1 diff --git a/hack/setup.sh b/hack/setup.sh index 80d68116c..fab0947c2 100644 --- a/hack/setup.sh +++ b/hack/setup.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -e -x if [ -z $APISERVER_IP ];then APISERVER_IP=47.97.236.85 fi diff --git a/test/cloud-config.json b/test/cloud-config.json new file mode 100644 index 000000000..8a6ae7f13 --- /dev/null +++ b/test/cloud-config.json @@ -0,0 +1,12 @@ +{ + "Global": + { + "KubernetesClusterTag":"", + "vpcid":"vpc-bp1sah8hqjiyusoh8ua09", + "region":"cn-hangzhou", + "zoneid":"cn-hangzhou-b", + "vswitchid":"vsw-bp1fznltga17mffpjm67y", + "accessKeyID":"base64 encoded", + "accessKeySecret":"base64 encoded" + } +} diff --git a/vendor/github.com/denverdino/aliyungo/slb/loadbalancers.go b/vendor/github.com/denverdino/aliyungo/slb/loadbalancers.go index 49beec29b..90d5736bb 100644 --- a/vendor/github.com/denverdino/aliyungo/slb/loadbalancers.go +++ b/vendor/github.com/denverdino/aliyungo/slb/loadbalancers.go @@ -187,6 +187,7 @@ type DescribeLoadBalancersArgs struct { Address string InternetChargeType InternetChargeType ServerId string + Tags string } type ListenerPortAndProtocolType struct {