Multiple ips

pkg/api/galaxy/constant/constant.go

@@ -30,22 +30,9 @@ const (
 
 	MultusCNIAnnotation = "k8s.v1.cni.cncf.io/networks"
 
-	CommonCNIArgsKey = "common"
-
 	// For fip crd object which has this label, it's reserved by admin manually. IPAM will not allocate it to pods.
 	ReserveFIPLabel = "reserved"
-)
-
-// ParseExtendedCNIArgs parses extended cni args from pod annotation
-func ParseExtendedCNIArgs(args string) (map[string]map[string]json.RawMessage, error) {
-	argsMap := map[string]map[string]json.RawMessage{}
-	if err := json.Unmarshal([]byte(args), &argsMap); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal %s value %s: %v", ExtendedCNIArgsAnnotation, args, err)
-	}
-	return argsMap, nil
-}
 
-const (
 	IPInfosKey = "ipinfos"
 )
 
@@ -56,38 +43,6 @@ type IPInfo struct {
 	Gateway net.IP      `json:"gateway"`
 }
 
-// FormatIPInfo formats ipInfos as extended CNI Args annotation value
-func FormatIPInfo(ipInfos []IPInfo) (string, error) {
-	data, err := json.Marshal(ipInfos)
-	if err != nil {
-		return "", err
-	}
-	raw := json.RawMessage(data)
-	str, err := json.Marshal(map[string]map[string]*json.RawMessage{CommonCNIArgsKey: {IPInfosKey: &raw}})
-	return string(str), err
-}
-
-// ParseIPInfo pareses ipInfo from annotation
-func ParseIPInfo(str string) ([]IPInfo, error) {
-	m := map[string]map[string]*json.RawMessage{}
-	if err := json.Unmarshal([]byte(str), &m); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal %s value %s: %v", ExtendedCNIArgsAnnotation, str, err)
-	}
-	commonMap := m[CommonCNIArgsKey]
-	if commonMap == nil {
-		return []IPInfo{}, nil
-	}
-	ipInfoStr := commonMap[IPInfosKey]
-	if ipInfoStr == nil {
-		return []IPInfo{}, nil
-	}
-	var ipInfos []IPInfo
-	if err := json.Unmarshal([]byte(*ipInfoStr), &ipInfos); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal %s value %s as common/ipInfos: %v", ExtendedCNIArgsAnnotation, str, err)
-	}
-	return ipInfos, nil
-}
-
 // ReleasePolicy defines floatingip release policy
 type ReleasePolicy uint16
 
@@ -124,3 +79,39 @@ const (
 	NameSpace    = "floating-ip"
 	IpType       = "ipType"
 )
+
+// CniArgs is the cni args in pod annotation
+type CniArgs struct {
+	// RequestIPRange is the requested ip candidates to allocate, one ip per []nets.IPRange
+	RequestIPRange [][]nets.IPRange `json:"request_ip_range,omitempty"`
+	// Common is the common args for cni plugins to setup network
+	Common CommonCniArgs `json:"common"`
+}
+
+type CommonCniArgs struct {
+	IPInfos []IPInfo `json:"ipinfos,omitempty"`
+}
+
+// UnmarshalCniArgs unmarshal cni args from input str
+func UnmarshalCniArgs(str string) (*CniArgs, error) {
+	if str == "" {
+		return nil, nil
+	}
+	var cniArgs CniArgs
+	if err := json.Unmarshal([]byte(str), &cniArgs); err != nil {
+		return nil, fmt.Errorf("unmarshal pod cni args: %v", err)
+	}
+	return &cniArgs, nil
+}
+
+// MarshalCniArgs marshal cni args of the given ipInfos
+func MarshalCniArgs(ipInfos []IPInfo) (string, error) {
+	cniArgs := CniArgs{Common: CommonCniArgs{
+		IPInfos: ipInfos,
+	}}
+	data, err := json.Marshal(cniArgs)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}

pkg/galaxy/server.go

@@ -217,11 +217,9 @@ func (g *Galaxy) resolveNetworks(req *galaxyapi.PodRequest, pod *corev1.Pod) ([]
 	if err != nil {
 		return nil, err
 	}
-	if commonArgs, exist := extendedCNIArgs[constant.CommonCNIArgsKey]; exist {
-		for i := range networkInfos {
-			for k, v := range commonArgs {
-				networkInfos[i].Args[k] = string([]byte(v))
-			}
+	for i := range networkInfos {
+		for k, v := range extendedCNIArgs {
+			networkInfos[i].Args[k] = string(v)
 		}
 	}
 	glog.V(4).Infof("pod %s_%s networkInfo %v", pod.Name, pod.Namespace, networkInfos)
@@ -263,19 +261,23 @@ func (g *Galaxy) cmdAdd(req *galaxyapi.PodRequest, pod *corev1.Pod) (types.Resul
 }
 
 // parseExtendedCNIArgs parses extended cni args from pod's annotation
-func parseExtendedCNIArgs(pod *corev1.Pod) (map[string]map[string]json.RawMessage, error) {
+func parseExtendedCNIArgs(pod *corev1.Pod) (map[string]json.RawMessage, error) {
 	if pod.Annotations == nil {
 		return nil, nil
 	}
-	annotation := pod.Annotations[constant.ExtendedCNIArgsAnnotation]
-	if annotation == "" {
+	args := pod.Annotations[constant.ExtendedCNIArgsAnnotation]
+	if args == "" {
 		return nil, nil
 	}
-	argsMap, err := constant.ParseExtendedCNIArgs(annotation)
-	if err != nil {
-		return nil, err
+	// CniArgs is the cni args in pod annotation
+	var cniArgs struct {
+		// Common is the common args for cni plugins to setup network
+		Common map[string]json.RawMessage `json:"common"`
+	}
+	if err := json.Unmarshal([]byte(args), &cniArgs); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal cni args %s: %v", args, err)
 	}
-	return argsMap, nil
+	return cniArgs.Common, nil
 }
 
 func (g *Galaxy) setupIPtables() error {

pkg/api/galaxy/constant/constant_test.go → pkg/galaxy/server_test.go

@@ -14,38 +14,26 @@
  * WARRANTIES OF ANY KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations under the License.
  */
-package constant
+package galaxy
 
 import (
-	"net"
-	"reflect"
 	"testing"
 
-	"tkestack.io/galaxy/pkg/utils/nets"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"tkestack.io/galaxy/pkg/api/galaxy/constant"
 )
 
-func TestFormatParseIPInfo(t *testing.T) {
-	testCase := []IPInfo{
-		{
-			IP:      nets.NetsIPNet(&net.IPNet{IP: net.ParseIP("192.168.0.2"), Mask: net.IPv4Mask(255, 255, 0, 0)}),
-			Vlan:    2,
-			Gateway: net.ParseIP("192.168.0.1"),
-		},
-		{
-			IP:      nets.NetsIPNet(&net.IPNet{IP: net.ParseIP("192.168.0.3"), Mask: net.IPv4Mask(255, 255, 0, 0)}),
-			Vlan:    3,
-			Gateway: net.ParseIP("192.168.0.1"),
-		},
-	}
-	str, err := FormatIPInfo(testCase)
-	if err != nil {
-		t.Fatal(err)
-	}
-	parsed, err := ParseIPInfo(str)
+func TestParseExtendedCNIArgs(t *testing.T) {
+	m, err := parseExtendedCNIArgs(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+		constant.ExtendedCNIArgsAnnotation: `{"request_ip_range":[["10.0.0.2~10.0.0.30"],["10.0.0.200~10.0.0.238"]],"common":{"ipinfos":[{"ip":"10.0.0.3/24","vlan":0,"gateway":"10.0.0.1"},{"ip":"10.0.0.200/24","vlan":0,"gateway":"10.0.0.1"}]}}`,
+	}}})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(parsed, testCase) {
-		t.Fatalf("real: %v, expect: %v", parsed, testCase)
+	if val, ok := m["ipinfos"]; !ok {
+		t.Fatal()
+	} else if string(val) != `[{"ip":"10.0.0.3/24","vlan":0,"gateway":"10.0.0.1"},{"ip":"10.0.0.200/24","vlan":0,"gateway":"10.0.0.1"}]` {
+		t.Fatal()
 	}
 }

pkg/ipam/api/api.go

@@ -322,35 +322,39 @@ func (c *Controller) ReleaseIPs(req *restful.Request, resp *restful.Response) {
 
 // listIPs lists ips from ipams
 func listIPs(keyword string, ipam floatingip.IPAM, fuzzyQuery bool) ([]FloatingIP, error) {
-	var fips []floatingip.FloatingIP
-	var err error
+	var result []FloatingIP
 	if fuzzyQuery {
-		fips, err = ipam.ByKeyword(keyword)
+		fips, err := ipam.ByKeyword(keyword)
+		if err != nil {
+			return nil, err
+		}
+		for i := range fips {
+			result = append(result, convert(&fips[i]))
+		}
 	} else {
-		fips, err = ipam.ByPrefix(keyword)
-	}
-	if err != nil {
-		return nil, err
+		fips, err := ipam.ByPrefix(keyword)
+		if err != nil {
+			return nil, err
+		}
+		for i := range fips {
+			result = append(result, convert(&fips[i].FloatingIP))
+		}
 	}
-	return transform(fips), nil
+	return result, nil
 }
 
-// transform converts `floatingip.FloatingIP` slice to `FloatingIP` slice
-func transform(fips []floatingip.FloatingIP) []FloatingIP {
-	var res []FloatingIP
-	for i := range fips {
-		keyObj := util.ParseKey(fips[i].Key)
-		res = append(res, FloatingIP{IP: fips[i].IP.String(),
-			Namespace:  keyObj.Namespace,
-			AppName:    keyObj.AppName,
-			PodName:    keyObj.PodName,
-			PoolName:   keyObj.PoolName,
-			AppType:    toAppType(keyObj.AppTypePrefix),
-			Policy:     fips[i].Policy,
-			UpdateTime: fips[i].UpdatedAt,
-			labels:     fips[i].Labels})
-	}
-	return res
+// convert converts `floatingip.FloatingIP` to `FloatingIP`
+func convert(fip *floatingip.FloatingIP) FloatingIP {
+	keyObj := util.ParseKey(fip.Key)
+	return FloatingIP{IP: fip.IP.String(),
+		Namespace:  keyObj.Namespace,
+		AppName:    keyObj.AppName,
+		PodName:    keyObj.PodName,
+		PoolName:   keyObj.PoolName,
+		AppType:    toAppType(keyObj.AppTypePrefix),
+		Policy:     fip.Policy,
+		UpdateTime: fip.UpdatedAt,
+		labels:     fip.Labels}
 }
 
 // batchReleaseIPs release ips from ipams

pkg/ipam/api/pool.go

@@ -144,7 +144,7 @@ func (c *PoolController) preAllocateIP(req *restful.Request, resp *restful.Respo
 		httputil.InternalError(resp, err)
 		return
 	}
-	subnetSet, err := c.IPAM.NodeSubnetsByKey("")
+	subnetSet, err := c.IPAM.NodeSubnetsByIPRanges(nil)
 	if err != nil {
 		httputil.InternalError(resp, err)
 		return

pkg/ipam/cloudprovider/testing/fake_cloud_provider.go

@@ -15,3 +15,38 @@
  * specific language governing permissions and limitations under the License.
  */
 package testing
+
+import (
+	"fmt"
+
+	"tkestack.io/galaxy/pkg/ipam/cloudprovider"
+	"tkestack.io/galaxy/pkg/ipam/cloudprovider/rpc"
+)
+
+var _ cloudprovider.CloudProvider = &FakeCloudProvider{}
+
+// FakeCloudProvider is a fake cloud provider for testing
+type FakeCloudProvider struct {
+	Assigned   map[string]string // Assigned ipaddress to nodeName
+	UnAssigned map[string]string // UnAssigned ipaddress to nodeName
+}
+
+func NewFakeCloudProvider() *FakeCloudProvider {
+	return &FakeCloudProvider{Assigned: map[string]string{}, UnAssigned: map[string]string{}}
+}
+
+func (f *FakeCloudProvider) AssignIP(in *rpc.AssignIPRequest) (*rpc.AssignIPReply, error) {
+	if in == nil {
+		return nil, fmt.Errorf("nil request")
+	}
+	f.Assigned[in.IPAddress] = in.NodeName
+	return &rpc.AssignIPReply{Success: true}, nil
+}
+
+func (f *FakeCloudProvider) UnAssignIP(in *rpc.UnAssignIPRequest) (*rpc.UnAssignIPReply, error) {
+	if in == nil {
+		return nil, fmt.Errorf("nil request")
+	}
+	f.UnAssigned[in.IPAddress] = in.NodeName
+	return &rpc.UnAssignIPReply{Success: true}, nil
+}

pkg/ipam/floatingip/floatingip.go

@@ -31,23 +31,23 @@ import (
 // FloatingIP defines a floating ip
 type FloatingIP struct {
 	Key       string
-	Subnets   sets.String // node subnet, not container ip's subnet
 	IP        net.IP
 	UpdatedAt time.Time
 	Labels    map[string]string
 	Policy    uint16
 	NodeName  string
 	PodUid    string
+	pool      *FloatingIPPool
 }
 
 func (f FloatingIP) String() string {
-	return fmt.Sprintf("FloatingIP{ip:%s key:%s policy:%d nodeName:%s podUid:%s subnets:%v}",
-		f.IP.String(), f.Key, f.Policy, f.NodeName, f.PodUid, f.Subnets)
+	return fmt.Sprintf("FloatingIP{ip:%s key:%s policy:%d nodeName:%s podUid:%s}",
+		f.IP.String(), f.Key, f.Policy, f.NodeName, f.PodUid)
 }
 
 // New creates a new FloatingIP
-func New(ip net.IP, subnets sets.String, key string, attr *Attr, updateAt time.Time) *FloatingIP {
-	fip := &FloatingIP{IP: ip, Subnets: subnets}
+func New(pool *FloatingIPPool, ip net.IP, key string, attr *Attr, updateAt time.Time) *FloatingIP {
+	fip := &FloatingIP{IP: ip, pool: pool}
 	fip.Assign(key, attr, updateAt)
 	return fip
 }
@@ -65,8 +65,8 @@ func (f *FloatingIP) Assign(key string, attr *Attr, updateAt time.Time) *Floatin
 // CloneWith creates a new FloatingIP and updates key, attr, updatedAt
 func (f *FloatingIP) CloneWith(key string, attr *Attr, updateAt time.Time) *FloatingIP {
 	fip := &FloatingIP{
-		IP:      f.IP,
-		Subnets: f.Subnets,
+		IP:   f.IP,
+		pool: f.pool,
 	}
 	return fip.Assign(key, attr, updateAt)
 }
@@ -76,6 +76,8 @@ type FloatingIPPool struct {
 	NodeSubnets []*net.IPNet // the node subnets
 	nets.SparseSubnet
 	sync.RWMutex
+	nodeSubnets sets.String // the node subnets, string set format
+	index       int         // the index of []FloatingIPPool
 }
 
 // FloatingIPPoolConf is FloatingIP config structure.