diff --git a/.travis.yml b/.travis.yml index 745bb8a68..74f29a92a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,8 @@ script: - ./hack/build-go.sh - KUBEBUILDER_ASSETS="$(pwd)/bin" $GOPATH/bin/goveralls -service=travis-ci - docker build -t dougbtv/whereabouts . + - docker build -t dougbtv/whereabouts-ocp -f Dockerfile.openshift . + - docker images deploy: # Push images to Dockerhub on merge to master @@ -38,6 +40,7 @@ deploy: bash -c ' docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS"; docker push dougbtv/whereabouts:latest; + docker push dougbtv/whereabouts-ocp:latest; echo done' - provider: script skip_cleanup: true diff --git a/Dockerfile.openshift b/Dockerfile.openshift index 9c41d73fe..9337a88b1 100644 --- a/Dockerfile.openshift +++ b/Dockerfile.openshift @@ -28,4 +28,4 @@ COPY --from=rhel8 /go/src/github.com/dougbtv/whereabouts/bin/whereabouts /usr/sr LABEL io.k8s.display-name="Whereabouts CNI" \ io.k8s.description="This is a component of OpenShift Container Platform and provides a cluster-wide IPAM CNI plugin." \ io.openshift.tags="openshift" \ - maintainer="CTO Networking " \ No newline at end of file + maintainer="CTO Networking " diff --git a/README.md b/README.md index 5f74badf5..98e8dc4c4 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ You can install this plugin with a Daemonset, using: ``` git clone https://github.com/dougbtv/whereabouts && cd whereabouts -kubectl apply -f ./doc/daemonset-install.yaml -f ./doc/whereabouts.cni.cncf.io_ippools.yaml +kubectl apply -f ./doc/daemonset-install.yaml -f ./doc/whereabouts.cni.cncf.io_ippools.yaml -f ./doc/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml ``` *NOTE*: This daemonset is for use with Kubernetes version 1.16 and later. It may also be useful with previous versions, however you'll need to change the `apiVersion` of the daemonset in the provided yaml, [see the deprecation notice](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/). @@ -178,6 +178,14 @@ There are two optional parameters for logging, they are: * `log_file`: A file path to a logfile to log to. * `log_level`: Set the logging verbosity, from most to least: `debug`,`error`,`panic` +### Overlapping Ranges + +The overlapping ranges feature is enabled by default, and will not allow an IP address to be re-assigned across two different ranges which overlap. However, this can be disabled. + +* `enable_overlapping_ranges`: *(boolean)* Checks to see if an IP has been allocated across another range before assigning it (defaults to `true`). + +Please note: This feature is only implemented for the Kubernetes storage backend. + ## Flatfile configuration There is one option for flat file configuration: @@ -271,9 +279,8 @@ The typeface used in the logo is [AZONIX](https://www.dafont.com/azonix.font), b ## Known limitations -* If you specify overlapping ranges -- you're almost certain to have collisions, so if you specify one config with `192.168.0.0/16` and another with `192.168.0.0/24`, you'll have collisions. - - This could be fixed with an admission controller. - - And admission controller could also prevent you from starting a pod in a given range if you were out of addresses within that range. +* A hard system crash on a node might leave behind stranded IP allocations, so if you have a trashing system, this might exhaust IPs. + - Potentially we need an operator to ensure data is clean, even if just at some kind of interval (e.g. with a cron job) * There's probably a lot of comparison of IP addresses that could be optimized, lots of string conversion. * The etcd method has a number of limitations, in that it uses an all ASCII methodology. If this was binary, it could probably store more and have more efficient IP address comparison. * Unlikely to work in Canada, apparently it would have to be "where aboots?" for Canadians to be able to operate it. diff --git a/cmd/whereabouts.go b/cmd/whereabouts.go index 96752be72..e2603bdfb 100644 --- a/cmd/whereabouts.go +++ b/cmd/whereabouts.go @@ -41,7 +41,7 @@ func cmdAdd(args *skel.CmdArgs) error { newip, err := storage.IPManagement(types.Allocate, *ipamConf, args.ContainerID) if err != nil { logging.Errorf("Error assigning IP: %s", err) - return fmt.Errorf("Error assigning IP: %s", err) + return fmt.Errorf("Error assigning IP: %w", err) } // Determine if v4 or v6. @@ -75,7 +75,7 @@ func cmdDel(args *skel.CmdArgs) error { return err } logging.Debugf("DEL - IPAM configuration successfully read: %+v", filterConf(*ipamConf)) - logging.Debugf("ContainerID: %v", args.ContainerID) + logging.Debugf("Beginning delete for ContainerID: %v", args.ContainerID) _, err = storage.IPManagement(types.Deallocate, *ipamConf, args.ContainerID) if err != nil { diff --git a/cmd/whereabouts_test.go b/cmd/whereabouts_test.go index bc93d7720..5b44e9125 100644 --- a/cmd/whereabouts_test.go +++ b/cmd/whereabouts_test.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "net" "strings" @@ -9,6 +10,7 @@ import ( "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/plugins/pkg/testutils" + "github.com/dougbtv/whereabouts/pkg/allocate" whereaboutstypes "github.com/dougbtv/whereabouts/pkg/types" . "github.com/onsi/ginkgo" @@ -159,6 +161,319 @@ var _ = Describe("Whereabouts operations", func() { AllocateAndReleaseAddressesTest(ipVersion, ipRange, ipGateway, []string{expectedAddress}, whereaboutstypes.DatastoreKubernetes) }) + It("detects IPv4 addresses used in other ranges, to allow for overlapping IP address ranges", func() { + const ifname string = "eth0" + const nspath string = "/some/where" + + // ----------------------------- range 1 + + conf := fmt.Sprintf(`{ + "cniVersion": "0.3.1", + "name": "mynet", + "type": "ipvlan", + "master": "foo0", + "ipam": { + "type": "whereabouts", + "datastore": "kubernetes", + "log_file" : "/tmp/whereabouts.log", + "log_level" : "debug", + "kubernetes": {"kubeconfig": "%s"}, + "range": "192.168.22.0/24" + } + }`, kubeConfigPath) + + args := &skel.CmdArgs{ + ContainerID: "dummyfirstrange", + Netns: nspath, + IfName: ifname, + StdinData: []byte(conf), + } + + // Allocate the IP + r, raw, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + // fmt.Printf("!bang raw: %s\n", raw) + Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0)) + + result, err := current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + // Gomega is cranky about slices with different caps + Expect(*result.IPs[0]).To(Equal( + current.IPConfig{ + Version: "4", + Address: mustCIDR("192.168.22.1/24"), + })) + + // ----------------------------- range 2 + + confsecond := fmt.Sprintf(`{ + "cniVersion": "0.3.1", + "name": "mynet", + "type": "ipvlan", + "master": "foo0", + "ipam": { + "type": "whereabouts", + "datastore": "kubernetes", + "log_file" : "/tmp/whereabouts.log", + "log_level" : "debug", + "kubernetes": {"kubeconfig": "%s"}, + "range": "192.168.22.0/28" + } + }`, kubeConfigPath) + + argssecond := &skel.CmdArgs{ + ContainerID: "dummysecondrange", + Netns: nspath, + IfName: ifname, + StdinData: []byte(confsecond), + } + + // Allocate the IP + r, raw, err = testutils.CmdAddWithArgs(argssecond, func() error { + return cmdAdd(argssecond) + }) + Expect(err).NotTo(HaveOccurred()) + // fmt.Printf("!bang raw: %s\n", raw) + Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0)) + + result, err = current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + // Gomega is cranky about slices with different caps + Expect(*result.IPs[0]).To(Equal( + current.IPConfig{ + Version: "4", + Address: mustCIDR("192.168.22.2/28"), + })) + + // ------------------------ deallocation + + // Release the IP, first range + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + + // Release the IP, second range + err = testutils.CmdDelWithArgs(argssecond, func() error { + return cmdDel(argssecond) + }) + Expect(err).NotTo(HaveOccurred()) + + }) + + It("detects IPv6 addresses used in other ranges, to allow for overlapping IP address ranges", func() { + const ifname string = "eth0" + const nspath string = "/some/where" + + // ----------------------------- range 1 + + conf := fmt.Sprintf(`{ + "cniVersion": "0.3.1", + "name": "mynet", + "type": "ipvlan", + "master": "foo0", + "ipam": { + "type": "whereabouts", + "datastore": "kubernetes", + "log_file" : "/tmp/whereabouts.log", + "log_level" : "debug", + "kubernetes": {"kubeconfig": "%s"}, + "range": "2001::2:3:0/124" + } + }`, kubeConfigPath) + + args := &skel.CmdArgs{ + ContainerID: "dummyfirstrange", + Netns: nspath, + IfName: ifname, + StdinData: []byte(conf), + } + + // Allocate the IP + r, raw, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + // fmt.Printf("!bang raw: %s\n", raw) + Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0)) + + result, err := current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + // Gomega is cranky about slices with different caps + Expect(*result.IPs[0]).To(Equal( + current.IPConfig{ + Version: "6", + Address: mustCIDR("2001::2:3:1/124"), + })) + + // ----------------------------- range 2 + + confsecond := fmt.Sprintf(`{ + "cniVersion": "0.3.1", + "name": "mynet", + "type": "ipvlan", + "master": "foo0", + "ipam": { + "type": "whereabouts", + "datastore": "kubernetes", + "log_file" : "/tmp/whereabouts.log", + "log_level" : "debug", + "kubernetes": {"kubeconfig": "%s"}, + "range": "2001::2:3:0/126" + } + }`, kubeConfigPath) + + argssecond := &skel.CmdArgs{ + ContainerID: "dummysecondrange", + Netns: nspath, + IfName: ifname, + StdinData: []byte(confsecond), + } + + // Allocate the IP + r, raw, err = testutils.CmdAddWithArgs(argssecond, func() error { + return cmdAdd(argssecond) + }) + Expect(err).NotTo(HaveOccurred()) + // fmt.Printf("!bang raw: %s\n", raw) + Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0)) + + result, err = current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + // Gomega is cranky about slices with different caps + Expect(*result.IPs[0]).To(Equal( + current.IPConfig{ + Version: "6", + Address: mustCIDR("2001::2:3:2/126"), + })) + + // ------------------------ deallocation + + // Release the IP, first range + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + + // Release the IP, second range + err = testutils.CmdDelWithArgs(argssecond, func() error { + return cmdDel(argssecond) + }) + Expect(err).NotTo(HaveOccurred()) + + }) + + It("allows IP collisions across ranges when enable_overlapping_ranges is set to false", func() { + const ifname string = "eth0" + const nspath string = "/some/where" + + // ----------------------------- range 1 + + conf := fmt.Sprintf(`{ + "cniVersion": "0.3.1", + "name": "mynet", + "type": "ipvlan", + "master": "foo0", + "ipam": { + "type": "whereabouts", + "datastore": "kubernetes", + "log_file" : "/tmp/whereabouts.log", + "log_level" : "debug", + "kubernetes": {"kubeconfig": "%s"}, + "enable_overlapping_ranges": false, + "range": "192.168.33.0/24" + } + }`, kubeConfigPath) + + args := &skel.CmdArgs{ + ContainerID: "dummyfirstrange", + Netns: nspath, + IfName: ifname, + StdinData: []byte(conf), + } + + // Allocate the IP + r, raw, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + // fmt.Printf("!bang raw: %s\n", raw) + Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0)) + + result, err := current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + // Gomega is cranky about slices with different caps + Expect(*result.IPs[0]).To(Equal( + current.IPConfig{ + Version: "4", + Address: mustCIDR("192.168.33.1/24"), + })) + + // ----------------------------- range 2 + + confsecond := fmt.Sprintf(`{ + "cniVersion": "0.3.1", + "name": "mynet", + "type": "ipvlan", + "master": "foo0", + "ipam": { + "type": "whereabouts", + "datastore": "kubernetes", + "log_file" : "/tmp/whereabouts.log", + "log_level" : "debug", + "kubernetes": {"kubeconfig": "%s"}, + "range": "192.168.33.0/28" + } + }`, kubeConfigPath) + + argssecond := &skel.CmdArgs{ + ContainerID: "dummysecondrange", + Netns: nspath, + IfName: ifname, + StdinData: []byte(confsecond), + } + + // Allocate the IP + r, raw, err = testutils.CmdAddWithArgs(argssecond, func() error { + return cmdAdd(argssecond) + }) + Expect(err).NotTo(HaveOccurred()) + // fmt.Printf("!bang raw: %s\n", raw) + Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0)) + + result, err = current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + // Gomega is cranky about slices with different caps + Expect(*result.IPs[0]).To(Equal( + current.IPConfig{ + Version: "4", + Address: mustCIDR("192.168.33.1/28"), + })) + + // ------------------------ deallocation + + // Release the IP, first range + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + + // Release the IP, second range + err = testutils.CmdDelWithArgs(argssecond, func() error { + return cmdDel(argssecond) + }) + Expect(err).NotTo(HaveOccurred()) + + }) + It("excludes a range of addresses", func() { const ifname string = "eth0" const nspath string = "/some/where" @@ -417,6 +732,82 @@ var _ = Describe("Whereabouts operations", func() { Expect(err).NotTo(HaveOccurred()) }) + It("allocates addresses using range_end as an upper limit", func() { + const ifname string = "eth0" + const nspath string = "/some/where" + + conf := fmt.Sprintf(`{ + "cniVersion": "0.3.1", + "name": "mynet", + "type": "ipvlan", + "master": "foo0", + "ipam": { + "type": "whereabouts", + "log_file" : "/tmp/whereabouts.log", + "log_level" : "debug", + "etcd_host": "%s", + "range": "192.168.1.0/24", + "range_start": "192.168.1.5", + "range_end": "192.168.1.12", + "gateway": "192.168.10.1" + } + }`, etcdHost) + + var ipArgs []*skel.CmdArgs + // allocate 8 IPs (192.168.1.5 - 192.168.1.12); the entirety of the pool defined above + for i := 0; i < 8; i++ { + args := &skel.CmdArgs{ + ContainerID: fmt.Sprintf("dummy-%d", i), + Netns: nspath, + IfName: ifname, + StdinData: []byte(conf), + } + r, raw, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + + Expect(err).NotTo(HaveOccurred()) + Expect(strings.Index(string(raw), "\"version\":")).Should(BeNumerically(">", 0)) + + result, err := current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + Expect(*result.IPs[0]).To(Equal( + current.IPConfig{ + Version: "4", + Address: mustCIDR(fmt.Sprintf("192.168.1.%d/24", 5+i)), + Gateway: net.ParseIP("192.168.10.1"), + })) + ipArgs = append(ipArgs, args) + } + + // assigning more IPs should result in error due to the defined range_start - range_end + args := &skel.CmdArgs{ + ContainerID: fmt.Sprintf("dummy-failure"), + Netns: nspath, + IfName: ifname, + StdinData: []byte(conf), + } + _, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).To(HaveOccurred()) + // ensure the error is of the correct type + switch e := errors.Unwrap(err); e.(type) { + case allocate.AssignmentError: + default: + Fail(fmt.Sprintf("expected AssignmentError, got: %s", e)) + } + + // Release assigned IPs + for _, args := range ipArgs { + err := testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + } + }) + It("fails when there's an invalid range specified", func() { const ifname string = "eth0" const nspath string = "/some/where" diff --git a/doc/daemonset-install.yaml b/doc/daemonset-install.yaml index 6cc3436be..e65440b32 100644 --- a/doc/daemonset-install.yaml +++ b/doc/daemonset-install.yaml @@ -27,6 +27,7 @@ rules: - whereabouts.cni.cncf.io resources: - ippools + - overlappingrangeipreservations verbs: - get - list @@ -57,6 +58,7 @@ spec: app: whereabouts name: whereabouts spec: + hostNetwork: true serviceAccountName: whereabouts nodeSelector: beta.kubernetes.io/arch: amd64 diff --git a/doc/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml b/doc/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml new file mode 100644 index 000000000..99197630e --- /dev/null +++ b/doc/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml @@ -0,0 +1,47 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: overlappingrangeipreservations.whereabouts.cni.cncf.io +spec: + group: whereabouts.cni.cncf.io + names: + kind: OverlappingRangeIPReservation + plural: overlappingrangeipreservations + scope: "" + validation: + openAPIV3Schema: + description: OverlappingRangeIPReservation is the Schema for the OverlappingRangeIPReservations + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OverlappingRangeIPReservationSpec defines the desired state + of OverlappingRangeIPReservation + properties: + containerid: + type: string + required: + - containerid + type: object + required: + - spec + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true \ No newline at end of file diff --git a/pkg/allocate/allocate.go b/pkg/allocate/allocate.go index e287bf240..a3272be1e 100644 --- a/pkg/allocate/allocate.go +++ b/pkg/allocate/allocate.go @@ -2,12 +2,24 @@ package allocate import ( "fmt" - "github.com/dougbtv/whereabouts/pkg/logging" - "github.com/dougbtv/whereabouts/pkg/types" "math/big" "net" + + "github.com/dougbtv/whereabouts/pkg/logging" + "github.com/dougbtv/whereabouts/pkg/types" ) +// AssignmentError defines an IP assignment error. +type AssignmentError struct { + firstIP net.IP + lastIP net.IP + ipnet net.IPNet +} + +func (a AssignmentError) Error() string { + return fmt.Sprintf("Could not allocate IP in range: ip: %v / - %v / range: %#v", a.firstIP, a.lastIP, a.ipnet) +} + // AssignIP assigns an IP using a range and a reserve list. func AssignIP(ipamConf types.IPAMConfig, reservelist []types.IPReservation, containerID string) (net.IPNet, []types.IPReservation, error) { @@ -23,18 +35,20 @@ func AssignIP(ipamConf types.IPAMConfig, reservelist []types.IPReservation, cont } // DeallocateIP assigns an IP using a range and a reserve list. -func DeallocateIP(iprange string, reservelist []types.IPReservation, containerID string) ([]types.IPReservation, error) { +func DeallocateIP(ipamConf types.IPAMConfig, iprange string, reservelist []types.IPReservation, containerID string) ([]types.IPReservation, net.IP, error) { - updatedreservelist, err := IterateForDeallocation(reservelist, containerID) + updatedreservelist, hadip, err := IterateForDeallocation(reservelist, containerID) if err != nil { - return nil, err + return nil, nil, err } - return updatedreservelist, nil + logging.Debugf("Deallocating given previously used IP: %v", hadip) + + return updatedreservelist, hadip, nil } // IterateForDeallocation iterates overs currently reserved IPs and the deallocates given the container id. -func IterateForDeallocation(reservelist []types.IPReservation, containerID string) ([]types.IPReservation, error) { +func IterateForDeallocation(reservelist []types.IPReservation, containerID string) ([]types.IPReservation, net.IP, error) { // Cycle through and find the index that corresponds to our containerID foundidx := -1 @@ -47,11 +61,13 @@ func IterateForDeallocation(reservelist []types.IPReservation, containerID strin // Check if it's a valid index if foundidx < 0 { - return reservelist, fmt.Errorf("Did not find reserved IP for container %v", containerID) + return reservelist, nil, fmt.Errorf("Did not find reserved IP for container %v", containerID) } + returnip := reservelist[foundidx].IP + updatedreservelist := removeIdxFromSlice(reservelist, foundidx) - return updatedreservelist, nil + return updatedreservelist, returnip, nil } func removeIdxFromSlice(s []types.IPReservation, i int) []types.IPReservation { @@ -65,9 +81,7 @@ func IterateForAssignment(ipnet net.IPNet, rangeStart net.IP, rangeEnd net.IP, r firstip := rangeStart var lastip net.IP if rangeEnd != nil { - end := IPToBigInt(rangeEnd) - end = end.Add(end, big.NewInt(1)) - lastip = BigIntToIP(*end) + lastip = rangeEnd } else { var err error firstip, lastip, err = GetIPRange(rangeStart, ipnet) @@ -134,7 +148,7 @@ MAINITERATION: } if !performedassignment { - return net.IP{}, reservelist, fmt.Errorf("Could not allocate IP in range: ip: %v / - %v / range: %#v", firstip, lastip, ipnet) + return net.IP{}, reservelist, AssignmentError{firstip, lastip, ipnet} } return assignedip, reservelist, nil diff --git a/pkg/api/v1alpha1/overlappingrangeipreservation_types.go b/pkg/api/v1alpha1/overlappingrangeipreservation_types.go new file mode 100644 index 000000000..486029dc1 --- /dev/null +++ b/pkg/api/v1alpha1/overlappingrangeipreservation_types.go @@ -0,0 +1,32 @@ +package v1alpha1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +// OverlappingRangeIPReservationSpec defines the desired state of OverlappingRangeIPReservation +type OverlappingRangeIPReservationSpec struct { + ContainerID string `json:"containerid"` +} + +// +kubebuilder:object:root=true + +// OverlappingRangeIPReservation is the Schema for the OverlappingRangeIPReservations API +type OverlappingRangeIPReservation struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec OverlappingRangeIPReservationSpec `json:"spec"` +} + +// +kubebuilder:object:root=true + +// OverlappingRangeIPReservationList contains a list of OverlappingRangeIPReservation +type OverlappingRangeIPReservationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []OverlappingRangeIPReservation `json:"items"` +} + +func init() { + SchemeBuilder.Register(&OverlappingRangeIPReservation{}, &OverlappingRangeIPReservationList{}) +} diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 3f23132f7..f741cd5cb 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -102,3 +102,76 @@ func (in *IPPoolSpec) DeepCopy() *IPPoolSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OverlappingRangeIPReservation) DeepCopyInto(out *OverlappingRangeIPReservation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OverlappingRangeIPReservation. +func (in *OverlappingRangeIPReservation) DeepCopy() *OverlappingRangeIPReservation { + if in == nil { + return nil + } + out := new(OverlappingRangeIPReservation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OverlappingRangeIPReservation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OverlappingRangeIPReservationList) DeepCopyInto(out *OverlappingRangeIPReservationList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]OverlappingRangeIPReservation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OverlappingRangeIPReservationList. +func (in *OverlappingRangeIPReservationList) DeepCopy() *OverlappingRangeIPReservationList { + if in == nil { + return nil + } + out := new(OverlappingRangeIPReservationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OverlappingRangeIPReservationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OverlappingRangeIPReservationSpec) DeepCopyInto(out *OverlappingRangeIPReservationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OverlappingRangeIPReservationSpec. +func (in *OverlappingRangeIPReservationSpec) DeepCopy() *OverlappingRangeIPReservationSpec { + if in == nil { + return nil + } + out := new(OverlappingRangeIPReservationSpec) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/config/config.go b/pkg/config/config.go index fb308e9b8..c2429a318 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -33,7 +33,11 @@ func canonicalizeIP(ip *net.IP) error { func LoadIPAMConfig(bytes []byte, envArgs string) (*types.IPAMConfig, string, error) { // We first load up what we already have, before we start reading a file... - n := types.Net{} + n := types.Net{ + IPAM: &types.IPAMConfig{ + OverlappingRanges: true, + }, + } if err := json.Unmarshal(bytes, &n); err != nil { return nil, "", fmt.Errorf("LoadIPAMConfig - JSON Parsing Error: %s / bytes: %s", err, bytes) } diff --git a/pkg/storage/etcd.go b/pkg/storage/etcd.go index 83c18873e..cea3a582f 100644 --- a/pkg/storage/etcd.go +++ b/pkg/storage/etcd.go @@ -10,6 +10,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/concurrency" "github.com/coreos/etcd/pkg/transport" + "github.com/dougbtv/whereabouts/pkg/logging" "github.com/dougbtv/whereabouts/pkg/types" ) @@ -74,6 +75,28 @@ func (i *ETCDIPAM) Status(ctx context.Context) error { return err } +// EtcdOverlappingRangeStore represents a set of cluster wide resources +type EtcdOverlappingRangeStore struct { + client *clientv3.Client +} + +// GetOverlappingRangeStore returns an OverlappingRangeStore interface +func (i *ETCDIPAM) GetOverlappingRangeStore() (OverlappingRangeStore, error) { + return &EtcdOverlappingRangeStore{i.client}, nil +} + +// IsAllocatedInOverlappingRange checks to see if the IP is allocated across the whole cluster (and not just the current range) +func (i *EtcdOverlappingRangeStore) IsAllocatedInOverlappingRange(ctx context.Context, ip net.IP) (bool, error) { + logging.Debugf("ETCD IsAllocatedInOverlappingRange is NOT IMPLEMENTED!!!! TODO") + return false, nil +} + +// UpdateOverlappingRangeAllocation updates our clusterwide allocation for overlapping ranges. +func (i *EtcdOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx context.Context, mode int, ip net.IP, containerID string) error { + logging.Debugf("ETCD UpdateOverlappingRangeWide is NOT IMPLEMENTED!!!! TODO") + return nil +} + // Close shuts down the clients etcd connections func (i *ETCDIPAM) Close() error { defer i.client.Close() diff --git a/pkg/storage/kubernetes.go b/pkg/storage/kubernetes.go index 42c8f2544..aff9291f1 100644 --- a/pkg/storage/kubernetes.go +++ b/pkg/storage/kubernetes.go @@ -101,7 +101,7 @@ func toAllocationMap(reservelist []whereaboutstypes.IPReservation, firstip net.I // GetIPPool returns a storage.IPPool for the given range func (i *KubernetesIPAM) GetIPPool(ctx context.Context, ipRange string) (IPPool, error) { // v6 filter - normalized := strings.ReplaceAll(ipRange, "::", "-") + normalized := strings.ReplaceAll(ipRange, ":", "-") // replace subnet cidr slash normalized = strings.ReplaceAll(normalized, "/", "-") @@ -154,6 +154,80 @@ func (i *KubernetesIPAM) Close() error { return nil } +// KubernetesOverlappingRangeStore represents a OverlappingRangeStore interface +type KubernetesOverlappingRangeStore struct { + client client.Client + containerID string + namespace string +} + +// GetOverlappingRangeStore returns a clusterstore interface +func (i *KubernetesIPAM) GetOverlappingRangeStore() (OverlappingRangeStore, error) { + return &KubernetesOverlappingRangeStore{i.client, i.containerID, i.namespace}, nil +} + +// IsAllocatedInOverlappingRange checks for IP addresses to see if they're allocated cluster wide, for overlapping ranges +func (c *KubernetesOverlappingRangeStore) IsAllocatedInOverlappingRange(ctx context.Context, ip net.IP) (bool, error) { + + // IPv6 doesn't make for valid CR names, so normalize it. + normalizedip := strings.ReplaceAll(fmt.Sprint(ip), ":", "-") + + logging.Debugf("OverlappingRangewide allocation check for IP: %v", normalizedip) + + // clusteripres := &whereaboutsv1alpha1.OverlappingRangeIPReservation{ + // ObjectMeta: metav1.ObjectMeta{Name: normalizedip, Namespace: i.namespace}, + // } + clusteripres := &whereaboutsv1alpha1.OverlappingRangeIPReservation{ + ObjectMeta: metav1.ObjectMeta{Name: normalizedip, Namespace: c.namespace}, + } + if err := c.client.Get(ctx, types.NamespacedName{Name: normalizedip, Namespace: c.namespace}, clusteripres); errors.IsNotFound(err) { + // cluster ip reservation does not exist, this appears to be good news. + // logging.Debugf("IP %v is not reserved cluster wide, allowing.", ip) + return false, nil + } else if err != nil { + logging.Errorf("k8s get OverlappingRangeIPReservation error: %s", err) + return false, fmt.Errorf("k8s get OverlappingRangeIPReservation error: %s", err) + } + + logging.Debugf("IP %v is reserved cluster wide.", ip) + return true, nil +} + +// UpdateOverlappingRangeAllocation updates clusterwide allocation for overlapping ranges. +func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx context.Context, mode int, ip net.IP, containerID string) error { + // Normalize the IP + normalizedip := strings.ReplaceAll(fmt.Sprint(ip), ":", "-") + + clusteripres := &whereaboutsv1alpha1.OverlappingRangeIPReservation{ + ObjectMeta: metav1.ObjectMeta{Name: normalizedip, Namespace: c.namespace}, + } + + var err error + var verb string + switch mode { + case whereaboutstypes.Allocate: + // Put together our cluster ip reservation + verb = "allocate" + + clusteripres.Spec = whereaboutsv1alpha1.OverlappingRangeIPReservationSpec{ + ContainerID: containerID, + } + + err = c.client.Create(ctx, clusteripres) + + case whereaboutstypes.Deallocate: + verb = "deallocate" + err = c.client.Delete(ctx, clusteripres) + } + + if err != nil { + return err + } + + logging.Debugf("K8s UpdateOverlappingRangeAllocation success on %v: %+v", verb, clusteripres) + return nil +} + // KubernetesIPPool represents an IPPool resource and its parsed set of allocations type KubernetesIPPool struct { client client.Client @@ -192,7 +266,7 @@ func (p *KubernetesIPPool) Update(ctx context.Context, reservations []whereabout // add additional tests to the patch ops := []jsonpatch.Operation{ // ensure patch is applied to appropriate resource version only - jsonpatch.Operation{Operation: "test", Path: "/metadata/resourceVersion", Value: orig.ObjectMeta.ResourceVersion}, + {Operation: "test", Path: "/metadata/resourceVersion", Value: orig.ObjectMeta.ResourceVersion}, } for _, o := range patch { // safeguard add ops -- "add" will update existing paths, this "test" ensures the path is empty diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index d7493355e..cb197b7c3 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -28,10 +28,17 @@ type IPPool interface { // Store is the interface that wraps the basic IP Allocation methods on the underlying storage backend type Store interface { GetIPPool(ctx context.Context, ipRange string) (IPPool, error) + GetOverlappingRangeStore() (OverlappingRangeStore, error) Status(ctx context.Context) error Close() error } +// OverlappingRangeStore is an interface for wrapping overlappingrange storage options +type OverlappingRangeStore interface { + IsAllocatedInOverlappingRange(ctx context.Context, ip net.IP) (bool, error) + UpdateOverlappingRangeAllocation(ctx context.Context, mode int, ip net.IP, containerID string) error +} + // IPManagement manages ip allocation and deallocation from a storage perspective func IPManagement(mode int, ipamConf types.IPAMConfig, containerID string) (net.IPNet, error) { @@ -46,6 +53,7 @@ func IPManagement(mode int, ipamConf types.IPAMConfig, containerID string) (net. } var ipam Store + var overlappingrangestore OverlappingRangeStore var pool IPPool var err error switch ipamConf.Datastore { @@ -70,6 +78,8 @@ func IPManagement(mode int, ipamConf types.IPAMConfig, containerID string) (net. } // handle the ip add/del until successful + var overlappingrangeallocations []types.IPReservation + var ipforoverlappingrangeupdate net.IP RETRYLOOP: for j := 0; j < DatastoreRetries; j++ { select { @@ -79,6 +89,12 @@ RETRYLOOP: // retry the IPAM loop if the context has not been cancelled } + overlappingrangestore, err = ipam.GetOverlappingRangeStore() + if err != nil { + logging.Errorf("IPAM error getting OverlappingRangeStore: %v", err) + return newip, err + } + pool, err = ipam.GetIPPool(ctx, ipamConf.Range) if err != nil { logging.Errorf("IPAM error reading pool allocations (attempt: %d): %v", j, err) @@ -89,6 +105,7 @@ RETRYLOOP: } reservelist := pool.Allocations() + reservelist = append(reservelist, overlappingrangeallocations...) var updatedreservelist []types.IPReservation switch mode { case types.Allocate: @@ -97,15 +114,43 @@ RETRYLOOP: logging.Errorf("Error assigning IP: %v", err) return newip, err } + // Now check if this is allocated overlappingrange wide + // When it's allocated overlappingrange wide, we add it to a local reserved list + // And we try again. + if ipamConf.OverlappingRanges { + isallocated, err := overlappingrangestore.IsAllocatedInOverlappingRange(ctx, newip.IP) + if err != nil { + logging.Errorf("Error checking overlappingrange allocation: %v", err) + return newip, err + } + + if isallocated { + logging.Debugf("Continuing loop, IP is already allocated (possibly from another range): %v", newip) + // We create "dummy" records here for evaluation, but, we need to filter those out later. + overlappingrangeallocations = append(overlappingrangeallocations, types.IPReservation{IP: newip.IP, IsAllocated: true}) + continue + } + + ipforoverlappingrangeupdate = newip.IP + } + case types.Deallocate: - updatedreservelist, err = allocate.DeallocateIP(ipamConf.Range, reservelist, containerID) + updatedreservelist, ipforoverlappingrangeupdate, err = allocate.DeallocateIP(ipamConf, ipamConf.Range, reservelist, containerID) if err != nil { logging.Errorf("Error deallocating IP: %v", err) return newip, err } } - err = pool.Update(ctx, updatedreservelist) + // Clean out any dummy records from the reservelist... + var usereservelist []types.IPReservation + for _, rl := range updatedreservelist { + if rl.IsAllocated != true { + usereservelist = append(usereservelist, rl) + } + } + + err = pool.Update(ctx, usereservelist) if err != nil { logging.Errorf("IPAM error updating pool (attempt: %d): %v", j, err) if e, ok := err.(temporary); ok && e.Temporary() { @@ -116,5 +161,13 @@ RETRYLOOP: break RETRYLOOP } + if ipamConf.OverlappingRanges { + err = overlappingrangestore.UpdateOverlappingRangeAllocation(ctx, mode, ipforoverlappingrangeupdate, containerID) + if err != nil { + logging.Errorf("Error performing UpdateOverlappingRangeAllocation: %v", err) + return newip, err + } + } + return newip, err } diff --git a/pkg/types/types.go b/pkg/types/types.go index e06f789e7..57b99fd9f 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -41,6 +41,7 @@ type IPAMConfig struct { EtcdCACertFile string `json:"etcd_ca_cert_file,omitempty"` LogFile string `json:"log_file"` LogLevel string `json:"log_level"` + OverlappingRanges bool `json:"enable_overlapping_ranges,omitempty"` Gateway net.IP Kubernetes KubernetesConfig `json:"kubernetes,omitempty"` ConfigurationPath string `json:"configuration_path"` @@ -71,6 +72,7 @@ type Address struct { type IPReservation struct { IP net.IP `json:"ip"` ContainerID string `json:"id"` + IsAllocated bool } const (