Start getting FVs set up to work with KDD mode

Makefile

@@ -116,6 +116,7 @@ SRCFILES=cmd/kube-controllers/main.go $(shell find pkg -name '*.go')
 clean:
 	rm -rf bin image.created-$(ARCH)
 	-docker rmi $(BUILD_IMAGE)
+	-docker rmi $(BUILD_IMAGE):latest-amd64
 	rm -f tests/fv/fv.test
 	rm -f report/*.xml
 
@@ -318,7 +319,12 @@ ut: vendor
 ## Build and run the FV tests.
 fv: tests/fv/fv.test image
 	@echo Running Go FVs.
-	cd tests/fv && ETCD_IMAGE=$(ETCD_IMAGE) HYPERKUBE_IMAGE=$(HYPERKUBE_IMAGE) CONTAINER_NAME=$(BUILD_IMAGE):latest-$(ARCH) PRIVATE_KEY=`pwd`/private.key ./fv.test $(GINKGO_ARGS) -ginkgo.slowSpecThreshold 30
+	cd tests/fv && ETCD_IMAGE=$(ETCD_IMAGE) \
+		HYPERKUBE_IMAGE=$(HYPERKUBE_IMAGE) \
+		CONTAINER_NAME=$(BUILD_IMAGE):latest-$(ARCH) \
+		PRIVATE_KEY=`pwd`/private.key \
+		CRDS_FILE=${PWD}/vendor/github.com/projectcalico/libcalico-go/test/crds.yaml \
+		./fv.test $(GINKGO_ARGS) -ginkgo.slowSpecThreshold 30
 
 tests/fv/fv.test: $(shell find ./tests -type f -name '*.go' -print)
 	# We pre-build the test binary so that we can run it outside a container and allow it

glide.yaml

@@ -14,7 +14,7 @@ import:
   subpackages:
   - fv/containers
 - package: github.com/projectcalico/libcalico-go
-  version: 61a81ef2608e8744731f00d878b8c76a3ab95486
+  version: 13c0e7d9fbf4de88aefdeb92df0e503b6123098e
   subpackages:
   - lib/apiconfig
   - lib/apis/v3

pkg/controllers/node/etcd.go

@@ -1,3 +1,17 @@
+// Copyright (c) 2019 Tigera, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package node
 
 import (

pkg/controllers/node/kdd.go

@@ -1,3 +1,17 @@
+// Copyright (c) 2019 Tigera, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package node
 
 import (
@@ -13,11 +27,6 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-type allocation struct {
-	handle     string
-	attributes map[string]string
-}
-
 // syncDeleteKDD cleans up any IPAM resources which should no longer exist based on nodes in the cluster.
 // It returns an error if it is determined that there are resources which should be cleaned up, but is unable to do so.
 // It does not return an error if it is successful, or if there is no action to take.
@@ -38,7 +47,7 @@ func (c *NodeController) syncDeleteKDD() error {
 
 	// Build a list of all the nodes in the cluster based on IPAM allocations across all
 	// blocks, plus affinities.
-	nodes := map[string][]allocation{}
+	nodes := map[string][]model.AllocationAttribute{}
 	for _, kvp := range blocks.KVPairs {
 		b := kvp.Value.(*model.AllocationBlock)
 
@@ -47,7 +56,7 @@ func (c *NodeController) syncDeleteKDD() error {
 		if b.Affinity != nil {
 			n := strings.TrimLeft(*b.Affinity, "host:")
 			if _, ok := nodes[n]; !ok {
-				nodes[n] = []allocation{}
+				nodes[n] = []model.AllocationAttribute{}
 			}
 		}
 
@@ -62,7 +71,7 @@ func (c *NodeController) syncDeleteKDD() error {
 			// Track nodes based on IP allocations.
 			if val, ok := attr.AttrSecondary["node"]; ok {
 				if _, ok := nodes[val]; !ok {
-					nodes[val] = []allocation{}
+					nodes[val] = []model.AllocationAttribute{}
 				}
 
 				// If there is no handle, then skip this IP. We need the handle
@@ -74,10 +83,7 @@ func (c *NodeController) syncDeleteKDD() error {
 
 				// Add this allocation to the node, so we can release it later if
 				// we need to.
-				nodes[val] = append(nodes[val], allocation{
-					handle:     *attr.AttrPrimary,
-					attributes: attr.AttrSecondary,
-				})
+				nodes[val] = append(nodes[val], attr)
 			}
 		}
 	}
@@ -99,8 +105,8 @@ func (c *NodeController) syncDeleteKDD() error {
 		// extra sure that the node is gone before we clean it up.
 		canDelete := true
 		for _, a := range allocations {
-			ns := a.attributes["namespace"]
-			pod := a.attributes["pod"]
+			ns := a.AttrSecondary["namespace"]
+			pod := a.AttrSecondary["pod"]
 
 			// TODO: Need to handle IPIP addresses somehow. They are allocated in calico-ipam, but
 			// currently don't have this metadata attached.
@@ -138,31 +144,23 @@ func (c *NodeController) syncDeleteKDD() error {
 	return nil
 }
 
-func (c *NodeController) cleanupNode(node string, allocations []allocation) error {
-	// Release the affinities for this node.
-	logc := log.WithField("node", node)
-	if err := c.calicoClient.IPAM().ReleaseHostAffinities(c.ctx, node); err != nil {
-		logc.WithError(err).Errorf("Failed to release node affinity")
-		return err
-	}
-	logc.Debug("Released all affinities for node")
-
+func (c *NodeController) cleanupNode(node string, allocations []model.AllocationAttribute) error {
 	// At this point, we've verified that the node isn't in Kubernetes and that all the allocations
-	// are tied to pods which don't exist any more. We've released the affinity, so clean up
-	// any allocations which may still be laying around.
+	// are tied to pods which don't exist any more. Clean up any allocations which may still be laying around.
+	logc := log.WithField("node", node)
 	retry := false
 	for _, a := range allocations {
-		if err := c.calicoClient.IPAM().ReleaseByHandle(c.ctx, a.handle); err != nil {
+		if err := c.calicoClient.IPAM().ReleaseByHandle(c.ctx, *a.AttrPrimary); err != nil {
 			if _, ok := err.(cerrors.ErrorResourceDoesNotExist); ok {
 				// If it doesn't exist, we're OK, since we don't want it to!
 				// Try to release any other allocations, but we'll still return an error
 				// to retry the whole thing from the top. On the retry,
 				// we should no longer see any allocations.
-				logc.WithField("handle", a.handle).Debug("IP already released")
+				logc.WithField("handle", *a.AttrPrimary).Debug("IP already released")
 				retry = true
 				continue
 			}
-			logc.WithError(err).WithField("handle", a.handle).Warning("Failed to release IP")
+			logc.WithError(err).WithField("handle", *a.AttrPrimary).Warning("Failed to release IP")
 			retry = true
 			break
 		}
@@ -173,6 +171,13 @@ func (c *NodeController) cleanupNode(node string, allocations []allocation) erro
 		return fmt.Errorf("Couldn't release all IPs")
 	}
 
+	// Release the affinities for this node, requiring that the blocks are empty.
+	if err := c.calicoClient.IPAM().ReleaseHostAffinities(c.ctx, node, true); err != nil {
+		logc.WithError(err).Errorf("Failed to release block affinities for node")
+		return err
+	}
+	logc.Debug("Released all affinities for node")
+
 	return nil
 }
 

tests/fv/fv_resiliency_test.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/projectcalico/felix/fv/containers"
 	"github.com/projectcalico/kube-controllers/tests/testutils"
+	"github.com/projectcalico/libcalico-go/lib/apiconfig"
 	api "github.com/projectcalico/libcalico-go/lib/apis/v3"
 	client "github.com/projectcalico/libcalico-go/lib/clientv3"
 	"github.com/projectcalico/libcalico-go/lib/options"
@@ -52,20 +53,20 @@ var _ = Describe("[Resilience] PolicyController", func() {
 	BeforeEach(func() {
 		// Run etcd.
 		calicoEtcd = testutils.RunEtcd()
-		calicoClient = testutils.GetCalicoClient(calicoEtcd.IP)
+		calicoClient = testutils.GetCalicoClient(apiconfig.EtcdV3, calicoEtcd.IP, "")
 
 		// Run apiserver.
 		k8sEtcd = testutils.RunEtcd()
 		apiserver = testutils.RunK8sApiserver(k8sEtcd.IP)
 
 		// Write out a kubeconfig file
-		kfconfigfile, err := ioutil.TempFile("", "ginkgo-policycontroller")
+		kconfigfile, err := ioutil.TempFile("", "ginkgo-policycontroller")
 		Expect(err).NotTo(HaveOccurred())
-		defer os.Remove(kfconfigfile.Name())
+		defer os.Remove(kconfigfile.Name())
 		data := fmt.Sprintf(testutils.KubeconfigTemplate, apiserver.IP)
-		kfconfigfile.Write([]byte(data))
+		kconfigfile.Write([]byte(data))
 
-		k8sClient, err = testutils.GetK8sClient(kfconfigfile.Name())
+		k8sClient, err = testutils.GetK8sClient(kconfigfile.Name())
 		Expect(err).NotTo(HaveOccurred())
 
 		// Wait for the apiserver to be available and for the default namespace to exist.
@@ -100,7 +101,7 @@ var _ = Describe("[Resilience] PolicyController", func() {
 			Do().Error()
 		Expect(err).NotTo(HaveOccurred())
 
-		policyController = testutils.RunPolicyController(calicoEtcd.IP, kfconfigfile.Name())
+		policyController = testutils.RunPolicyController(apiconfig.EtcdV3, calicoEtcd.IP, kconfigfile.Name(), "")
 
 		// Wait for it to appear in Calico's etcd.
 		Eventually(func() *api.NetworkPolicy {

tests/fv/fv_test.go

@@ -36,6 +36,7 @@ import (
 
 	"github.com/projectcalico/felix/fv/containers"
 	"github.com/projectcalico/kube-controllers/tests/testutils"
+	"github.com/projectcalico/libcalico-go/lib/apiconfig"
 	api "github.com/projectcalico/libcalico-go/lib/apis/v3"
 	client "github.com/projectcalico/libcalico-go/lib/clientv3"
 	"github.com/projectcalico/libcalico-go/lib/ipam"
@@ -62,21 +63,21 @@ var _ = Describe("kube-controllers FV tests", func() {
 	BeforeEach(func() {
 		// Run etcd.
 		etcd = testutils.RunEtcd()
-		calicoClient = testutils.GetCalicoClient(etcd.IP)
+		calicoClient = testutils.GetCalicoClient(apiconfig.EtcdV3, etcd.IP, "")
 
 		// Run apiserver.
 		apiserver = testutils.RunK8sApiserver(etcd.IP)
 
 		// Write out a kubeconfig file
-		kfconfigfile, err := ioutil.TempFile("", "ginkgo-policycontroller")
+		kconfigfile, err := ioutil.TempFile("", "ginkgo-policycontroller")
 		Expect(err).NotTo(HaveOccurred())
-		defer os.Remove(kfconfigfile.Name())
+		defer os.Remove(kconfigfile.Name())
 		data := fmt.Sprintf(testutils.KubeconfigTemplate, apiserver.IP)
-		kfconfigfile.Write([]byte(data))
+		kconfigfile.Write([]byte(data))
 
-		policyController = testutils.RunPolicyController(etcd.IP, kfconfigfile.Name())
+		policyController = testutils.RunPolicyController(apiconfig.EtcdV3, etcd.IP, kconfigfile.Name(), "")
 
-		k8sClient, err = testutils.GetK8sClient(kfconfigfile.Name())
+		k8sClient, err = testutils.GetK8sClient(kconfigfile.Name())
 		Expect(err).NotTo(HaveOccurred())
 
 		// Wait for the apiserver to be available.
@@ -753,7 +754,7 @@ var _ = Describe("kube-controllers FV tests", func() {
 			wep.Name = wepName
 			wep.Namespace = podNamespace
 			wep.Labels = map[string]string{
-				"foo":                            "label1",
+				"foo": "label1",
 				"projectcalico.org/namespace":    podNamespace,
 				"projectcalico.org/orchestrator": api.OrchestratorKubernetes,
 			}