IPAM utilization reporting

Makefile

@@ -5,7 +5,7 @@ default: build
 all: build
 
 ## Run the tests for the current platform/architecture
-test: ut st
+test: ut fv st
 
 ###############################################################################
 # Both native and cross architecture builds are supported.
@@ -143,7 +143,7 @@ build-all: $(addprefix bin/calicoctl-linux-,$(VALIDARCHES)) bin/calicoctl-window
 build: bin/calicoctl-$(OS)-$(ARCH)
 
 ## Create the vendor directory
-vendor: glide.yaml
+vendor: glide.lock
 	# Ensure that the glide cache directory exists.
 	mkdir -p $(HOME)/.glide
 
@@ -319,6 +319,21 @@ ut: bin/calicoctl-linux-amd64
 		$(LOCAL_BUILD_MOUNTS) \
 		$(CALICO_BUILD) sh -c 'cd /go/src/$(PACKAGE_NAME) && ginkgo -cover -r --skipPackage vendor calicoctl/* $(GINKGO_ARGS)'
 
+###############################################################################
+# FVs
+###############################################################################
+.PHONY: fv
+## Run the tests in a container. Useful for CI, Mac dev.
+fv: bin/calicoctl-linux-amd64
+	$(MAKE) run-etcd-host
+	docker run --rm -v $(CURDIR):/go/src/$(PACKAGE_NAME):rw \
+		--net=host -e LOCAL_USER_ID=$(LOCAL_USER_ID) \
+		$(LOCAL_BUILD_MOUNTS) \
+	        -v $(CURDIR)/.go-pkg-cache:/go-cache/:rw \
+                -e GOCACHE=/go-cache \
+		$(CALICO_BUILD) sh -c 'cd /go/src/$(PACKAGE_NAME) && go test ./tests/fv'
+	$(MAKE) stop-etcd
+
 ###############################################################################
 # STs
 ###############################################################################
@@ -329,7 +344,8 @@ ST_OPTIONS?=
 
 .PHONY: st
 ## Run the STs in a container
-st: bin/calicoctl-linux-amd64 run-etcd-host
+st: bin/calicoctl-linux-amd64
+	$(MAKE) run-etcd-host
 	# Use the host, PID and network namespaces from the host.
 	# Privileged is needed since 'calico node' write to /proc (to enable ip_forwarding)
 	# Map the docker socket in so docker can be used from inside the container
@@ -342,7 +358,6 @@ st: bin/calicoctl-linux-amd64 run-etcd-host
 	           -v /var/run/docker.sock:/var/run/docker.sock \
 	           $(TEST_CONTAINER_NAME) \
 	           sh -c 'nosetests $(ST_TO_RUN) -sv --nologcapture  --with-xunit --xunit-file="/code/report/nosetests.xml" --with-timer $(ST_OPTIONS)'
-
 	$(MAKE) stop-etcd
 
 ## Etcd is used by the STs
@@ -383,7 +398,7 @@ foss-checks: vendor
 ###############################################################################
 .PHONY: ci
 ## Run what CI runs
-ci: clean build-all static-checks ut st image-all
+ci: clean build-all static-checks test image-all
 
 ###############################################################################
 # CD

calicoctl/commands/ipam.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2016 Tigera, Inc. All rights reserved.
+// Copyright (c) 2016-2019 Tigera, Inc. All rights reserved.
 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ import (
 	"strings"
 
 	"github.com/docopt/docopt-go"
+
 	"github.com/projectcalico/calicoctl/calicoctl/commands/constants"
 	"github.com/projectcalico/calicoctl/calicoctl/commands/ipam"
 )
@@ -29,7 +30,8 @@ func IPAM(args []string) error {
   calicoctl ipam <command> [<args>...]
 
     release      Release a Calico assigned IP address.
-    show         Show details of a Calico assigned IP address.
+    show         Show details of a Calico assigned IP address,
+                 or of overall IP usage.
 
 Options:
   -h --help      Show this screen.

calicoctl/commands/ipam/show.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2016 Tigera, Inc. All rights reserved.
+// Copyright (c) 2016-2019 Tigera, Inc. All rights reserved.
 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -17,31 +17,36 @@ package ipam
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 
+	"github.com/olekukonko/tablewriter"
+
 	"github.com/projectcalico/calicoctl/calicoctl/commands/constants"
+	"github.com/projectcalico/libcalico-go/lib/ipam"
 
 	docopt "github.com/docopt/docopt-go"
+
 	"github.com/projectcalico/calicoctl/calicoctl/commands/argutils"
 	"github.com/projectcalico/calicoctl/calicoctl/commands/clientmgr"
 )
 
 // IPAM takes keyword with an IP address then calls the subcommands.
 func Show(args []string) error {
 	doc := constants.DatastoreIntro + `Usage:
-  calicoctl ipam show --ip=<IP> [--config=<CONFIG>]
+  calicoctl ipam show [--ip=<IP> | --show-blocks] [--config=<CONFIG>]
 
 Options:
   -h --help             Show this screen.
-     --ip=<IP>          IP address to show.
+     --ip=<IP>          Report whether this specific IP address is in use.
+     --show-blocks      Show detailed information for IP blocks as well as pools.
   -c --config=<CONFIG>  Path to the file containing connection configuration in
                         YAML or JSON format.
                         [default: ` + constants.DefaultConfigPath + `]
 
 Description:
-  The ipam show command prints information about a given IP address, such as
-  special attributes defined for the IP or whether the IP has been reserved by
-  a user of the Calico IP Address Manager.
+  The ipam show command prints information about a given IP address, or about
+  overall IP usage.
 `
 	parsedArgs, err := docopt.Parse(doc, args, true, "", false, false)
 	if err != nil {
@@ -60,26 +65,64 @@ Description:
 		fmt.Println(err)
 	}
 
+	showBlocks := parsedArgs["--show-blocks"].(bool)
+
 	ipamClient := client.IPAM()
-	passedIP := parsedArgs["--ip"].(string)
-	ip := argutils.ValidateIP(passedIP)
-	attr, err := ipamClient.GetAssignmentAttributes(ctx, ip)
+	if passedIP := parsedArgs["--ip"]; passedIP != nil {
+		ip := argutils.ValidateIP(passedIP.(string))
+		attr, err := ipamClient.GetAssignmentAttributes(ctx, ip)
 
-	// IP address is not assigned, this prints message like
-	// `IP 192.168.71.1 is not assigned in block`. This is not exactly an error,
-	// so not returning it to the caller.
-	if err != nil {
-		fmt.Println(err)
+		// IP address is not assigned, this prints message like `IP 192.168.71.1
+		// is not assigned in block`. This is not exactly an error, so not
+		// returning it to the caller.
+		if err != nil {
+			fmt.Println(err)
+			return nil
+		}
+
+		// IP address is assigned.
+		fmt.Printf("IP %s is in use\n", ip)
+		if len(attr) != 0 {
+			fmt.Printf("Attributes: %v\n", attr)
+		} else {
+			fmt.Println("No attributes defined")
+		}
 		return nil
 	}
 
-	// IP address is assigned with attributes.
-	if len(attr) != 0 {
-		fmt.Println(attr)
-	} else {
-		// IP address is assigned but attributes are not set.
-		fmt.Printf("No attributes defined for IP %s\n", ip)
+	usage, err := ipamClient.GetUtilization(ctx, ipam.GetUtilizationArgs{})
+	if err != nil {
+		return err
+	}
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Grouping", "CIDR", "IPs in use", "IPs available"})
+	genRow := func(kind, cidr string, available, capacity int) []string {
+		return []string{
+			kind,
+			cidr,
+			// Note: the '+capacity/2' bits here give us rounding to the nearest
+			// integer, instead of rounding down, and so ensure that the two percentages
+			// add up to 100.
+			fmt.Sprintf("%v/%v (%v%%)", capacity-available, capacity, (100*(capacity-available)+capacity/2)/capacity),
+			fmt.Sprintf("%v/%v (%v%%)", available, capacity, (100*available+capacity/2)/capacity),
+		}
+	}
+	for _, poolUse := range usage {
+		var blockRows [][]string
+		var poolAvailable, poolCapacity int
+		for _, blockUse := range poolUse.Blocks {
+			blockRows = append(blockRows, genRow("Block", blockUse.CIDR.String(), blockUse.Available, blockUse.Capacity))
+			poolAvailable += blockUse.Available
+			poolCapacity += blockUse.Capacity
+		}
+		if poolCapacity > 0 {
+			table.Append(genRow("IP Pool", poolUse.CIDR.String(), poolAvailable, poolCapacity))
+			if showBlocks {
+				table.AppendBulk(blockRows)
+			}
+		}
 	}
+	table.Render()
 
 	return nil
 }

tests/fv/ipam_test.go

@@ -0,0 +1,118 @@
+// Copyright (c) 2019 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fv_test
+
+import (
+	"context"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+
+	. "github.com/onsi/gomega"
+
+	log "github.com/sirupsen/logrus"
+
+	. "github.com/projectcalico/calicoctl/tests/fv/utils"
+	"github.com/projectcalico/libcalico-go/lib/apiconfig"
+	"github.com/projectcalico/libcalico-go/lib/apis/v3"
+	"github.com/projectcalico/libcalico-go/lib/clientv3"
+	"github.com/projectcalico/libcalico-go/lib/ipam"
+	"github.com/projectcalico/libcalico-go/lib/logutils"
+	"github.com/projectcalico/libcalico-go/lib/options"
+)
+
+func init() {
+	log.AddHook(logutils.ContextHook{})
+	log.SetFormatter(&logutils.Formatter{})
+}
+
+func TestIPAM(t *testing.T) {
+	RegisterTestingT(t)
+
+	ctx := context.Background()
+
+	// Create a Calico client.
+	config := apiconfig.NewCalicoAPIConfig()
+	config.Spec.DatastoreType = "etcdv3"
+	config.Spec.EtcdEndpoints = "http://127.0.0.1:2379"
+	client, err := clientv3.New(*config)
+	Expect(err).NotTo(HaveOccurred())
+
+	// Create an IPv4 pool.
+	pool := v3.NewIPPool()
+	pool.Name = "ipam-test-v4"
+	pool.Spec.CIDR = "10.65.0.0/16"
+	_, err = client.IPPools().Create(ctx, pool, options.SetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	// Create an IPv6 pool.
+	pool = v3.NewIPPool()
+	pool.Name = "ipam-test-v6"
+	pool.Spec.CIDR = "fd5f:abcd:64::0/48"
+	_, err = client.IPPools().Create(ctx, pool, options.SetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	// Create a Node resource for this host.
+	node := v3.NewNode()
+	node.Name, err = os.Hostname()
+	Expect(err).NotTo(HaveOccurred())
+	_, err = client.Nodes().Create(ctx, node, options.SetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	// ipam show with specific unallocated IP.
+	out := Calicoctl("ipam", "show", "--ip=10.65.0.2")
+	Expect(out).To(ContainSubstring("10.65.0.2 is not assigned"))
+
+	// ipam show, with no allocations yet.
+	out = Calicoctl("ipam", "show")
+	Expect(out).To(ContainSubstring("IPS IN USE"))
+
+	// Assign some IPs.
+	client.IPAM().AutoAssign(ctx, ipam.AutoAssignArgs{Num4: 5, Num6: 7})
+
+	// ipam show, pools only.
+	out = Calicoctl("ipam", "show")
+	Expect(out).To(ContainSubstring("IPS IN USE"))
+	Expect(out).To(ContainSubstring("10.65.0.0/16"))
+	Expect(out).To(ContainSubstring("5/64 (8%)"))
+	Expect(out).To(ContainSubstring("7/64 (11%)"))
+
+	// ipam show, including blocks.
+	out = Calicoctl("ipam", "show", "--show-blocks")
+	Expect(out).To(ContainSubstring("IPS IN USE"))
+	Expect(out).To(ContainSubstring("Block"))
+	Expect(out).To(ContainSubstring("5/64 (8%)"))
+
+	// Find out the allocation block.
+	var allocatedIP string
+	r, err := regexp.Compile("(10\\.65\\.[0-9]+\\.)([0-9]+)/26")
+	Expect(err).NotTo(HaveOccurred())
+	for _, line := range strings.Split(out, "\n") {
+		sm := r.FindStringSubmatch(line)
+		if len(sm) > 0 {
+			ordinalBase, err := strconv.Atoi(sm[2])
+			Expect(err).NotTo(HaveOccurred())
+			allocatedIP = sm[1] + strconv.Itoa(ordinalBase+2)
+			break
+		}
+	}
+	Expect(allocatedIP).NotTo(BeEmpty())
+
+	// ipam show with specific IP that is now allocated.
+	out = Calicoctl("ipam", "show", "--ip="+allocatedIP)
+	Expect(out).To(ContainSubstring(allocatedIP + " is in use"))
+}

tests/fv/utils/calicoctl.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2019 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"os/exec"
+	"strings"
+
+	. "github.com/onsi/gomega"
+	log "github.com/sirupsen/logrus"
+)
+
+var calicoctl = "/go/src/github.com/projectcalico/calicoctl/bin/calicoctl-linux-amd64"
+
+func Calicoctl(args ...string) string {
+	cmd := exec.Command(calicoctl, args...)
+	cmd.Env = []string{"ETCD_ENDPOINTS=http://127.0.0.1:2379"}
+	out, err := cmd.CombinedOutput()
+	log.Infof("Run: calicoctl %v", strings.Join(args, " "))
+	log.Infof("Output:\n%v", string(out))
+	Expect(err).NotTo(HaveOccurred())
+	return string(out)
+}