fix: [CNI] handle getting endpoints when state file is empty

Makefile

@@ -279,6 +279,7 @@ tools: acncli
 
 .PHONY: tools-images
 tools-images:
+	mkdir -p $(IMAGE_DIR)
 	docker build --no-cache -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) .
 	docker save $(AZURE_CNI_IMAGE):$(VERSION) | gzip -c > $(IMAGE_DIR)/$(CNI_IMAGE_ARCHIVE_NAME)
 

cni/client/client_integration_test.go

@@ -16,51 +16,52 @@ import (
 	"k8s.io/utils/exec"
 )
 
-func TestMain(m *testing.M) {
-	testutils.RequireRootforTestMain()
+const (
+	stateFilePath = "/var/run/azure-vnet.json"
+)
+
+func setTestFile() error {
 	var err error
 	// copy test state file to /var/run/azure-vnet.json
 	in, err := os.Open("./testresources/azure-vnet-test.json")
 	if err != nil {
-		return
+		return err
 	}
 
-	out, err := os.Create("/var/run/azure-vnet.json")
+	out, err := os.Create(stateFilePath)
 	if err != nil {
-		return
+		return err
 	}
 
-	exit := 0
-	defer func() {
-		if in != nil {
-			in.Close()
-		}
-
-		if out != nil {
-			out.Close()
-		}
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
 
-		err := os.Remove("/var/run/azure-vnet.json")
-		if err != nil {
-			log.Print(err)
-			os.Exit(1)
-		}
+	return nil
+}
 
-		os.Exit(exit)
-	}()
+func cleanupTestFile() {
+	err := os.Remove(stateFilePath)
+	if err != nil {
+		log.Print(err)
+	}
+}
 
-	_, err = io.Copy(out, in)
+func TestMain(m *testing.M) {
+	testutils.RequireRootforTestMain()
+	err := setTestFile()
 	if err != nil {
-		return
+		log.Panic(err)
 	}
 
-	exit = m.Run()
+	os.Exit(m.Run())
+	cleanupTestFile()
 }
 
 // todo: enable this test in CI, requires built azure vnet
 func TestGetStateFromAzureCNI(t *testing.T) {
-
-	c := AzureCNIClient{exec: exec.New()}
+	c := New(exec.New())
 	state, err := c.GetEndpointState()
 	require.NoError(t, err)
 
@@ -75,7 +76,7 @@ func TestGetStateFromAzureCNI(t *testing.T) {
 }
 
 func TestGetVersion(t *testing.T) {
-	c := &AzureCNIClient{exec: exec.New()}
+	c := New(exec.New())
 	version, err := c.GetVersion()
 	require.NoError(t, err)
 
@@ -84,3 +85,24 @@ func TestGetVersion(t *testing.T) {
 
 	require.Equal(t, expectedVersion, version)
 }
+
+func TestGetStateWithEmptyStateFile(t *testing.T) {
+	defer func() {
+		cleanupTestFile()
+		setTestFile()
+	}()
+
+	c := New(exec.New())
+
+	err := os.Remove(stateFilePath)
+	require.NoError(t, err)
+
+	out, err := os.Create(stateFilePath)
+	require.NoError(t, err)
+	out.Close()
+
+	state, err := c.GetEndpointState()
+	require.NoError(t, err)
+	require.Exactly(t, 0, len(state.ContainerInterfaces))
+
+}

cni/network/network.go

@@ -27,6 +27,7 @@ import (
 	"github.com/Azure/azure-container-networking/network/policy"
 	"github.com/Azure/azure-container-networking/platform"
 	nnscontracts "github.com/Azure/azure-container-networking/proto/nodenetworkservice/3.302.0.744"
+	"github.com/Azure/azure-container-networking/store"
 	"github.com/Azure/azure-container-networking/telemetry"
 	cniSkel "github.com/containernetworking/cni/pkg/skel"
 	cniTypes "github.com/containernetworking/cni/pkg/types"
@@ -161,7 +162,9 @@ func (plugin *netPlugin) GetAllEndpointState(networkid string) (*api.AzureCNISta
 	}
 
 	eps, err := plugin.nm.GetAllEndpoints(networkid)
-	if err != nil {
+	if err == store.ErrStoreEmpty {
+		log.Printf("failed to retrieve endpoint state with err %v", err)
+	} else if err != nil {
 		return nil, err
 	}
 

cni/network/network_test.go

@@ -130,3 +130,11 @@ func TestGetAllEndpointState(t *testing.T) {
 
 	require.Exactly(t, res, state)
 }
+
+func TestEndpointsWithEmptyState(t *testing.T) {
+	plugin, _ := getTestResources()
+	networkid := "azure"
+	state, err := plugin.GetAllEndpointState(networkid)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(state.ContainerInterfaces))
+}

cns/api.go

@@ -62,6 +62,45 @@ func (i IPConfigurationStatus) String() string {
 		i.ID, i.NCID, i.IPAddress, i.State, i.PodInfo)
 }
 
+// UnmarshalJSON is a custom unmarshaller for IPConfigurationStatus that
+// is capable of unmarshalling to interface type `PodInfo` contained in the
+// struct. Without this custom unmarshaller, the default unmarshaller can't
+// deserialize the json data in to that interface type.
+func (i *IPConfigurationStatus) UnmarshalJSON(b []byte) error {
+	m := map[string]json.RawMessage{}
+	if err := json.Unmarshal(b, &m); err != nil {
+		return err
+	}
+	if s, ok := m["NCID"]; ok {
+		if err := json.Unmarshal(s, &(i.NCID)); err != nil {
+			return err
+		}
+	}
+	if s, ok := m["ID"]; ok {
+		if err := json.Unmarshal(s, &(i.ID)); err != nil {
+			return err
+		}
+	}
+	if s, ok := m["IPAddress"]; ok {
+		if err := json.Unmarshal(s, &(i.IPAddress)); err != nil {
+			return err
+		}
+	}
+	if s, ok := m["State"]; ok {
+		if err := json.Unmarshal(s, &(i.State)); err != nil {
+			return err
+		}
+	}
+	if s, ok := m["PodInfo"]; ok {
+		pi, err := UnmarshalPodInfo(s)
+		if err != nil {
+			return err
+		}
+		i.PodInfo = pi
+	}
+	return nil
+}
+
 // SetEnvironmentRequest describes the Request to set the environment in CNS.
 type SetEnvironmentRequest struct {
 	Location    string

ipam/manager.go

@@ -120,6 +120,9 @@ func (am *addressManager) restore() error {
 		if err == store.ErrKeyNotFound {
 			log.Printf("[ipam] store key not found")
 			return nil
+		} else if err == store.ErrStoreEmpty {
+			log.Printf("[ipam] store empty")
+			return nil
 		} else {
 			log.Printf("[ipam] Failed to restore state, err:%v\n", err)
 			return err

network/manager.go

@@ -123,6 +123,9 @@ func (nm *networkManager) restore(isRehydrationRequired bool) error {
 			log.Printf("[net] network store key not found")
 			// Considered successful.
 			return nil
+		} else if err == store.ErrStoreEmpty {
+			log.Printf("[net] network store empty")
+			return nil
 		} else {
 			log.Printf("[net] Failed to restore state, err:%v\n", err)
 			return err
@@ -384,13 +387,19 @@ func (nm *networkManager) GetAllEndpoints(networkId string) (map[string]*Endpoin
 	nm.Lock()
 	defer nm.Unlock()
 
+	eps := make(map[string]*EndpointInfo)
+
+	// Special case when CNS invokes CNI, but there is no state, but return gracefully
+	if len(nm.ExternalInterfaces) == 0 {
+		log.Printf("Network manager has no external interfaces, is the state file populated?")
+		return eps, store.ErrStoreEmpty
+	}
+
 	nw, err := nm.getNetwork(networkId)
 	if err != nil {
 		return nil, err
 	}
 
-	eps := make(map[string]*EndpointInfo)
-
 	for epid, ep := range nw.Endpoints {
 		eps[epid] = ep.getInfo()
 	}

store/json.go

@@ -71,8 +71,18 @@ func (kvs *jsonFileStore) Read(key string, value interface{}) error {
 		}
 		defer file.Close()
 
+		b, err := ioutil.ReadAll(file)
+		if err != nil {
+			return err
+		}
+
+		if len(b) == 0 {
+			log.Printf("Unable to read file %s, was empty", kvs.fileName)
+			return ErrStoreEmpty
+		}
+
 		// Decode to raw JSON messages.
-		if err := json.NewDecoder(file).Decode(&kvs.data); err != nil {
+		if err := json.Unmarshal(b, &kvs.data); err != nil {
 			return err
 		}
 

store/store.go

@@ -26,6 +26,7 @@ var (
 	ErrKeyNotFound                    = fmt.Errorf("key not found")
 	ErrStoreLocked                    = fmt.Errorf("store is already locked")
 	ErrStoreNotLocked                 = fmt.Errorf("store is not locked")
+	ErrStoreEmpty                     = fmt.Errorf("store is empty")
 	ErrTimeoutLockingStore            = fmt.Errorf("timed out locking store")
 	ErrNonBlockingLockIsAlreadyLocked = fmt.Errorf("attempted to perform non-blocking lock on an already locked store")
 )