diff --git a/Makefile b/Makefile index 6a96023f24..ac6559988f 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,8 @@ CNSFILES = \ $(wildcard cns/networkcontainers/*.go) \ $(wildcard cns/requestcontroller/*.go) \ $(wildcard cns/requestcontroller/kubecontroller/*.go) \ + $(wildcard cns/multitenantcontroller/*.go) \ + $(wildcard cns/multitenantcontroller/multitenantoperator/*.go) \ $(wildcard cns/fakes/*.go) \ $(COREFILES) \ $(CNMFILES) @@ -181,13 +183,13 @@ azure-npm: $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT) npm-archive endif ifeq ($(GOOS),linux) -all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-cnms azure-npm +all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-cnms azure-npm else all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns endif ifeq ($(GOOS),linux) -all-images: azure-npm-image azure-cns-image +all-images: azure-npm-image azure-cns-image else all-images: @echo "Nothing to build. Skip." @@ -223,7 +225,7 @@ $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT): $(CNIFILES) # Build the Azure CLI network plugin. $(ACNCLI_BUILD_DIR)/acncli$(EXE_EXT): $(CNIFILES) CGO_ENABLED=0 go build -v -o $(ACNCLI_BUILD_DIR)/acn$(EXE_EXT) -ldflags "-X main.version=$(VERSION)" -gcflags="-dwarflocationlists=true" $(ACNCLI_DIR)/*.go - + # Build the Azure CNS Service. $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT): $(CNSFILES) go build -v -o $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -X $(cnsaipath)=$(CNS_AI_ID)" -gcflags="-dwarflocationlists=true" $(CNS_DIR)/*.go @@ -259,7 +261,7 @@ all-containerized: # Make both linux and windows binaries .PHONY: all-binaries-platforms -all-binaries-platforms: +all-binaries-platforms: export GOOS=linux; make all-binaries export GOOS=windows; make all-binaries @@ -268,7 +270,7 @@ all-binaries-platforms: tools: acncli .PHONY: tools-images -tools-images: +tools-images: docker build --no-cache -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) . # Build the Azure CNM plugin image, installable with "docker plugin install". @@ -409,7 +411,7 @@ ifeq ($(GOOS),linux) cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_SWIFT_BUILD_DIR) chmod 0755 $(CNI_SWIFT_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_SWIFT_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) cd $(CNI_SWIFT_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_SWIFT_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config -endif +endif # Create a CNM archive for the target platform. .PHONY: cnm-archive @@ -453,7 +455,7 @@ endif .PHONY: release release: ./scripts/semver-release.sh - + PRETTYGOTEST := $(shell command -v gotest 2> /dev/null) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 7b28b5502f..2089dbc72e 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -69,9 +69,10 @@ const ( // ChannelMode :- CNS channel modes const ( - Direct = "Direct" - Managed = "Managed" - CRD = "CRD" + Direct = "Direct" + Managed = "Managed" + CRD = "CRD" + MultiTenantCRD = "MultiTenantCRD" ) // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. diff --git a/cns/cnsclient/apiclient.go b/cns/cnsclient/apiclient.go index 6982926790..d39742907c 100644 --- a/cns/cnsclient/apiclient.go +++ b/cns/cnsclient/apiclient.go @@ -8,5 +8,8 @@ import ( // APIClient interface to update cns state type APIClient interface { ReconcileNCState(nc *cns.CreateNetworkContainerRequest, pods map[string]cns.KubernetesPodInfo, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error - CreateOrUpdateNC(nc cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error + CreateOrUpdateNC(nc cns.CreateNetworkContainerRequest) error + UpdateIPAMPoolMonitor(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error + GetNC(nc cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, error) + DeleteNC(nc cns.DeleteNetworkContainerRequest) error } diff --git a/cns/cnsclient/cnsclient_test.go b/cns/cnsclient/cnsclient_test.go index 0185baeb85..15297df012 100644 --- a/cns/cnsclient/cnsclient_test.go +++ b/cns/cnsclient/cnsclient_test.go @@ -73,10 +73,15 @@ func addTestStateToRestServer(t *testing.T, secondaryIps []string) { Version: "-1", } - returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize)) + returnCode := svc.CreateOrUpdateNetworkContainerInternal(req) if returnCode != 0 { t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode) } + + returnCode = svc.UpdateIPAMPoolMonitorInternal(fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize)) + if returnCode != 0 { + t.Fatalf("Failed to UpdateIPAMPoolMonitorInternal, err: %d", returnCode) + } } func getIPNetFromResponse(resp *cns.IPConfigResponse) (net.IPNet, error) { diff --git a/cns/cnsclient/httpapi/client.go b/cns/cnsclient/httpapi/client.go index d3add84ce8..7307e17638 100644 --- a/cns/cnsclient/httpapi/client.go +++ b/cns/cnsclient/httpapi/client.go @@ -14,8 +14,8 @@ type Client struct { } // CreateOrUpdateNC updates cns state -func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error { - returnCode := client.RestService.CreateOrUpdateNetworkContainerInternal(ncRequest, scalar, spec) +func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest) error { + returnCode := client.RestService.CreateOrUpdateNetworkContainerInternal(ncRequest) if returnCode != 0 { return fmt.Errorf("Failed to Create NC request: %+v, errorCode: %d", ncRequest, returnCode) @@ -24,6 +24,17 @@ func (client *Client) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerReque return nil } +// UpdateIPAMPoolMonitor updates IPAM pool monitor. +func (client *Client) UpdateIPAMPoolMonitor(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error { + returnCode := client.RestService.UpdateIPAMPoolMonitorInternal(scalar, spec) + + if returnCode != 0 { + return fmt.Errorf("Failed to update IPAM pool monitor scalar: %+v, spec: %+v, errorCode: %d", scalar, spec, returnCode) + } + + return nil +} + // ReconcileNCState initializes cns state func (client *Client) ReconcileNCState(ncRequest *cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.KubernetesPodInfo, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error { returnCode := client.RestService.ReconcileNCState(ncRequest, podInfoByIP, scalar, spec) @@ -34,3 +45,24 @@ func (client *Client) ReconcileNCState(ncRequest *cns.CreateNetworkContainerRequ return nil } + +func (client *Client) GetNC(req cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, error) { + response, returnCode := client.RestService.GetNetworkContainerInternal(req) + if returnCode != 0 { + if returnCode == restserver.UnknownContainerID { + return response, fmt.Errorf("NotFound") + } + return response, fmt.Errorf("Failed to get NC, request: %+v, errorCode: %d", req, returnCode) + } + + return response, nil +} + +func (client *Client) DeleteNC(req cns.DeleteNetworkContainerRequest) error { + returnCode := client.RestService.DeleteNetworkContainerInternal(req) + if returnCode != 0 { + return fmt.Errorf("Failed to delete NC, request: %+v, errorCode: %d", req, returnCode) + } + + return nil +} diff --git a/cns/multitenantcontroller/mockclients/README.md b/cns/multitenantcontroller/mockclients/README.md new file mode 100644 index 0000000000..1e8378e4e0 --- /dev/null +++ b/cns/multitenantcontroller/mockclients/README.md @@ -0,0 +1,8 @@ +# Mock Clients + +Run the following command to generate mock clients: + +```sh +mockgen -source=$GOPATH/src/github.com/Azure/azure-container-networking/cns/cnsclient/apiclient.go -package=mockclients APIClient >cnsclient.go +mockgen -source=$GOPATH/src/sigs.k8s.io/controller-runtime/pkg/client/interfaces.go -package=mockclients Client >kubeclient.go +``` diff --git a/cns/multitenantcontroller/mockclients/cnsclient.go b/cns/multitenantcontroller/mockclients/cnsclient.go new file mode 100644 index 0000000000..6ab1a390c8 --- /dev/null +++ b/cns/multitenantcontroller/mockclients/cnsclient.go @@ -0,0 +1,106 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: /go/src/github.com/Azure/azure-container-networking/cns/cnsclient/apiclient.go + +// Package mockclients is a generated GoMock package. +package mockclients + +import ( + cns "github.com/Azure/azure-container-networking/cns" + v1alpha "github.com/Azure/azure-container-networking/nodenetworkconfig/api/v1alpha" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockAPIClient is a mock of APIClient interface +type MockAPIClient struct { + ctrl *gomock.Controller + recorder *MockAPIClientMockRecorder +} + +// MockAPIClientMockRecorder is the mock recorder for MockAPIClient +type MockAPIClientMockRecorder struct { + mock *MockAPIClient +} + +// NewMockAPIClient creates a new mock instance +func NewMockAPIClient(ctrl *gomock.Controller) *MockAPIClient { + mock := &MockAPIClient{ctrl: ctrl} + mock.recorder = &MockAPIClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockAPIClient) EXPECT() *MockAPIClientMockRecorder { + return m.recorder +} + +// ReconcileNCState mocks base method +func (m *MockAPIClient) ReconcileNCState(nc *cns.CreateNetworkContainerRequest, pods map[string]cns.KubernetesPodInfo, scalar v1alpha.Scaler, spec v1alpha.NodeNetworkConfigSpec) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReconcileNCState", nc, pods, scalar, spec) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReconcileNCState indicates an expected call of ReconcileNCState +func (mr *MockAPIClientMockRecorder) ReconcileNCState(nc, pods, scalar, spec interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileNCState", reflect.TypeOf((*MockAPIClient)(nil).ReconcileNCState), nc, pods, scalar, spec) +} + +// CreateOrUpdateNC mocks base method +func (m *MockAPIClient) CreateOrUpdateNC(nc cns.CreateNetworkContainerRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOrUpdateNC", nc) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateOrUpdateNC indicates an expected call of CreateOrUpdateNC +func (mr *MockAPIClientMockRecorder) CreateOrUpdateNC(nc interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateNC", reflect.TypeOf((*MockAPIClient)(nil).CreateOrUpdateNC), nc) +} + +// UpdateIPAMPoolMonitor mocks base method +func (m *MockAPIClient) UpdateIPAMPoolMonitor(scalar v1alpha.Scaler, spec v1alpha.NodeNetworkConfigSpec) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateIPAMPoolMonitor", scalar, spec) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateIPAMPoolMonitor indicates an expected call of UpdateIPAMPoolMonitor +func (mr *MockAPIClientMockRecorder) UpdateIPAMPoolMonitor(scalar, spec interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateIPAMPoolMonitor", reflect.TypeOf((*MockAPIClient)(nil).UpdateIPAMPoolMonitor), scalar, spec) +} + +// GetNC mocks base method +func (m *MockAPIClient) GetNC(nc cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNC", nc) + ret0, _ := ret[0].(cns.GetNetworkContainerResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNC indicates an expected call of GetNC +func (mr *MockAPIClientMockRecorder) GetNC(nc interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNC", reflect.TypeOf((*MockAPIClient)(nil).GetNC), nc) +} + +// DeleteNC mocks base method +func (m *MockAPIClient) DeleteNC(nc cns.DeleteNetworkContainerRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteNC", nc) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteNC indicates an expected call of DeleteNC +func (mr *MockAPIClientMockRecorder) DeleteNC(nc interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteNC", reflect.TypeOf((*MockAPIClient)(nil).DeleteNC), nc) +} diff --git a/cns/multitenantcontroller/mockclients/kubeclient.go b/cns/multitenantcontroller/mockclients/kubeclient.go new file mode 100644 index 0000000000..d0af4d9c07 --- /dev/null +++ b/cns/multitenantcontroller/mockclients/kubeclient.go @@ -0,0 +1,540 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: /go/src/sigs.k8s.io/controller-runtime/pkg/client/interfaces.go + +// Package mockclients is a generated GoMock package. +package mockclients + +import ( + context "context" + gomock "github.com/golang/mock/gomock" + runtime "k8s.io/apimachinery/pkg/runtime" + types "k8s.io/apimachinery/pkg/types" + reflect "reflect" + client "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MockPatch is a mock of Patch interface +type MockPatch struct { + ctrl *gomock.Controller + recorder *MockPatchMockRecorder +} + +// MockPatchMockRecorder is the mock recorder for MockPatch +type MockPatchMockRecorder struct { + mock *MockPatch +} + +// NewMockPatch creates a new mock instance +func NewMockPatch(ctrl *gomock.Controller) *MockPatch { + mock := &MockPatch{ctrl: ctrl} + mock.recorder = &MockPatchMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockPatch) EXPECT() *MockPatchMockRecorder { + return m.recorder +} + +// Type mocks base method +func (m *MockPatch) Type() types.PatchType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Type") + ret0, _ := ret[0].(types.PatchType) + return ret0 +} + +// Type indicates an expected call of Type +func (mr *MockPatchMockRecorder) Type() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Type", reflect.TypeOf((*MockPatch)(nil).Type)) +} + +// Data mocks base method +func (m *MockPatch) Data(obj runtime.Object) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Data", obj) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Data indicates an expected call of Data +func (mr *MockPatchMockRecorder) Data(obj interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Data", reflect.TypeOf((*MockPatch)(nil).Data), obj) +} + +// MockReader is a mock of Reader interface +type MockReader struct { + ctrl *gomock.Controller + recorder *MockReaderMockRecorder +} + +// MockReaderMockRecorder is the mock recorder for MockReader +type MockReaderMockRecorder struct { + mock *MockReader +} + +// NewMockReader creates a new mock instance +func NewMockReader(ctrl *gomock.Controller) *MockReader { + mock := &MockReader{ctrl: ctrl} + mock.recorder = &MockReaderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockReader) EXPECT() *MockReaderMockRecorder { + return m.recorder +} + +// Get mocks base method +func (m *MockReader) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, key, obj) + ret0, _ := ret[0].(error) + return ret0 +} + +// Get indicates an expected call of Get +func (mr *MockReaderMockRecorder) Get(ctx, key, obj interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockReader)(nil).Get), ctx, key, obj) +} + +// List mocks base method +func (m *MockReader) List(ctx context.Context, list runtime.Object, opts ...client.ListOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, list} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "List", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// List indicates an expected call of List +func (mr *MockReaderMockRecorder) List(ctx, list interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, list}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockReader)(nil).List), varargs...) +} + +// MockWriter is a mock of Writer interface +type MockWriter struct { + ctrl *gomock.Controller + recorder *MockWriterMockRecorder +} + +// MockWriterMockRecorder is the mock recorder for MockWriter +type MockWriterMockRecorder struct { + mock *MockWriter +} + +// NewMockWriter creates a new mock instance +func NewMockWriter(ctrl *gomock.Controller) *MockWriter { + mock := &MockWriter{ctrl: ctrl} + mock.recorder = &MockWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockWriter) EXPECT() *MockWriterMockRecorder { + return m.recorder +} + +// Create mocks base method +func (m *MockWriter) Create(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Create", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create +func (mr *MockWriterMockRecorder) Create(ctx, obj interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockWriter)(nil).Create), varargs...) +} + +// Delete mocks base method +func (m *MockWriter) Delete(ctx context.Context, obj runtime.Object, opts ...client.DeleteOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Delete", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete +func (mr *MockWriterMockRecorder) Delete(ctx, obj interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockWriter)(nil).Delete), varargs...) +} + +// Update mocks base method +func (m *MockWriter) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Update", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update +func (mr *MockWriterMockRecorder) Update(ctx, obj interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockWriter)(nil).Update), varargs...) +} + +// Patch mocks base method +func (m *MockWriter) Patch(ctx context.Context, obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj, patch} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Patch", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Patch indicates an expected call of Patch +func (mr *MockWriterMockRecorder) Patch(ctx, obj, patch interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj, patch}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockWriter)(nil).Patch), varargs...) +} + +// DeleteAllOf mocks base method +func (m *MockWriter) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...client.DeleteAllOfOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteAllOf", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAllOf indicates an expected call of DeleteAllOf +func (mr *MockWriterMockRecorder) DeleteAllOf(ctx, obj interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllOf", reflect.TypeOf((*MockWriter)(nil).DeleteAllOf), varargs...) +} + +// MockStatusClient is a mock of StatusClient interface +type MockStatusClient struct { + ctrl *gomock.Controller + recorder *MockStatusClientMockRecorder +} + +// MockStatusClientMockRecorder is the mock recorder for MockStatusClient +type MockStatusClientMockRecorder struct { + mock *MockStatusClient +} + +// NewMockStatusClient creates a new mock instance +func NewMockStatusClient(ctrl *gomock.Controller) *MockStatusClient { + mock := &MockStatusClient{ctrl: ctrl} + mock.recorder = &MockStatusClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockStatusClient) EXPECT() *MockStatusClientMockRecorder { + return m.recorder +} + +// Status mocks base method +func (m *MockStatusClient) Status() client.StatusWriter { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Status") + ret0, _ := ret[0].(client.StatusWriter) + return ret0 +} + +// Status indicates an expected call of Status +func (mr *MockStatusClientMockRecorder) Status() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockStatusClient)(nil).Status)) +} + +// MockStatusWriter is a mock of StatusWriter interface +type MockStatusWriter struct { + ctrl *gomock.Controller + recorder *MockStatusWriterMockRecorder +} + +// MockStatusWriterMockRecorder is the mock recorder for MockStatusWriter +type MockStatusWriterMockRecorder struct { + mock *MockStatusWriter +} + +// NewMockStatusWriter creates a new mock instance +func NewMockStatusWriter(ctrl *gomock.Controller) *MockStatusWriter { + mock := &MockStatusWriter{ctrl: ctrl} + mock.recorder = &MockStatusWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockStatusWriter) EXPECT() *MockStatusWriterMockRecorder { + return m.recorder +} + +// Update mocks base method +func (m *MockStatusWriter) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Update", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update +func (mr *MockStatusWriterMockRecorder) Update(ctx, obj interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockStatusWriter)(nil).Update), varargs...) +} + +// Patch mocks base method +func (m *MockStatusWriter) Patch(ctx context.Context, obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj, patch} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Patch", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Patch indicates an expected call of Patch +func (mr *MockStatusWriterMockRecorder) Patch(ctx, obj, patch interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj, patch}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockStatusWriter)(nil).Patch), varargs...) +} + +// MockClient is a mock of Client interface +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder +} + +// MockClientMockRecorder is the mock recorder for MockClient +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// Get mocks base method +func (m *MockClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, key, obj) + ret0, _ := ret[0].(error) + return ret0 +} + +// Get indicates an expected call of Get +func (mr *MockClientMockRecorder) Get(ctx, key, obj interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockClient)(nil).Get), ctx, key, obj) +} + +// List mocks base method +func (m *MockClient) List(ctx context.Context, list runtime.Object, opts ...client.ListOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, list} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "List", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// List indicates an expected call of List +func (mr *MockClientMockRecorder) List(ctx, list interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, list}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockClient)(nil).List), varargs...) +} + +// Create mocks base method +func (m *MockClient) Create(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Create", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create +func (mr *MockClientMockRecorder) Create(ctx, obj interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockClient)(nil).Create), varargs...) +} + +// Delete mocks base method +func (m *MockClient) Delete(ctx context.Context, obj runtime.Object, opts ...client.DeleteOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Delete", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete +func (mr *MockClientMockRecorder) Delete(ctx, obj interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockClient)(nil).Delete), varargs...) +} + +// Update mocks base method +func (m *MockClient) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Update", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update +func (mr *MockClientMockRecorder) Update(ctx, obj interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockClient)(nil).Update), varargs...) +} + +// Patch mocks base method +func (m *MockClient) Patch(ctx context.Context, obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj, patch} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Patch", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// Patch indicates an expected call of Patch +func (mr *MockClientMockRecorder) Patch(ctx, obj, patch interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj, patch}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockClient)(nil).Patch), varargs...) +} + +// DeleteAllOf mocks base method +func (m *MockClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...client.DeleteAllOfOption) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, obj} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteAllOf", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAllOf indicates an expected call of DeleteAllOf +func (mr *MockClientMockRecorder) DeleteAllOf(ctx, obj interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, obj}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllOf", reflect.TypeOf((*MockClient)(nil).DeleteAllOf), varargs...) +} + +// Status mocks base method +func (m *MockClient) Status() client.StatusWriter { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Status") + ret0, _ := ret[0].(client.StatusWriter) + return ret0 +} + +// Status indicates an expected call of Status +func (mr *MockClientMockRecorder) Status() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockClient)(nil).Status)) +} + +// MockFieldIndexer is a mock of FieldIndexer interface +type MockFieldIndexer struct { + ctrl *gomock.Controller + recorder *MockFieldIndexerMockRecorder +} + +// MockFieldIndexerMockRecorder is the mock recorder for MockFieldIndexer +type MockFieldIndexerMockRecorder struct { + mock *MockFieldIndexer +} + +// NewMockFieldIndexer creates a new mock instance +func NewMockFieldIndexer(ctrl *gomock.Controller) *MockFieldIndexer { + mock := &MockFieldIndexer{ctrl: ctrl} + mock.recorder = &MockFieldIndexerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockFieldIndexer) EXPECT() *MockFieldIndexerMockRecorder { + return m.recorder +} + +// IndexField mocks base method +func (m *MockFieldIndexer) IndexField(ctx context.Context, obj runtime.Object, field string, extractValue client.IndexerFunc) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IndexField", ctx, obj, field, extractValue) + ret0, _ := ret[0].(error) + return ret0 +} + +// IndexField indicates an expected call of IndexField +func (mr *MockFieldIndexerMockRecorder) IndexField(ctx, obj, field, extractValue interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IndexField", reflect.TypeOf((*MockFieldIndexer)(nil).IndexField), ctx, obj, field, extractValue) +} diff --git a/cns/multitenantcontroller/multitenantcontrollerinterface.go b/cns/multitenantcontroller/multitenantcontrollerinterface.go new file mode 100644 index 0000000000..716af47f0e --- /dev/null +++ b/cns/multitenantcontroller/multitenantcontrollerinterface.go @@ -0,0 +1,7 @@ +package multitenantcontroller + +// MultiTenantController defines the interface for multi-tenant network container operations. +type MultiTenantController interface { + StartMultiTenantController(exitChan <-chan struct{}) error + IsStarted() bool +} diff --git a/cns/multitenantcontroller/multitenantoperator/multitenantcrdcontroller.go b/cns/multitenantcontroller/multitenantoperator/multitenantcrdcontroller.go new file mode 100644 index 0000000000..681ff54242 --- /dev/null +++ b/cns/multitenantcontroller/multitenantoperator/multitenantcrdcontroller.go @@ -0,0 +1,159 @@ +package multitenantoperator + +import ( + "errors" + "os" + "sync" + + "github.com/Azure/azure-container-networking/cns/cnsclient" + "github.com/Azure/azure-container-networking/cns/cnsclient/httpapi" + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/restserver" + ncapi "github.com/Azure/azure-container-networking/networkcontainer/api/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const ( + nodeNameEnvVar = "NODENAME" + prometheusAddress = "0" //0 means disabled +) + +// multiTenantController operates multi-tenant CRD. +type multiTenantController struct { + mgr manager.Manager //Manager starts the reconcile loop which watches for crd status changes + KubeClient client.Client //KubeClient is a cached client which interacts with API server + CNSClient cnsclient.APIClient + nodeName string //name of node running this program + Reconciler *multiTenantCrdReconciler + Started bool + lock sync.Mutex +} + +// MultiTenantController create a new multi-tenant CRD operator. +func NewMultiTenantController(restService *restserver.HTTPRestService, kubeconfig *rest.Config) (*multiTenantController, error) { + // Check that logger package has been initialized. + if logger.Log == nil { + return nil, errors.New("Must initialize logger before calling") + } + + // Check that NODENAME environment variable is set. NODENAME is name of node running this program. + nodeName := os.Getenv(nodeNameEnvVar) + if nodeName == "" { + return nil, errors.New("Must declare " + nodeNameEnvVar + " environment variable.") + } + + // Add client-go scheme to runtime scheme so manager can recognize it. + var scheme = runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return nil, errors.New("Error adding client-go scheme to runtime scheme") + } + + // Add CRD scheme to runtime sheme so manager can recognize it. + if err := ncapi.AddToScheme(scheme); err != nil { + return nil, errors.New("Error adding NetworkContainer scheme to runtime scheme") + } + + // Create manager for multiTenantController. + mgr, err := ctrl.NewManager(kubeconfig, ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: prometheusAddress, + }) + if err != nil { + logger.Errorf("Error creating new multiTenantController: %v", err) + return nil, err + } + + // Create httpClient + httpClient := &httpapi.Client{ + RestService: restService, + } + + // Create multiTenantCrdReconciler + reconciler := &multiTenantCrdReconciler{ + KubeClient: mgr.GetClient(), + NodeName: nodeName, + CNSClient: httpClient, + } + if err := reconciler.SetupWithManager(mgr); err != nil { + logger.Errorf("Error setting up new multiTenantCrdReconciler: %v", err) + return nil, err + } + + // Create the multiTenantController + return &multiTenantController{ + mgr: mgr, + KubeClient: mgr.GetClient(), + CNSClient: httpClient, + nodeName: nodeName, + Reconciler: reconciler, + }, nil +} + +// StartMultiTenantController starts the Reconciler loop which watches for CRD status updates. +// Blocks until SIGINT or SIGTERM is received +// Notifies exitChan when kill signal received +func (mtc *multiTenantController) StartMultiTenantController(exitChan <-chan struct{}) error { + logger.Printf("Starting MultiTenantController") + + // Setting the started state + mtc.lock.Lock() + mtc.Started = true + mtc.lock.Unlock() + + logger.Printf("Starting reconcile loop") + if err := mtc.mgr.Start(exitChan); err != nil { + if mtc.isNotDefined(err) { + logger.Errorf("multi-tenant CRD is not defined on cluster, starting reconcile loop failed: %v", err) + os.Exit(1) + } + + return err + } + + return nil +} + +// return if RequestController is started +func (mtc *multiTenantController) IsStarted() bool { + defer mtc.lock.Unlock() + mtc.lock.Lock() + return mtc.Started +} + +// isNotDefined tells whether the given error is a CRD not defined error +func (mtc *multiTenantController) isNotDefined(err error) bool { + var ( + statusError *apierrors.StatusError + ok bool + notDefined bool + cause metav1.StatusCause + ) + + if err == nil { + return false + } + + if statusError, ok = err.(*apierrors.StatusError); !ok { + return false + } + + if len(statusError.ErrStatus.Details.Causes) > 0 { + for _, cause = range statusError.ErrStatus.Details.Causes { + if cause.Type == metav1.CauseTypeUnexpectedServerResponse { + if apierrors.IsNotFound(err) { + notDefined = true + break + } + } + } + } + + return notDefined +} diff --git a/cns/multitenantcontroller/multitenantoperator/multitenantcrdcontroller_test.go b/cns/multitenantcontroller/multitenantoperator/multitenantcrdcontroller_test.go new file mode 100644 index 0000000000..f6b2580c01 --- /dev/null +++ b/cns/multitenantcontroller/multitenantoperator/multitenantcrdcontroller_test.go @@ -0,0 +1,39 @@ +package multitenantoperator + +import ( + "os" + + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/restserver" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/client-go/rest" +) + +var _ = Describe("multiTenantController", func() { + BeforeEach(func() { + logger.InitLogger("multiTenantController", 0, 0, "") + }) + + Context("lifecycle", func() { + restService := &restserver.HTTPRestService{} + kubeconfig := &rest.Config{} + + It("Should exist with an error when nodeName is not set", func() { + ctl, err := NewMultiTenantController(restService, kubeconfig) + Expect(ctl).To(BeNil()) + Expect(err).NotTo(BeNil()) + Expect(err.Error()).To(Equal("Must declare NODENAME environment variable.")) + }) + + It("Should report an error when apiserver is not available", func() { + val := os.Getenv(nodeNameEnvVar) + os.Setenv(nodeNameEnvVar, "nodeName") + ctl, err := NewMultiTenantController(nil, nil) + os.Setenv(nodeNameEnvVar, val) + Expect(ctl).To(BeNil()) + Expect(err).NotTo(BeNil()) + Expect(err.Error()).To(Equal("must specify Config")) + }) + }) +}) diff --git a/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler.go b/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler.go new file mode 100644 index 0000000000..3e9796f03a --- /dev/null +++ b/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler.go @@ -0,0 +1,156 @@ +package multitenantoperator + +import ( + "context" + "net" + "strings" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/cnsclient" + "github.com/Azure/azure-container-networking/cns/logger" + ncapi "github.com/Azure/azure-container-networking/networkcontainer/api/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + NCStateInitialized = "Initialized" // NC has been initialized by DNC. + NCStateSucceeded = "Succeeded" // NC has been persisted by CNS + NCStateTerminated = "Terminated" // NC has been cleaned up from CNS +) + +// multiTenantCrdReconciler reconciles multi-tenant network containers. +type multiTenantCrdReconciler struct { + KubeClient client.Client + NodeName string + CNSClient cnsclient.APIClient +} + +// Reconcile is called on multi-tenant CRD status changes. +func (r *multiTenantCrdReconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { + ctx := context.Background() + var nc ncapi.NetworkContainer + + if err := r.KubeClient.Get(ctx, request.NamespacedName, &nc); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + + logger.Errorf("Failed to fetch network container: %v", err) + return ctrl.Result{}, err + } + + if !nc.ObjectMeta.DeletionTimestamp.IsZero() { + // Do nothing if the NC has already in Terminated state. + if nc.Status.State == NCStateTerminated { + return ctrl.Result{}, nil + } + + // Remove the deleted network container from CNS. + err := r.CNSClient.DeleteNC(cns.DeleteNetworkContainerRequest{ + NetworkContainerid: nc.Spec.UUID, + }) + if err != nil { + logger.Errorf("Failed to delete NC %s from CNS: %v", nc.Spec.UUID, err) + return ctrl.Result{}, err + } + + // Update NC state to Terminated. + nc.Status.State = NCStateTerminated + if err := r.KubeClient.Status().Update(ctx, &nc); err != nil { + logger.Errorf("Failed to update network container state for %s: %v", nc.Spec.UUID, err) + return ctrl.Result{}, err + } + + logger.Printf("NC has been terminated for %s", nc.Spec.UUID) + return ctrl.Result{}, nil + } + + // Do nothing if the network container hasn't been initialized yet from control plane. + if nc.Status.State != NCStateInitialized { + return ctrl.Result{}, nil + } + + // Check CNS NC states. + _, err := r.CNSClient.GetNC(cns.GetNetworkContainerRequest{ + NetworkContainerid: nc.Spec.UUID, + }) + if err == nil { + logger.Printf("NC %s has already been created in CNS", nc.Spec.UUID) + return ctrl.Result{}, nil + } else if err.Error() != "NotFound" { + logger.Errorf("Failed to fetch NC from CNS: %v", err) + return ctrl.Result{}, err + } + + // Persist NC states into CNS. + _, ipNet, err := net.ParseCIDR(nc.Status.IPSubnet) + if err != nil { + logger.Errorf("Failed to parse IPSubnet %s for NC %s: %v", nc.Status.IPSubnet, nc.Spec.UUID, err) + return ctrl.Result{}, err + } + prefixLength, _ := ipNet.Mask.Size() + networkContainerRequest := cns.CreateNetworkContainerRequest{ + NetworkContainerid: nc.Spec.UUID, + IPConfiguration: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: nc.Status.IP, + PrefixLength: uint8(prefixLength), + }, + GatewayIPAddress: nc.Status.Gateway, + }, + } + if err = r.CNSClient.CreateOrUpdateNC(networkContainerRequest); err != nil { + logger.Errorf("Failed to persist state for NC %s to CNS: %v", nc.Spec.UUID, err) + return ctrl.Result{}, err + } + + // Update NC state to Succeeded. + nc.Status.State = NCStateSucceeded + if err := r.KubeClient.Status().Update(ctx, &nc); err != nil { + logger.Errorf("Failed to update network container state for %s: %v", nc.Spec.UUID, err) + return ctrl.Result{}, err + } + + logger.Printf("Reconciled NC %s", nc.Spec.UUID) + return reconcile.Result{}, nil +} + +// SetupWithManager Sets up the reconciler with a new manager, filtering using NodeNetworkConfigFilter +func (r *multiTenantCrdReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&ncapi.NetworkContainer{}). + WithEventFilter(r.predicate()). + Complete(r) +} + +func (r *multiTenantCrdReconciler) predicate() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return r.equalNode(e.Object) + }, + GenericFunc: func(e event.GenericEvent) bool { + return r.equalNode(e.Object) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return r.equalNode(e.ObjectNew) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return r.equalNode(e.Object) + }, + } +} + +func (r *multiTenantCrdReconciler) equalNode(o runtime.Object) bool { + nc, ok := o.(*ncapi.NetworkContainer) + if ok { + return strings.EqualFold(nc.Spec.Node, r.NodeName) + } + + return false +} diff --git a/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler_test.go b/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler_test.go new file mode 100644 index 0000000000..6622edf4af --- /dev/null +++ b/cns/multitenantcontroller/multitenantoperator/multitenantcrdreconciler_test.go @@ -0,0 +1,161 @@ +package multitenantoperator + +import ( + "fmt" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/multitenantcontroller/mockclients" + ncapi "github.com/Azure/azure-container-networking/networkcontainer/api/v1alpha1" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var _ = Describe("multiTenantCrdReconciler", func() { + var kubeClient *mockclients.MockClient + var cnsClient *mockclients.MockAPIClient + var mockCtl *gomock.Controller + var reconciler *multiTenantCrdReconciler + var mockNodeName = "mockNodeName" + var namespacedName = types.NamespacedName{ + Namespace: "test", + Name: "test", + } + + BeforeEach(func() { + logger.InitLogger("multiTenantCrdReconciler", 0, 0, "") + mockCtl = gomock.NewController(GinkgoT()) + kubeClient = mockclients.NewMockClient(mockCtl) + cnsClient = mockclients.NewMockAPIClient(mockCtl) + reconciler = &multiTenantCrdReconciler{ + KubeClient: kubeClient, + NodeName: mockNodeName, + CNSClient: cnsClient, + } + }) + + Context("lifecycle", func() { + + It("Should succeed when the NC has already been deleted", func() { + expectedError := &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, + }, + } + kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).Return(expectedError) + _, err := reconciler.Reconcile(reconcile.Request{ + NamespacedName: namespacedName, + }) + Expect(err).To(BeNil()) + }) + + It("Should fail when the kube client reports failure", func() { + expectedError := &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonInternalError, + }, + } + kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).Return(expectedError) + _, err := reconciler.Reconcile(reconcile.Request{ + NamespacedName: namespacedName, + }) + Expect(err).NotTo(BeNil()) + Expect(err).To(Equal(expectedError)) + }) + + It("Should succeed when the NC is in Terminated state", func() { + var nc ncapi.NetworkContainer = ncapi.NetworkContainer{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &metav1.Time{}, + }, + Status: ncapi.NetworkContainerStatus{ + State: "Terminated", + }, + } + kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc) + _, err := reconciler.Reconcile(reconcile.Request{ + NamespacedName: namespacedName, + }) + Expect(err).To(BeNil()) + }) + + It("Should succeed when the NC is not in Initialized state", func() { + var nc ncapi.NetworkContainer = ncapi.NetworkContainer{ + Status: ncapi.NetworkContainerStatus{ + State: "Pending", + }, + } + kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc) + _, err := reconciler.Reconcile(reconcile.Request{ + NamespacedName: namespacedName, + }) + Expect(err).To(BeNil()) + }) + + It("Should succeed when the NC is in Initialized state and it has already been persisted in CNS", func() { + var uuid = "uuid" + var nc ncapi.NetworkContainer = ncapi.NetworkContainer{ + Spec: ncapi.NetworkContainerSpec{ + UUID: uuid, + }, + Status: ncapi.NetworkContainerStatus{ + State: "Initialized", + }, + } + kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc) + cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{NetworkContainerid: uuid}).Return(cns.GetNetworkContainerResponse{}, nil) + _, err := reconciler.Reconcile(reconcile.Request{ + NamespacedName: namespacedName, + }) + Expect(err).To(BeNil()) + }) + + It("Should fail when the NC subnet isn't in correct format", func() { + var uuid = "uuid" + var nc ncapi.NetworkContainer = ncapi.NetworkContainer{ + Spec: ncapi.NetworkContainerSpec{ + UUID: uuid, + }, + Status: ncapi.NetworkContainerStatus{ + State: "Initialized", + IPSubnet: "1.2.3.4.5", + }, + } + kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc) + cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{NetworkContainerid: uuid}).Return(cns.GetNetworkContainerResponse{}, fmt.Errorf("NotFound")) + _, err := reconciler.Reconcile(reconcile.Request{ + NamespacedName: namespacedName, + }) + Expect(err).NotTo(BeNil()) + Expect(err.Error()).To(ContainSubstring("invalid CIDR address")) + }) + + It("Should succeed when the NC subnet is in correct format", func() { + var uuid = "uuid" + var nc ncapi.NetworkContainer = ncapi.NetworkContainer{ + Spec: ncapi.NetworkContainerSpec{ + UUID: uuid, + }, + Status: ncapi.NetworkContainerStatus{ + State: "Initialized", + IPSubnet: "1.2.3.0/24", + }, + } + kubeClient.EXPECT().Get(gomock.Any(), namespacedName, gomock.Any()).SetArg(2, nc) + statusWriter := mockclients.NewMockStatusWriter(mockCtl) + statusWriter.EXPECT().Update(gomock.Any(), gomock.Any()).Return(nil) + kubeClient.EXPECT().Status().Return(statusWriter) + cnsClient.EXPECT().GetNC(cns.GetNetworkContainerRequest{NetworkContainerid: uuid}).Return(cns.GetNetworkContainerResponse{}, fmt.Errorf("NotFound")) + cnsClient.EXPECT().CreateOrUpdateNC(gomock.Any()).Return(nil) + _, err := reconciler.Reconcile(reconcile.Request{ + NamespacedName: namespacedName, + }) + Expect(err).To(BeNil()) + }) + }) +}) diff --git a/cns/multitenantcontroller/multitenantoperator/multitenantoperator_suite_test.go b/cns/multitenantcontroller/multitenantoperator/multitenantoperator_suite_test.go new file mode 100644 index 0000000000..b58dea2dc3 --- /dev/null +++ b/cns/multitenantcontroller/multitenantoperator/multitenantoperator_suite_test.go @@ -0,0 +1,13 @@ +package multitenantoperator + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestMultitenantoperator(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Multitenantoperator Suite") +} diff --git a/cns/requestcontroller/kubecontroller/crdreconciler.go b/cns/requestcontroller/kubecontroller/crdreconciler.go index c2255e944c..0b9318814e 100644 --- a/cns/requestcontroller/kubecontroller/crdreconciler.go +++ b/cns/requestcontroller/kubecontroller/crdreconciler.go @@ -67,12 +67,18 @@ func (r *CrdReconciler) Reconcile(request reconcile.Request) (reconcile.Result, return reconcile.Result{}, err } - if err = r.CNSClient.CreateOrUpdateNC(ncRequest, nodeNetConfig.Status.Scaler, nodeNetConfig.Spec); err != nil { + if err = r.CNSClient.CreateOrUpdateNC(ncRequest); err != nil { logger.Errorf("[cns-rc] Error creating or updating NC in reconcile: %v", err) // requeue return reconcile.Result{}, err } + if err = r.CNSClient.UpdateIPAMPoolMonitor(nodeNetConfig.Status.Scaler, nodeNetConfig.Spec); err != nil { + logger.Errorf("[cns-rc] Error update IPAM pool monitor in reconcile: %v", err) + // requeue + return reconcile.Result{}, err + } + return reconcile.Result{}, err } diff --git a/cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go b/cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go index 753d19c7eb..267d837733 100644 --- a/cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go +++ b/cns/requestcontroller/kubecontroller/crdrequestcontroller_test.go @@ -99,11 +99,23 @@ type MockCNSClient struct { } // we're just testing that reconciler interacts with CNS on Reconcile(). -func (mi *MockCNSClient) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error { +func (mi *MockCNSClient) CreateOrUpdateNC(ncRequest cns.CreateNetworkContainerRequest) error { mi.MockCNSUpdated = true return nil } +func (mi *MockCNSClient) UpdateIPAMPoolMonitor(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error { + return nil +} + +func (mi *MockCNSClient) DeleteNC(nc cns.DeleteNetworkContainerRequest) error { + return nil +} + +func (mi *MockCNSClient) GetNC(nc cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, error) { + return cns.GetNetworkContainerResponse{NetworkContainerID: nc.NetworkContainerid}, nil +} + func (mi *MockCNSClient) ReconcileNCState(ncRequest *cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.KubernetesPodInfo, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) error { mi.MockCNSInitialized = true mi.Pods = podInfoByIP diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index 3dc3d9815a..d4c8a86430 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -214,9 +214,12 @@ func (service *HTTPRestService) ReconcileNCState(ncRequest *cns.CreateNetworkCon return Success } - returnCode := service.CreateOrUpdateNetworkContainerInternal(*ncRequest, scalar, spec) - // If the NC was created successfully, then reconcile the allocated pod state + returnCode := service.CreateOrUpdateNetworkContainerInternal(*ncRequest) + if returnCode != Success { + return returnCode + } + returnCode = service.UpdateIPAMPoolMonitorInternal(scalar, spec) if returnCode != Success { return returnCode } @@ -255,8 +258,42 @@ func (service *HTTPRestService) ReconcileNCState(ncRequest *cns.CreateNetworkCon return 0 } +// GetNetworkContainerInternal gets network container details. +func (service *HTTPRestService) GetNetworkContainerInternal(req cns.GetNetworkContainerRequest) (cns.GetNetworkContainerResponse, int) { + getNetworkContainerResponse := service.getNetworkContainerResponse(req) + returnCode := getNetworkContainerResponse.Response.ReturnCode + return getNetworkContainerResponse, returnCode +} + +// DeleteNetworkContainerInternal deletes a network container. +func (service *HTTPRestService) DeleteNetworkContainerInternal(req cns.DeleteNetworkContainerRequest) int { + _, exist := service.getNetworkContainerDetails(req.NetworkContainerid) + if !exist { + logger.Printf("network container for id %v doesn't exist", req.NetworkContainerid) + return Success + } + + service.Lock() + defer service.Unlock() + if service.state.ContainerStatus != nil { + delete(service.state.ContainerStatus, req.NetworkContainerid) + } + + if service.state.ContainerIDByOrchestratorContext != nil { + for orchestratorContext, networkContainerID := range service.state.ContainerIDByOrchestratorContext { + if networkContainerID == req.NetworkContainerid { + delete(service.state.ContainerIDByOrchestratorContext, orchestratorContext) + break + } + } + } + + service.saveState() + return Success +} + // This API will be called by CNS RequestController on CRD update. -func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.CreateNetworkContainerRequest, scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) int { +func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.CreateNetworkContainerRequest) int { if req.NetworkContainerid == "" { logger.Errorf("[Azure CNS] Error. NetworkContainerid is empty") return NetworkContainerNotSpecified @@ -287,7 +324,6 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.C // Validate if state exists already existingNCInfo, ok := service.getNetworkContainerDetails(req.NetworkContainerid) - if ok { existingReq := existingNCInfo.CreateNetworkContainerRequest if reflect.DeepEqual(existingReq.IPConfiguration, req.IPConfiguration) != true { @@ -306,11 +342,15 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.C logger.Errorf(returnMessage) } - if err = service.IPAMPoolMonitor.Update(scalar, spec); err != nil { + return returnCode +} + +func (service *HTTPRestService) UpdateIPAMPoolMonitorInternal(scalar nnc.Scaler, spec nnc.NodeNetworkConfigSpec) int { + if err := service.IPAMPoolMonitor.Update(scalar, spec); err != nil { logger.Errorf("[cns-rc] Error creating or updating IPAM Pool Monitor: %v", err) // requeue return UnexpectedError } - return returnCode + return 0 } diff --git a/cns/restserver/internalapi_test.go b/cns/restserver/internalapi_test.go index 5b30b89064..42af4f64f5 100644 --- a/cns/restserver/internalapi_test.go +++ b/cns/restserver/internalapi_test.go @@ -367,10 +367,14 @@ func validateCreateOrUpdateNCInternal(t *testing.T, secondaryIpCount int, ncVers func createAndValidateNCRequest(t *testing.T, secondaryIPConfigs map[string]cns.SecondaryIPConfig, ncId, ncVersion string) { req := generateNetworkContainerRequest(secondaryIPConfigs, ncId, ncVersion) - returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize)) + returnCode := svc.CreateOrUpdateNetworkContainerInternal(req) if returnCode != 0 { t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode) } + returnCode = svc.UpdateIPAMPoolMonitorInternal(fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize)) + if returnCode != 0 { + t.Fatalf("Failed to UpdateIPAMPoolMonitorInternal, err: %d", returnCode) + } validateNetworkRequest(t, req) } @@ -535,10 +539,14 @@ func validateNCStateAfterReconcile(t *testing.T, ncRequest *cns.CreateNetworkCon func createNCReqInternal(t *testing.T, secondaryIPConfigs map[string]cns.SecondaryIPConfig, ncID, ncVersion string) cns.CreateNetworkContainerRequest { req := generateNetworkContainerRequest(secondaryIPConfigs, ncID, ncVersion) - returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize)) + returnCode := svc.CreateOrUpdateNetworkContainerInternal(req) if returnCode != 0 { t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode) } + returnCode = svc.UpdateIPAMPoolMonitorInternal(fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize)) + if returnCode != 0 { + t.Fatalf("Failed to UpdateIPAMPoolMonitorInternal, err: %d", returnCode) + } return req } diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 209d8fddf3..5ab176c60c 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -600,10 +600,14 @@ func TestIPAMMarkIPAsPendingWithPendingProgrammingIPs(t *testing.T) { // createNCRequest with NC version 0 req := generateNetworkContainerRequest(secondaryIPConfigs, testNCID, strconv.Itoa(0)) - returnCode := svc.CreateOrUpdateNetworkContainerInternal(req, fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize)) + returnCode := svc.CreateOrUpdateNetworkContainerInternal(req) if returnCode != 0 { t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode) } + returnCode = svc.UpdateIPAMPoolMonitorInternal(fakes.NewFakeScalar(releasePercent, requestPercent, batchSize), fakes.NewFakeNodeNetworkConfigSpec(initPoolSize)) + if returnCode != 0 { + t.Fatalf("Failed to UpdateIPAMPoolMonitorInternal, req: %+v, err: %d", req, returnCode) + } // Release pending programming IPs ips, err := svc.MarkIPAsPendingRelease(2) diff --git a/cns/service/main.go b/cns/service/main.go index e026456fc0..fff500df7d 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -5,12 +5,9 @@ package main import ( "bytes" + "context" "encoding/json" "fmt" - "github.com/Azure/azure-container-networking/cns/ipampoolmonitor" - "github.com/Azure/azure-container-networking/cns/requestcontroller" - "github.com/Azure/azure-container-networking/cns/requestcontroller/kubecontroller" - localtls "github.com/Azure/azure-container-networking/server/tls" "net/http" "os" "os/signal" @@ -20,9 +17,6 @@ import ( "syscall" "time" - "github.com/Azure/azure-container-networking/cns/nmagentclient" - - "context" "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/cnm/ipam" "github.com/Azure/azure-container-networking/cnm/network" @@ -32,12 +26,20 @@ import ( "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/hnsclient" "github.com/Azure/azure-container-networking/cns/imdsclient" + "github.com/Azure/azure-container-networking/cns/ipampoolmonitor" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/multitenantcontroller" + "github.com/Azure/azure-container-networking/cns/multitenantcontroller/multitenantoperator" + "github.com/Azure/azure-container-networking/cns/nmagentclient" + "github.com/Azure/azure-container-networking/cns/requestcontroller" + "github.com/Azure/azure-container-networking/cns/requestcontroller/kubecontroller" "github.com/Azure/azure-container-networking/cns/restserver" acn "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" + localtls "github.com/Azure/azure-container-networking/server/tls" "github.com/Azure/azure-container-networking/store" + ctrl "sigs.k8s.io/controller-runtime" ) const ( @@ -414,6 +416,8 @@ func main() { nodeID = cnsconfig.ManagedSettings.NodeID } else if cnsconfig.ChannelMode == cns.CRD { config.ChannelMode = cns.CRD + } else if cnsconfig.ChannelMode == cns.MultiTenantCRD { + config.ChannelMode = cns.MultiTenantCRD } else if acn.GetArg(acn.OptManaged).(bool) { config.ChannelMode = cns.Managed } @@ -504,13 +508,25 @@ func main() { if config.ChannelMode == cns.CRD { requestControllerStopChannel := make(chan struct{}) defer close(requestControllerStopChannel) - err = IniitalizeCRDState(httpRestService, cnsconfig, requestControllerStopChannel) + err = InitializeCRDState(httpRestService, cnsconfig, requestControllerStopChannel) if err != nil { logger.Errorf("Failed to start CRD Controller, err:%v.\n", err) return } } + // Initialize multi-tenant controller if the CNS is running in MultiTenantCRD mode. + // It must be started before we start HTTPRestService. + if config.ChannelMode == cns.MultiTenantCRD { + multiTenantControllerStopChannel := make(chan struct{}) + defer close(multiTenantControllerStopChannel) + err = InitializeMultiTenantController(httpRestService, cnsconfig, multiTenantControllerStopChannel) + if err != nil { + logger.Errorf("Failed to start multiTenantController, err:%v.\n", err) + return + } + } + logger.Printf("[Azure CNS] Start HTTP listener") if httpRestService != nil { err = httpRestService.Start(&config) @@ -670,8 +686,74 @@ func main() { logger.Close() } +func InitializeMultiTenantController(httpRestService cns.HTTPService, cnsconfig configuration.CNSConfig, exitChan <-chan struct{}) error { + var multiTenantController multitenantcontroller.MultiTenantController + kubeConfig, err := ctrl.GetConfig() + if err != nil { + return err + } + + //convert interface type to implementation type + httpRestServiceImpl, ok := httpRestService.(*restserver.HTTPRestService) + if !ok { + logger.Errorf("Failed to convert interface httpRestService to implementation: %v", httpRestService) + return fmt.Errorf("Failed to convert interface httpRestService to implementation: %v", + httpRestService) + } + + // Set orchestrator type + orchestrator := cns.SetOrchestratorTypeRequest{ + OrchestratorType: cns.KubernetesCRD, + } + httpRestServiceImpl.SetNodeOrchestrator(&orchestrator) + + // Create multiTenantController. + multiTenantController, err = multitenantoperator.NewMultiTenantController(httpRestServiceImpl, kubeConfig) + if err != nil { + logger.Errorf("Failed to create multiTenantController:%v", err) + return err + } + + // Wait for multiTenantController to start. + go func() { + for { + if err := multiTenantController.StartMultiTenantController(exitChan); err != nil { + logger.Errorf("Failed to start multiTenantController: %v", err) + } else { + logger.Printf("Exiting multiTenantController") + return + } + + // Retry after 1sec + time.Sleep(time.Second) + } + }() + for { + if multiTenantController.IsStarted() { + logger.Printf("MultiTenantController is started") + break + } + + logger.Printf("Waiting for multiTenantController to start...") + time.Sleep(time.Millisecond * 500) + } + + // TODO: do we need this to be running? + logger.Printf("Starting SyncHostNCVersion") + rootCxt := context.Background() + go func() { + // Periodically poll vfp programmed NC version from NMAgent + for { + <-time.NewTicker(cnsconfig.SyncHostNCVersionIntervalMs * time.Millisecond).C + httpRestServiceImpl.SyncHostNCVersion(rootCxt, cnsconfig.ChannelMode, cnsconfig.SyncHostNCTimeoutMs) + } + }() + + return nil +} + // initializeCRD state -func IniitalizeCRDState(httpRestService cns.HTTPService, cnsconfig configuration.CNSConfig, exitChan <-chan struct{}) error { +func InitializeCRDState(httpRestService cns.HTTPService, cnsconfig configuration.CNSConfig, exitChan <-chan struct{}) error { var requestController requestcontroller.RequestController logger.Printf("[Azure CNS] Starting request controller") diff --git a/go.mod b/go.mod index 3acaf711a1..50af1a8671 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/libnetwork v0.8.0-dev.2.0.20210525090646-64b7a4574d14 github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect + github.com/golang/mock v1.2.0 github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.8.0 github.com/hashicorp/golang-lru v0.5.3 // indirect diff --git a/go.sum b/go.sum index bbfbcc560b..b17f3b4cc1 100644 --- a/go.sum +++ b/go.sum @@ -163,6 +163,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -401,6 +402,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -578,6 +580,7 @@ k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= k8s.io/client-go v0.18.19 h1:ym6jwLYcdWFKrIm0tU4Ct6evujnA8/OQTVdwLKJp5rY= k8s.io/client-go v0.18.19/go.mod h1:lB+d4UqdzSjaU41VODLYm/oon3o05LAzsVpm6Me5XkY= k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= +k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= diff --git a/vendor/github.com/golang/mock/AUTHORS b/vendor/github.com/golang/mock/AUTHORS new file mode 100644 index 0000000000..660b8ccc8a --- /dev/null +++ b/vendor/github.com/golang/mock/AUTHORS @@ -0,0 +1,12 @@ +# This is the official list of GoMock authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# Please keep the list sorted. + +Alex Reece +Google Inc. diff --git a/vendor/github.com/golang/mock/CONTRIBUTORS b/vendor/github.com/golang/mock/CONTRIBUTORS new file mode 100644 index 0000000000..def849cab1 --- /dev/null +++ b/vendor/github.com/golang/mock/CONTRIBUTORS @@ -0,0 +1,37 @@ +# This is the official list of people who can contribute (and typically +# have contributed) code to the gomock repository. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# The submission process automatically checks to make sure +# that people submitting code are listed in this file (by email address). +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# http://code.google.com/legal/individual-cla-v1.0.html +# http://code.google.com/legal/corporate-cla-v1.0.html +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. + +# Names should be added to this file like so: +# Name +# +# An entry with two email addresses specifies that the +# first address should be used in the submit logs and +# that the second address should be recognized as the +# same person when interacting with Rietveld. + +# Please keep the list sorted. + +Aaron Jacobs +Alex Reece +David Symonds +Ryan Barrett diff --git a/vendor/github.com/golang/mock/LICENSE b/vendor/github.com/golang/mock/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/vendor/github.com/golang/mock/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/golang/mock/gomock/call.go b/vendor/github.com/golang/mock/gomock/call.go new file mode 100644 index 0000000000..3d54d9f5d0 --- /dev/null +++ b/vendor/github.com/golang/mock/gomock/call.go @@ -0,0 +1,420 @@ +// Copyright 2010 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gomock + +import ( + "fmt" + "reflect" + "strconv" + "strings" +) + +// Call represents an expected call to a mock. +type Call struct { + t TestHelper // for triggering test failures on invalid call setup + + receiver interface{} // the receiver of the method call + method string // the name of the method + methodType reflect.Type // the type of the method + args []Matcher // the args + origin string // file and line number of call setup + + preReqs []*Call // prerequisite calls + + // Expectations + minCalls, maxCalls int + + numCalls int // actual number made + + // actions are called when this Call is called. Each action gets the args and + // can set the return values by returning a non-nil slice. Actions run in the + // order they are created. + actions []func([]interface{}) []interface{} +} + +// newCall creates a *Call. It requires the method type in order to support +// unexported methods. +func newCall(t TestHelper, receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call { + t.Helper() + + // TODO: check arity, types. + margs := make([]Matcher, len(args)) + for i, arg := range args { + if m, ok := arg.(Matcher); ok { + margs[i] = m + } else if arg == nil { + // Handle nil specially so that passing a nil interface value + // will match the typed nils of concrete args. + margs[i] = Nil() + } else { + margs[i] = Eq(arg) + } + } + + origin := callerInfo(3) + actions := []func([]interface{}) []interface{}{func([]interface{}) []interface{} { + // Synthesize the zero value for each of the return args' types. + rets := make([]interface{}, methodType.NumOut()) + for i := 0; i < methodType.NumOut(); i++ { + rets[i] = reflect.Zero(methodType.Out(i)).Interface() + } + return rets + }} + return &Call{t: t, receiver: receiver, method: method, methodType: methodType, + args: margs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions} +} + +// AnyTimes allows the expectation to be called 0 or more times +func (c *Call) AnyTimes() *Call { + c.minCalls, c.maxCalls = 0, 1e8 // close enough to infinity + return c +} + +// MinTimes requires the call to occur at least n times. If AnyTimes or MaxTimes have not been called, MinTimes also +// sets the maximum number of calls to infinity. +func (c *Call) MinTimes(n int) *Call { + c.minCalls = n + if c.maxCalls == 1 { + c.maxCalls = 1e8 + } + return c +} + +// MaxTimes limits the number of calls to n times. If AnyTimes or MinTimes have not been called, MaxTimes also +// sets the minimum number of calls to 0. +func (c *Call) MaxTimes(n int) *Call { + c.maxCalls = n + if c.minCalls == 1 { + c.minCalls = 0 + } + return c +} + +// DoAndReturn declares the action to run when the call is matched. +// The return values from this function are returned by the mocked function. +// It takes an interface{} argument to support n-arity functions. +func (c *Call) DoAndReturn(f interface{}) *Call { + // TODO: Check arity and types here, rather than dying badly elsewhere. + v := reflect.ValueOf(f) + + c.addAction(func(args []interface{}) []interface{} { + vargs := make([]reflect.Value, len(args)) + ft := v.Type() + for i := 0; i < len(args); i++ { + if args[i] != nil { + vargs[i] = reflect.ValueOf(args[i]) + } else { + // Use the zero value for the arg. + vargs[i] = reflect.Zero(ft.In(i)) + } + } + vrets := v.Call(vargs) + rets := make([]interface{}, len(vrets)) + for i, ret := range vrets { + rets[i] = ret.Interface() + } + return rets + }) + return c +} + +// Do declares the action to run when the call is matched. The function's +// return values are ignored to retain backward compatibility. To use the +// return values call DoAndReturn. +// It takes an interface{} argument to support n-arity functions. +func (c *Call) Do(f interface{}) *Call { + // TODO: Check arity and types here, rather than dying badly elsewhere. + v := reflect.ValueOf(f) + + c.addAction(func(args []interface{}) []interface{} { + vargs := make([]reflect.Value, len(args)) + ft := v.Type() + for i := 0; i < len(args); i++ { + if args[i] != nil { + vargs[i] = reflect.ValueOf(args[i]) + } else { + // Use the zero value for the arg. + vargs[i] = reflect.Zero(ft.In(i)) + } + } + v.Call(vargs) + return nil + }) + return c +} + +// Return declares the values to be returned by the mocked function call. +func (c *Call) Return(rets ...interface{}) *Call { + c.t.Helper() + + mt := c.methodType + if len(rets) != mt.NumOut() { + c.t.Fatalf("wrong number of arguments to Return for %T.%v: got %d, want %d [%s]", + c.receiver, c.method, len(rets), mt.NumOut(), c.origin) + } + for i, ret := range rets { + if got, want := reflect.TypeOf(ret), mt.Out(i); got == want { + // Identical types; nothing to do. + } else if got == nil { + // Nil needs special handling. + switch want.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + // ok + default: + c.t.Fatalf("argument %d to Return for %T.%v is nil, but %v is not nillable [%s]", + i, c.receiver, c.method, want, c.origin) + } + } else if got.AssignableTo(want) { + // Assignable type relation. Make the assignment now so that the generated code + // can return the values with a type assertion. + v := reflect.New(want).Elem() + v.Set(reflect.ValueOf(ret)) + rets[i] = v.Interface() + } else { + c.t.Fatalf("wrong type of argument %d to Return for %T.%v: %v is not assignable to %v [%s]", + i, c.receiver, c.method, got, want, c.origin) + } + } + + c.addAction(func([]interface{}) []interface{} { + return rets + }) + + return c +} + +// Times declares the exact number of times a function call is expected to be executed. +func (c *Call) Times(n int) *Call { + c.minCalls, c.maxCalls = n, n + return c +} + +// SetArg declares an action that will set the nth argument's value, +// indirected through a pointer. Or, in the case of a slice, SetArg +// will copy value's elements into the nth argument. +func (c *Call) SetArg(n int, value interface{}) *Call { + c.t.Helper() + + mt := c.methodType + // TODO: This will break on variadic methods. + // We will need to check those at invocation time. + if n < 0 || n >= mt.NumIn() { + c.t.Fatalf("SetArg(%d, ...) called for a method with %d args [%s]", + n, mt.NumIn(), c.origin) + } + // Permit setting argument through an interface. + // In the interface case, we don't (nay, can't) check the type here. + at := mt.In(n) + switch at.Kind() { + case reflect.Ptr: + dt := at.Elem() + if vt := reflect.TypeOf(value); !vt.AssignableTo(dt) { + c.t.Fatalf("SetArg(%d, ...) argument is a %v, not assignable to %v [%s]", + n, vt, dt, c.origin) + } + case reflect.Interface: + // nothing to do + case reflect.Slice: + // nothing to do + default: + c.t.Fatalf("SetArg(%d, ...) referring to argument of non-pointer non-interface non-slice type %v [%s]", + n, at, c.origin) + } + + c.addAction(func(args []interface{}) []interface{} { + v := reflect.ValueOf(value) + switch reflect.TypeOf(args[n]).Kind() { + case reflect.Slice: + setSlice(args[n], v) + default: + reflect.ValueOf(args[n]).Elem().Set(v) + } + return nil + }) + return c +} + +// isPreReq returns true if other is a direct or indirect prerequisite to c. +func (c *Call) isPreReq(other *Call) bool { + for _, preReq := range c.preReqs { + if other == preReq || preReq.isPreReq(other) { + return true + } + } + return false +} + +// After declares that the call may only match after preReq has been exhausted. +func (c *Call) After(preReq *Call) *Call { + c.t.Helper() + + if c == preReq { + c.t.Fatalf("A call isn't allowed to be its own prerequisite") + } + if preReq.isPreReq(c) { + c.t.Fatalf("Loop in call order: %v is a prerequisite to %v (possibly indirectly).", c, preReq) + } + + c.preReqs = append(c.preReqs, preReq) + return c +} + +// Returns true if the minimum number of calls have been made. +func (c *Call) satisfied() bool { + return c.numCalls >= c.minCalls +} + +// Returns true iff the maximum number of calls have been made. +func (c *Call) exhausted() bool { + return c.numCalls >= c.maxCalls +} + +func (c *Call) String() string { + args := make([]string, len(c.args)) + for i, arg := range c.args { + args[i] = arg.String() + } + arguments := strings.Join(args, ", ") + return fmt.Sprintf("%T.%v(%s) %s", c.receiver, c.method, arguments, c.origin) +} + +// Tests if the given call matches the expected call. +// If yes, returns nil. If no, returns error with message explaining why it does not match. +func (c *Call) matches(args []interface{}) error { + if !c.methodType.IsVariadic() { + if len(args) != len(c.args) { + return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d", + c.origin, len(args), len(c.args)) + } + + for i, m := range c.args { + if !m.Matches(args[i]) { + return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v", + c.origin, strconv.Itoa(i), args[i], m) + } + } + } else { + if len(c.args) < c.methodType.NumIn()-1 { + return fmt.Errorf("Expected call at %s has the wrong number of matchers. Got: %d, want: %d", + c.origin, len(c.args), c.methodType.NumIn()-1) + } + if len(c.args) != c.methodType.NumIn() && len(args) != len(c.args) { + return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d", + c.origin, len(args), len(c.args)) + } + if len(args) < len(c.args)-1 { + return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: greater than or equal to %d", + c.origin, len(args), len(c.args)-1) + } + + for i, m := range c.args { + if i < c.methodType.NumIn()-1 { + // Non-variadic args + if !m.Matches(args[i]) { + return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v", + c.origin, strconv.Itoa(i), args[i], m) + } + continue + } + // The last arg has a possibility of a variadic argument, so let it branch + + // sample: Foo(a int, b int, c ...int) + if i < len(c.args) && i < len(args) { + if m.Matches(args[i]) { + // Got Foo(a, b, c) want Foo(matcherA, matcherB, gomock.Any()) + // Got Foo(a, b, c) want Foo(matcherA, matcherB, someSliceMatcher) + // Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC) + // Got Foo(a, b) want Foo(matcherA, matcherB) + // Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD) + continue + } + } + + // The number of actual args don't match the number of matchers, + // or the last matcher is a slice and the last arg is not. + // If this function still matches it is because the last matcher + // matches all the remaining arguments or the lack of any. + // Convert the remaining arguments, if any, into a slice of the + // expected type. + vargsType := c.methodType.In(c.methodType.NumIn() - 1) + vargs := reflect.MakeSlice(vargsType, 0, len(args)-i) + for _, arg := range args[i:] { + vargs = reflect.Append(vargs, reflect.ValueOf(arg)) + } + if m.Matches(vargs.Interface()) { + // Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, gomock.Any()) + // Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, someSliceMatcher) + // Got Foo(a, b) want Foo(matcherA, matcherB, gomock.Any()) + // Got Foo(a, b) want Foo(matcherA, matcherB, someEmptySliceMatcher) + break + } + // Wrong number of matchers or not match. Fail. + // Got Foo(a, b) want Foo(matcherA, matcherB, matcherC, matcherD) + // Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC, matcherD) + // Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD, matcherE) + // Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, matcherC, matcherD) + // Got Foo(a, b, c) want Foo(matcherA, matcherB) + return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v", + c.origin, strconv.Itoa(i), args[i:], c.args[i]) + + } + } + + // Check that all prerequisite calls have been satisfied. + for _, preReqCall := range c.preReqs { + if !preReqCall.satisfied() { + return fmt.Errorf("Expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v", + c.origin, preReqCall, c) + } + } + + // Check that the call is not exhausted. + if c.exhausted() { + return fmt.Errorf("Expected call at %s has already been called the max number of times.", c.origin) + } + + return nil +} + +// dropPrereqs tells the expected Call to not re-check prerequisite calls any +// longer, and to return its current set. +func (c *Call) dropPrereqs() (preReqs []*Call) { + preReqs = c.preReqs + c.preReqs = nil + return +} + +func (c *Call) call(args []interface{}) []func([]interface{}) []interface{} { + c.numCalls++ + return c.actions +} + +// InOrder declares that the given calls should occur in order. +func InOrder(calls ...*Call) { + for i := 1; i < len(calls); i++ { + calls[i].After(calls[i-1]) + } +} + +func setSlice(arg interface{}, v reflect.Value) { + va := reflect.ValueOf(arg) + for i := 0; i < v.Len(); i++ { + va.Index(i).Set(v.Index(i)) + } +} + +func (c *Call) addAction(action func([]interface{}) []interface{}) { + c.actions = append(c.actions, action) +} diff --git a/vendor/github.com/golang/mock/gomock/callset.go b/vendor/github.com/golang/mock/gomock/callset.go new file mode 100644 index 0000000000..c44a8a585b --- /dev/null +++ b/vendor/github.com/golang/mock/gomock/callset.go @@ -0,0 +1,108 @@ +// Copyright 2011 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gomock + +import ( + "bytes" + "fmt" +) + +// callSet represents a set of expected calls, indexed by receiver and method +// name. +type callSet struct { + // Calls that are still expected. + expected map[callSetKey][]*Call + // Calls that have been exhausted. + exhausted map[callSetKey][]*Call +} + +// callSetKey is the key in the maps in callSet +type callSetKey struct { + receiver interface{} + fname string +} + +func newCallSet() *callSet { + return &callSet{make(map[callSetKey][]*Call), make(map[callSetKey][]*Call)} +} + +// Add adds a new expected call. +func (cs callSet) Add(call *Call) { + key := callSetKey{call.receiver, call.method} + m := cs.expected + if call.exhausted() { + m = cs.exhausted + } + m[key] = append(m[key], call) +} + +// Remove removes an expected call. +func (cs callSet) Remove(call *Call) { + key := callSetKey{call.receiver, call.method} + calls := cs.expected[key] + for i, c := range calls { + if c == call { + // maintain order for remaining calls + cs.expected[key] = append(calls[:i], calls[i+1:]...) + cs.exhausted[key] = append(cs.exhausted[key], call) + break + } + } +} + +// FindMatch searches for a matching call. Returns error with explanation message if no call matched. +func (cs callSet) FindMatch(receiver interface{}, method string, args []interface{}) (*Call, error) { + key := callSetKey{receiver, method} + + // Search through the expected calls. + expected := cs.expected[key] + var callsErrors bytes.Buffer + for _, call := range expected { + err := call.matches(args) + if err != nil { + fmt.Fprintf(&callsErrors, "\n%v", err) + } else { + return call, nil + } + } + + // If we haven't found a match then search through the exhausted calls so we + // get useful error messages. + exhausted := cs.exhausted[key] + for _, call := range exhausted { + if err := call.matches(args); err != nil { + fmt.Fprintf(&callsErrors, "\n%v", err) + } + } + + if len(expected)+len(exhausted) == 0 { + fmt.Fprintf(&callsErrors, "there are no expected calls of the method %q for that receiver", method) + } + + return nil, fmt.Errorf(callsErrors.String()) +} + +// Failures returns the calls that are not satisfied. +func (cs callSet) Failures() []*Call { + failures := make([]*Call, 0, len(cs.expected)) + for _, calls := range cs.expected { + for _, call := range calls { + if !call.satisfied() { + failures = append(failures, call) + } + } + } + return failures +} diff --git a/vendor/github.com/golang/mock/gomock/controller.go b/vendor/github.com/golang/mock/gomock/controller.go new file mode 100644 index 0000000000..6fde25f508 --- /dev/null +++ b/vendor/github.com/golang/mock/gomock/controller.go @@ -0,0 +1,235 @@ +// Copyright 2010 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// GoMock - a mock framework for Go. +// +// Standard usage: +// (1) Define an interface that you wish to mock. +// type MyInterface interface { +// SomeMethod(x int64, y string) +// } +// (2) Use mockgen to generate a mock from the interface. +// (3) Use the mock in a test: +// func TestMyThing(t *testing.T) { +// mockCtrl := gomock.NewController(t) +// defer mockCtrl.Finish() +// +// mockObj := something.NewMockMyInterface(mockCtrl) +// mockObj.EXPECT().SomeMethod(4, "blah") +// // pass mockObj to a real object and play with it. +// } +// +// By default, expected calls are not enforced to run in any particular order. +// Call order dependency can be enforced by use of InOrder and/or Call.After. +// Call.After can create more varied call order dependencies, but InOrder is +// often more convenient. +// +// The following examples create equivalent call order dependencies. +// +// Example of using Call.After to chain expected call order: +// +// firstCall := mockObj.EXPECT().SomeMethod(1, "first") +// secondCall := mockObj.EXPECT().SomeMethod(2, "second").After(firstCall) +// mockObj.EXPECT().SomeMethod(3, "third").After(secondCall) +// +// Example of using InOrder to declare expected call order: +// +// gomock.InOrder( +// mockObj.EXPECT().SomeMethod(1, "first"), +// mockObj.EXPECT().SomeMethod(2, "second"), +// mockObj.EXPECT().SomeMethod(3, "third"), +// ) +// +// TODO: +// - Handle different argument/return types (e.g. ..., chan, map, interface). +package gomock + +import ( + "context" + "fmt" + "reflect" + "runtime" + "sync" +) + +// A TestReporter is something that can be used to report test failures. +// It is satisfied by the standard library's *testing.T. +type TestReporter interface { + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) +} + +// TestHelper is a TestReporter that has the Helper method. It is satisfied +// by the standard library's *testing.T. +type TestHelper interface { + TestReporter + Helper() +} + +// A Controller represents the top-level control of a mock ecosystem. +// It defines the scope and lifetime of mock objects, as well as their expectations. +// It is safe to call Controller's methods from multiple goroutines. +type Controller struct { + // T should only be called within a generated mock. It is not intended to + // be used in user code and may be changed in future versions. T is the + // TestReporter passed in when creating the Controller via NewController. + // If the TestReporter does not implment a TestHelper it will be wrapped + // with a nopTestHelper. + T TestHelper + mu sync.Mutex + expectedCalls *callSet + finished bool +} + +func NewController(t TestReporter) *Controller { + h, ok := t.(TestHelper) + if !ok { + h = nopTestHelper{t} + } + + return &Controller{ + T: h, + expectedCalls: newCallSet(), + } +} + +type cancelReporter struct { + TestHelper + cancel func() +} + +func (r *cancelReporter) Errorf(format string, args ...interface{}) { + r.TestHelper.Errorf(format, args...) +} +func (r *cancelReporter) Fatalf(format string, args ...interface{}) { + defer r.cancel() + r.TestHelper.Fatalf(format, args...) +} + +// WithContext returns a new Controller and a Context, which is cancelled on any +// fatal failure. +func WithContext(ctx context.Context, t TestReporter) (*Controller, context.Context) { + h, ok := t.(TestHelper) + if !ok { + h = nopTestHelper{t} + } + + ctx, cancel := context.WithCancel(ctx) + return NewController(&cancelReporter{h, cancel}), ctx +} + +type nopTestHelper struct { + TestReporter +} + +func (h nopTestHelper) Helper() {} + +func (ctrl *Controller) RecordCall(receiver interface{}, method string, args ...interface{}) *Call { + ctrl.T.Helper() + + recv := reflect.ValueOf(receiver) + for i := 0; i < recv.Type().NumMethod(); i++ { + if recv.Type().Method(i).Name == method { + return ctrl.RecordCallWithMethodType(receiver, method, recv.Method(i).Type(), args...) + } + } + ctrl.T.Fatalf("gomock: failed finding method %s on %T", method, receiver) + panic("unreachable") +} + +func (ctrl *Controller) RecordCallWithMethodType(receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call { + ctrl.T.Helper() + + call := newCall(ctrl.T, receiver, method, methodType, args...) + + ctrl.mu.Lock() + defer ctrl.mu.Unlock() + ctrl.expectedCalls.Add(call) + + return call +} + +func (ctrl *Controller) Call(receiver interface{}, method string, args ...interface{}) []interface{} { + ctrl.T.Helper() + + // Nest this code so we can use defer to make sure the lock is released. + actions := func() []func([]interface{}) []interface{} { + ctrl.T.Helper() + ctrl.mu.Lock() + defer ctrl.mu.Unlock() + + expected, err := ctrl.expectedCalls.FindMatch(receiver, method, args) + if err != nil { + origin := callerInfo(2) + ctrl.T.Fatalf("Unexpected call to %T.%v(%v) at %s because: %s", receiver, method, args, origin, err) + } + + // Two things happen here: + // * the matching call no longer needs to check prerequite calls, + // * and the prerequite calls are no longer expected, so remove them. + preReqCalls := expected.dropPrereqs() + for _, preReqCall := range preReqCalls { + ctrl.expectedCalls.Remove(preReqCall) + } + + actions := expected.call(args) + if expected.exhausted() { + ctrl.expectedCalls.Remove(expected) + } + return actions + }() + + var rets []interface{} + for _, action := range actions { + if r := action(args); r != nil { + rets = r + } + } + + return rets +} + +func (ctrl *Controller) Finish() { + ctrl.T.Helper() + + ctrl.mu.Lock() + defer ctrl.mu.Unlock() + + if ctrl.finished { + ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.") + } + ctrl.finished = true + + // If we're currently panicking, probably because this is a deferred call, + // pass through the panic. + if err := recover(); err != nil { + panic(err) + } + + // Check that all remaining expected calls are satisfied. + failures := ctrl.expectedCalls.Failures() + for _, call := range failures { + ctrl.T.Errorf("missing call(s) to %v", call) + } + if len(failures) != 0 { + ctrl.T.Fatalf("aborting test due to missing call(s)") + } +} + +func callerInfo(skip int) string { + if _, file, line, ok := runtime.Caller(skip + 1); ok { + return fmt.Sprintf("%s:%d", file, line) + } + return "unknown file" +} diff --git a/vendor/github.com/golang/mock/gomock/matchers.go b/vendor/github.com/golang/mock/gomock/matchers.go new file mode 100644 index 0000000000..189796f865 --- /dev/null +++ b/vendor/github.com/golang/mock/gomock/matchers.go @@ -0,0 +1,122 @@ +// Copyright 2010 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gomock + +import ( + "fmt" + "reflect" +) + +// A Matcher is a representation of a class of values. +// It is used to represent the valid or expected arguments to a mocked method. +type Matcher interface { + // Matches returns whether x is a match. + Matches(x interface{}) bool + + // String describes what the matcher matches. + String() string +} + +type anyMatcher struct{} + +func (anyMatcher) Matches(x interface{}) bool { + return true +} + +func (anyMatcher) String() string { + return "is anything" +} + +type eqMatcher struct { + x interface{} +} + +func (e eqMatcher) Matches(x interface{}) bool { + return reflect.DeepEqual(e.x, x) +} + +func (e eqMatcher) String() string { + return fmt.Sprintf("is equal to %v", e.x) +} + +type nilMatcher struct{} + +func (nilMatcher) Matches(x interface{}) bool { + if x == nil { + return true + } + + v := reflect.ValueOf(x) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, + reflect.Ptr, reflect.Slice: + return v.IsNil() + } + + return false +} + +func (nilMatcher) String() string { + return "is nil" +} + +type notMatcher struct { + m Matcher +} + +func (n notMatcher) Matches(x interface{}) bool { + return !n.m.Matches(x) +} + +func (n notMatcher) String() string { + // TODO: Improve this if we add a NotString method to the Matcher interface. + return "not(" + n.m.String() + ")" +} + +type assignableToTypeOfMatcher struct { + targetType reflect.Type +} + +func (m assignableToTypeOfMatcher) Matches(x interface{}) bool { + return reflect.TypeOf(x).AssignableTo(m.targetType) +} + +func (m assignableToTypeOfMatcher) String() string { + return "is assignable to " + m.targetType.Name() +} + +// Constructors +func Any() Matcher { return anyMatcher{} } +func Eq(x interface{}) Matcher { return eqMatcher{x} } +func Nil() Matcher { return nilMatcher{} } +func Not(x interface{}) Matcher { + if m, ok := x.(Matcher); ok { + return notMatcher{m} + } + return notMatcher{Eq(x)} +} + +// AssignableToTypeOf is a Matcher that matches if the parameter to the mock +// function is assignable to the type of the parameter to this function. +// +// Example usage: +// +// dbMock.EXPECT(). +// Insert(gomock.AssignableToTypeOf(&EmployeeRecord{})). +// Return(errors.New("DB error")) +// +func AssignableToTypeOf(x interface{}) Matcher { + return assignableToTypeOfMatcher{reflect.TypeOf(x)} +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 3d67be6c84..db7c68c299 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -84,6 +84,9 @@ github.com/gogo/protobuf/sortkeys # github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 ## explicit github.com/golang/groupcache/lru +# github.com/golang/mock v1.2.0 +## explicit +github.com/golang/mock/gomock # github.com/golang/protobuf v1.4.1 github.com/golang/protobuf/proto github.com/golang/protobuf/ptypes