diff --git a/network/hnswrapper/hnsv2wrapperfake.go b/network/hnswrapper/hnsv2wrapperfake.go index 50f392c884..7525e9dbe3 100644 --- a/network/hnswrapper/hnsv2wrapperfake.go +++ b/network/hnswrapper/hnsv2wrapperfake.go @@ -18,8 +18,6 @@ import ( "github.com/Microsoft/hcsshim/hcn" ) -const networkName = "azure" - var errorFakeHNS = errors.New("errorFakeHNS Error") func newErrorFakeHNS(errStr string) error { @@ -185,7 +183,9 @@ func (f Hnsv2wrapperFake) GetNetworkByName(networkName string) (*hcn.HostCompute if network, ok := f.Cache.networks[networkName]; ok { return network.GetHCNObj(), nil } - return nil, hcn.NetworkNotFoundError{} + return nil, hcn.NetworkNotFoundError{ + NetworkName: networkName, + } } func (f Hnsv2wrapperFake) GetNetworkByID(networkID string) (*hcn.HostComputeNetwork, error) { @@ -528,4 +528,5 @@ type FakeEndpointPolicy struct { LocalPorts string `json:",omitempty"` RemotePorts string `json:",omitempty"` Priority int `json:",omitempty"` + // FIXME should include RuleType too, but that will require updating every instance of this struct in UTs } diff --git a/npm/cmd/start.go b/npm/cmd/start.go index 485ba0254f..a8773a7276 100644 --- a/npm/cmd/start.go +++ b/npm/cmd/start.go @@ -32,8 +32,7 @@ import ( var npmV2DataplaneCfg = &dataplane.Config{ IPSetManagerCfg: &ipsets.IPSetManagerCfg{ - NetworkName: "azure", // FIXME should be specified in DP config instead - // NOTE: IPSetMode must be set later by the npm ConfigMap or default config + // NOTE: NetworkName and IPSetMode must be set later by the npm ConfigMap or default config }, PolicyManagerCfg: &policies.PolicyManagerCfg{ PolicyMode: policies.IPSetPolicyMode, @@ -124,6 +123,12 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error { stopChannel := wait.NeverStop if config.Toggles.EnableV2NPM { // update the dataplane config + if config.WindowsNetworkName == "" { + npmV2DataplaneCfg.NetworkName = util.AzureNetworkName + } else { + npmV2DataplaneCfg.NetworkName = config.WindowsNetworkName + } + npmV2DataplaneCfg.PlaceAzureChainFirst = config.Toggles.PlaceAzureChainFirst if config.Toggles.ApplyIPSetsOnNeed { npmV2DataplaneCfg.IPSetMode = ipsets.ApplyOnNeed diff --git a/npm/config/config.go b/npm/config/config.go index 66bda2c819..52d7844a41 100644 --- a/npm/config/config.go +++ b/npm/config/config.go @@ -16,6 +16,7 @@ const ( // DefaultConfig is the guaranteed configuration NPM can run in out of the box var DefaultConfig = Config{ + WindowsNetworkName: util.AzureNetworkName, ResyncPeriodInMinutes: defaultResyncPeriod, ListeningPort: defaultListeningPort, @@ -47,14 +48,14 @@ type GrpcServerConfig struct { } type Config struct { - ResyncPeriodInMinutes int `json:"ResyncPeriodInMinutes,omitempty"` - - ListeningPort int `json:"ListeningPort,omitempty"` - ListeningAddress string `json:"ListeningAddress,omitempty"` - - Transport GrpcServerConfig `json:"Transport,omitempty"` - - Toggles Toggles `json:"Toggles,omitempty"` + // WindowsNetworkName can be either 'azure' or 'Calico' (case sensitive). + // It can also be the empty string, which results in the default value of 'azure'. + WindowsNetworkName string `json:"WindowsNetworkName,omitempty"` + ResyncPeriodInMinutes int `json:"ResyncPeriodInMinutes,omitempty"` + ListeningPort int `json:"ListeningPort,omitempty"` + ListeningAddress string `json:"ListeningAddress,omitempty"` + Transport GrpcServerConfig `json:"Transport,omitempty"` + Toggles Toggles `json:"Toggles,omitempty"` } type Toggles struct { diff --git a/npm/examples/windows/azure-npm-capz.yaml b/npm/examples/windows/azure-npm-capz.yaml new file mode 100644 index 0000000000..b6c703aa00 --- /dev/null +++ b/npm/examples/windows/azure-npm-capz.yaml @@ -0,0 +1,162 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure-npm + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: azure-npm + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +rules: + - apiGroups: + - "" + resources: + - pods + - nodes + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: azure-npm-binding + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +subjects: + - kind: ServiceAccount + name: azure-npm + namespace: kube-system +roleRef: + kind: ClusterRole + name: azure-npm + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-npm-win + namespace: kube-system + labels: + app: azure-npm + addonmanager.kubernetes.io/mode: EnsureExists +spec: + selector: + matchLabels: + k8s-app: azure-npm + template: + metadata: + labels: + k8s-app: azure-npm + annotations: + azure.npm/scrapeable: "" + spec: + priorityClassName: system-node-critical + tolerations: + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: azure-npm + # setting to future version since it will use the updated Dockerfile + image: mcr.microsoft.com/containernetworking/azure-npm:v1.4.42 + command: ["powershell.exe"] + args: + [ + '.\setkubeconfigpath-capz.ps1', + ";", + "powershell.exe", + '.\npm.exe', + "start", + '--kubeconfig=.\kubeconfig', + ] + resources: + limits: + cpu: 250m + memory: 300Mi + requests: + cpu: 250m + env: + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: NPM_CONFIG + value: .\\etc\\azure-npm\\azure-npm.json + volumeMounts: + - name: azure-npm-config + mountPath: .\\etc\\azure-npm + nodeSelector: + kubernetes.io/os: windows + volumes: + - name: azure-npm-config + configMap: + name: azure-npm-config + serviceAccountName: azure-npm +--- +apiVersion: v1 +kind: Service +metadata: + name: npm-metrics-cluster-service + namespace: kube-system + labels: + app: npm-metrics +spec: + selector: + k8s-app: azure-npm + ports: + - port: 9000 + targetPort: 10091 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: azure-npm-config + namespace: kube-system +data: + azure-npm.json: | + { + "WindowsNetworkName": "Calico", + "ResyncPeriodInMinutes": 15, + "ListeningPort": 10091, + "ListeningAddress": "0.0.0.0", + "Toggles": { + "EnablePrometheusMetrics": true, + "EnablePprof": true, + "EnableHTTPDebugAPI": true, + "EnableV2NPM": true, + "PlaceAzureChainFirst": true, + "ApplyIPSetsOnNeed": false + }, + "Transport": { + "Address": "azure-npm.kube-system.svc.cluster.local", + "Port": 10092, + "ServicePort": 9001 + } + } diff --git a/npm/examples/windows/long-running-pod-for-capz.yaml b/npm/examples/windows/long-running-pod-for-capz.yaml new file mode 100644 index 0000000000..9d256daab9 --- /dev/null +++ b/npm/examples/windows/long-running-pod-for-capz.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: long-runner + namespace: npm-e2e-longrunner +spec: + replicas: 1 + selector: + matchLabels: + app: long-runner + template: + metadata: + labels: + app: long-runner + spec: + containers: + - command: + - /agnhost + - serve-hostname + - --tcp + - --http=false + - --port + - "80" + image: k8s.gcr.io/e2e-test-images/agnhost:2.33 + imagePullPolicy: IfNotPresent + name: cont-80-tcp + ports: + - containerPort: 80 + name: serve-80-tcp + protocol: TCP diff --git a/npm/examples/windows/setkubeconfigpath-capz.ps1 b/npm/examples/windows/setkubeconfigpath-capz.ps1 new file mode 100644 index 0000000000..883fe6b6fc --- /dev/null +++ b/npm/examples/windows/setkubeconfigpath-capz.ps1 @@ -0,0 +1,11 @@ +# pull the server value from the kubeconfig on host to construct our own kubeconfig, but using service principal settings +# this is required to build a kubeconfig using the kubeconfig on disk in c:\etc\kubernetes, and the service principle granted in the container mount, to generate clientset +$cpEndpoint = Get-Content C:\etc\kubernetes\kubelet.conf | ForEach-Object -Process {if($_.Contains("server:")) {$_.Trim().Split()[1]}} +$token = Get-Content -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\var\run\secrets\kubernetes.io\serviceaccount\token +$ca = Get-Content -Raw -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\var\run\secrets\kubernetes.io\serviceaccount\ca.crt +$caBase64 = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($ca)) +$server = "server: $cpEndpoint" +(Get-Content $env:CONTAINER_SANDBOX_MOUNT_POINT\kubeconfigtemplate.yaml). + replace("", $caBase64). + replace("", $server.Trim()). + replace("", $token) | Set-Content $env:CONTAINER_SANDBOX_MOUNT_POINT\kubeconfig -Force diff --git a/npm/pkg/dataplane/dataplane-test-cases_windows_test.go b/npm/pkg/dataplane/dataplane-test-cases_windows_test.go index 8d17c0922e..87dd6342c3 100644 --- a/npm/pkg/dataplane/dataplane-test-cases_windows_test.go +++ b/npm/pkg/dataplane/dataplane-test-cases_windows_test.go @@ -16,6 +16,7 @@ const ( nsCrudTag Tag = "namespace-crud" netpolCrudTag Tag = "netpol-crud" reconcileTag Tag = "reconcile" + calicoTag Tag = "calico" ) const ( @@ -53,6 +54,18 @@ var ( var ( defaultWindowsDPCfg = &Config{ IPSetManagerCfg: &ipsets.IPSetManagerCfg{ + NetworkName: "azure", + IPSetMode: ipsets.ApplyAllIPSets, + AddEmptySetToLists: true, + }, + PolicyManagerCfg: &policies.PolicyManagerCfg{ + PolicyMode: policies.IPSetPolicyMode, + }, + } + + windowsCalicoDPCfg = &Config{ + IPSetManagerCfg: &ipsets.IPSetManagerCfg{ + NetworkName: "Calico", IPSetMode: ipsets.ApplyAllIPSets, AddEmptySetToLists: true, }, @@ -88,7 +101,7 @@ func policyXBaseOnK1V1() *networkingv1.NetworkPolicy { } } -func getAllSerialTests() []*SerialTestCase { +func basicTests() []*SerialTestCase { return []*SerialTestCase{ { Description: "pod created", @@ -573,6 +586,243 @@ func getAllSerialTests() []*SerialTestCase { } } +func capzCalicoTests() []*SerialTestCase { + return []*SerialTestCase{ + { + Description: "Calico Network: base ACLs", + Actions: []*Action{ + CreateEndpoint(endpoint1, ip1), + CreatePod("x", "a", ip1, thisNode, map[string]string{"k1": "v1"}), + ApplyDP(), + }, + TestCaseMetadata: &TestCaseMetadata{ + Tags: []Tag{ + calicoTag, + podCrudTag, + }, + DpCfg: windowsCalicoDPCfg, + InitialEndpoints: nil, + ExpectedSetPolicies: []*hcn.SetPolicySetting{ + dptestutils.SetPolicy(emptySet), + dptestutils.SetPolicy(allNamespaces, emptySet.GetHashedName(), nsXSet.GetHashedName()), + dptestutils.SetPolicy(nsXSet, ip1), + dptestutils.SetPolicy(podK1Set, ip1), + dptestutils.SetPolicy(podK1V1Set, ip1), + }, + ExpectedEnpdointACLs: map[string][]*hnswrapper.FakeEndpointPolicy{ + endpoint1: { + { + ID: "azure-acl-baseazurewireserver", + Action: "Block", + Direction: "Out", + Priority: 200, + RemoteAddresses: "168.63.129.16/32", + RemotePorts: "80", + Protocols: "6", + }, + { + ID: "azure-acl-baseallowinswitch", + Action: "Allow", + Direction: "In", + Priority: 65499, + }, + { + ID: "azure-acl-baseallowoutswitch", + Action: "Allow", + Direction: "Out", + Priority: 65499, + }, + { + ID: "azure-acl-baseallowinhost", + Action: "Allow", + Direction: "In", + LocalAddresses: "", + Priority: 0, + RemoteAddresses: "", + // RuleType is unsupported in FakeEndpointPolicy + // RuleType: "Host", + }, + { + ID: "azure-acl-baseallowouthost", + Action: "Allow", + Direction: "Out", + LocalAddresses: "", + Priority: 0, + RemoteAddresses: "", + // RuleType is unsupported in FakeEndpointPolicy + // RuleType: "Host", + }, + }, + }, + }, + }, + { + Description: "Calico Network: add netpol", + Actions: []*Action{ + CreateEndpoint(endpoint1, ip1), + CreatePod("x", "a", ip1, thisNode, map[string]string{"k1": "v1"}), + ApplyDP(), + UpdatePolicy(policyXBaseOnK1V1()), + }, + TestCaseMetadata: &TestCaseMetadata{ + Tags: []Tag{ + calicoTag, + podCrudTag, + netpolCrudTag, + }, + DpCfg: windowsCalicoDPCfg, + InitialEndpoints: nil, + ExpectedSetPolicies: []*hcn.SetPolicySetting{ + dptestutils.SetPolicy(emptySet), + dptestutils.SetPolicy(allNamespaces, emptySet.GetHashedName(), nsXSet.GetHashedName()), + dptestutils.SetPolicy(nsXSet, ip1), + dptestutils.SetPolicy(podK1Set, ip1), + dptestutils.SetPolicy(podK1V1Set, ip1), + }, + ExpectedEnpdointACLs: map[string][]*hnswrapper.FakeEndpointPolicy{ + endpoint1: { + { + ID: "azure-acl-baseazurewireserver", + Action: "Block", + Direction: "Out", + Priority: 200, + RemoteAddresses: "168.63.129.16/32", + RemotePorts: "80", + Protocols: "6", + }, + { + ID: "azure-acl-baseallowinswitch", + Action: "Allow", + Direction: "In", + Priority: 65499, + }, + { + ID: "azure-acl-baseallowoutswitch", + Action: "Allow", + Direction: "Out", + Priority: 65499, + }, + { + ID: "azure-acl-baseallowinhost", + Action: "Allow", + Direction: "In", + LocalAddresses: "", + Priority: 0, + RemoteAddresses: "", + // RuleType is unsupported in FakeEndpointPolicy + // RuleType: "Host", + }, + { + ID: "azure-acl-baseallowouthost", + Action: "Allow", + Direction: "Out", + LocalAddresses: "", + Priority: 0, + RemoteAddresses: "", + // RuleType is unsupported in FakeEndpointPolicy + // RuleType: "Host", + }, + { + ID: "azure-acl-x-base", + Protocols: "", + Action: "Allow", + Direction: "In", + LocalAddresses: "", + RemoteAddresses: "", + LocalPorts: "", + RemotePorts: "", + Priority: 222, + }, + { + ID: "azure-acl-x-base", + Protocols: "", + Action: "Allow", + Direction: "Out", + LocalAddresses: "", + RemoteAddresses: "", + LocalPorts: "", + RemotePorts: "", + Priority: 222, + }, + }, + }, + }, + }, + { + Description: "Calico Network: add then remove netpol", + Actions: []*Action{ + CreateEndpoint(endpoint1, ip1), + CreatePod("x", "a", ip1, thisNode, map[string]string{"k1": "v1"}), + ApplyDP(), + UpdatePolicy(policyXBaseOnK1V1()), + DeletePolicyByObject(policyXBaseOnK1V1()), + }, + TestCaseMetadata: &TestCaseMetadata{ + Tags: []Tag{ + calicoTag, + podCrudTag, + netpolCrudTag, + }, + DpCfg: windowsCalicoDPCfg, + + InitialEndpoints: nil, + ExpectedSetPolicies: []*hcn.SetPolicySetting{ + dptestutils.SetPolicy(emptySet), + dptestutils.SetPolicy(allNamespaces, emptySet.GetHashedName(), nsXSet.GetHashedName()), + dptestutils.SetPolicy(nsXSet, ip1), + dptestutils.SetPolicy(podK1Set, ip1), + dptestutils.SetPolicy(podK1V1Set, ip1), + }, + ExpectedEnpdointACLs: map[string][]*hnswrapper.FakeEndpointPolicy{ + endpoint1: { + { + ID: "azure-acl-baseazurewireserver", + Action: "Block", + Direction: "Out", + Priority: 200, + RemoteAddresses: "168.63.129.16/32", + RemotePorts: "80", + Protocols: "6", + }, + { + ID: "azure-acl-baseallowinswitch", + Action: "Allow", + Direction: "In", + Priority: 65499, + }, + { + ID: "azure-acl-baseallowoutswitch", + Action: "Allow", + Direction: "Out", + Priority: 65499, + }, + { + ID: "azure-acl-baseallowinhost", + Action: "Allow", + Direction: "In", + LocalAddresses: "", + Priority: 0, + RemoteAddresses: "", + // RuleType is unsupported in FakeEndpointPolicy + // RuleType: "Host", + }, + { + ID: "azure-acl-baseallowouthost", + Action: "Allow", + Direction: "Out", + LocalAddresses: "", + Priority: 0, + RemoteAddresses: "", + // RuleType is unsupported in FakeEndpointPolicy + // RuleType: "Host", + }, + }, + }, + }, + }, + } +} + func getAllMultiJobTests() []*MultiJobTestCase { return []*MultiJobTestCase{ { diff --git a/npm/pkg/dataplane/dataplane_windows.go b/npm/pkg/dataplane/dataplane_windows.go index 9abb1f799f..73d89bc1bc 100644 --- a/npm/pkg/dataplane/dataplane_windows.go +++ b/npm/pkg/dataplane/dataplane_windows.go @@ -56,7 +56,7 @@ func (dp *DataPlane) getNetworkInfo() error { var err error for ; true; <-ticker.C { - err = dp.setNetworkIDByName(util.AzureNetworkName) + err = dp.setNetworkIDByName(dp.NetworkName) if err == nil || !isNetworkNotFoundErr(err) { return err } @@ -65,7 +65,7 @@ func (dp *DataPlane) getNetworkInfo() error { break } klog.Infof("[DataPlane Windows] Network with name %s not found. Retrying in %d seconds, Current retry number %d, max retries: %d", - util.AzureNetworkName, + dp.NetworkName, maxNoNetSleepTime, retryNumber, maxNoNetRetryCount, @@ -371,6 +371,14 @@ func (dp *DataPlane) refreshPodEndpoints() error { dp.endpointCache.cache[ip] = npmEP // NOTE: TSGs rely on this log line klog.Infof("updating endpoint cache to include %s: %+v", npmEP.ip, npmEP) + + if dp.NetworkName == util.CalicoNetworkName { + // NOTE 1: connectivity may be broken for an endpoint until this method is called + // NOTE 2: if NPM restarted, technically we could call into HNS to add the base ACLs even if they already exist on the Endpoint. + // It doesn't seem worthwhile to account for these edge-cases since using calico network is currently intended just for testing + klog.Infof("adding base ACLs for calico CNI endpoint. IP: %s. ID: %s", ip, npmEP.id) + dp.policyMgr.AddBaseACLsForCalicoCNI(npmEP.id) + } } else if oldNPMEP.id != endpoint.Id { // multiple endpoints can have the same IP address, but there should be one endpoint ID per pod // throw away old endpoints that have the same IP as a current endpoint (the old endpoint is getting deleted) @@ -388,6 +396,14 @@ func (dp *DataPlane) refreshPodEndpoints() error { // NOTE: TSGs rely on this log line klog.Infof("updating endpoint cache for previously cached IP %s: %+v with stalePodKey %+v", npmEP.ip, npmEP, npmEP.stalePodKey) } + + if dp.NetworkName == util.CalicoNetworkName { + // NOTE 1: connectivity may be broken for an endpoint until this method is called + // NOTE 2: if NPM restarted, technically we could call into HNS to add the base ACLs even if they already exist on the Endpoint. + // It doesn't seem worthwhile to account for these edge-cases since using calico network is currently intended just for testing + klog.Infof("adding base ACLs for calico CNI endpoint. IP: %s. ID: %s", ip, npmEP.id) + dp.policyMgr.AddBaseACLsForCalicoCNI(npmEP.id) + } } } @@ -428,5 +444,6 @@ func (dp *DataPlane) setNetworkIDByName(networkName string) error { } func isNetworkNotFoundErr(err error) bool { - return strings.Contains(err.Error(), fmt.Sprintf("Network name \"%s\" not found", util.AzureNetworkName)) + return strings.Contains(err.Error(), fmt.Sprintf("Network name %q not found", util.AzureNetworkName)) || + strings.Contains(err.Error(), fmt.Sprintf("Network name %q not found", util.CalicoNetworkName)) } diff --git a/npm/pkg/dataplane/dataplane_windows_test.go b/npm/pkg/dataplane/dataplane_windows_test.go index 5fd4a18b6f..da901ac9ec 100644 --- a/npm/pkg/dataplane/dataplane_windows_test.go +++ b/npm/pkg/dataplane/dataplane_windows_test.go @@ -11,6 +11,7 @@ import ( dptestutils "github.com/Azure/azure-container-networking/npm/pkg/dataplane/testutils" "github.com/pkg/errors" "github.com/stretchr/testify/require" + "k8s.io/klog" ) const ( @@ -18,15 +19,29 @@ const ( threadedHNSLatency = time.Duration(50 * time.Millisecond) ) -func TestAllSerialCases(t *testing.T) { - tests := getAllSerialTests() +func TestBasics(t *testing.T) { + testSerialCases(t, basicTests()) +} + +func TestCapzCalico(t *testing.T) { + testSerialCases(t, capzCalicoTests()) +} + +func TestAllMultiJobCases(t *testing.T) { + testMultiJobCases(t, getAllMultiJobTests()) +} + +func testSerialCases(t *testing.T, tests []*SerialTestCase) { + fmt.Printf("tests: %+v\n", tests) for i, tt := range tests { i := i tt := tt + klog.Infof("tt: %+v", tt) t.Run(tt.Description, func(t *testing.T) { + klog.Infof("tt in: %+v", tt) t.Logf("beginning test #%d. Description: [%s]. Tags: %+v", i, tt.Description, tt.Tags) - hns := ipsets.GetHNSFake(t) + hns := ipsets.GetHNSFake(t, tt.DpCfg.NetworkName) hns.Delay = defaultHNSLatency io := common.NewMockIOShimWithFakeHNS(hns) for _, ep := range tt.InitialEndpoints { @@ -36,6 +51,7 @@ func TestAllSerialCases(t *testing.T) { dp, err := NewDataPlane(thisNode, io, tt.DpCfg, nil) require.NoError(t, err, "failed to initialize dp") + require.NotNil(t, dp, "failed to initialize dp (nil)") for j, a := range tt.Actions { var err error @@ -53,15 +69,14 @@ func TestAllSerialCases(t *testing.T) { } } -func TestAllMultiJobCases(t *testing.T) { - tests := getAllMultiJobTests() +func testMultiJobCases(t *testing.T, tests []*MultiJobTestCase) { for i, tt := range tests { i := i tt := tt t.Run(tt.Description, func(t *testing.T) { t.Logf("beginning test #%d. Description: [%s]. Tags: %+v", i, tt.Description, tt.Tags) - hns := ipsets.GetHNSFake(t) + hns := ipsets.GetHNSFake(t, tt.DpCfg.NetworkName) hns.Delay = threadedHNSLatency io := common.NewMockIOShimWithFakeHNS(hns) for _, ep := range tt.InitialEndpoints { diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager.go b/npm/pkg/dataplane/ipsets/ipsetmanager.go index 7815965394..dc80dfae56 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager.go @@ -15,18 +15,18 @@ import ( type IPSetMode string /* - IPSet Modes - - - ApplyAllIPSets: - - all ipsets are added to the kernel - - ipsets are removed from the kernel when they are deleted from the cache - - creates empty ipsets - - adds empty/unreferenced ipsets to the toDelete cache periodically - - - ApplyOnNeed: - - ipsets are added to the kernel when they are referenced by network policies or lists in the kernel - - ipsets are removed from the kernel when they no longer have a reference - - removes empty/unreferenced ipsets from the cache periodically +IPSet Modes + +- ApplyAllIPSets: + - all ipsets are added to the kernel + - ipsets are removed from the kernel when they are deleted from the cache + - creates empty ipsets + - adds empty/unreferenced ipsets to the toDelete cache periodically + +- ApplyOnNeed: + - ipsets are added to the kernel when they are referenced by network policies or lists in the kernel + - ipsets are removed from the kernel when they no longer have a reference + - removes empty/unreferenced ipsets from the cache periodically */ const ( ApplyAllIPSets IPSetMode = "all" @@ -56,7 +56,7 @@ type IPSetManager struct { type IPSetManagerCfg struct { IPSetMode IPSetMode - // NetworkName can be left empty or set to 'azure' (the only supported network) + // NetworkName can be left empty or set to 'azure' or 'Calico' (case sensitive) NetworkName string // AddEmptySetToLists determines whether all lists should have an empty set as a member. // This is necessary for HNS (Windows); otherwise, an allow ACL with a list condition @@ -75,9 +75,9 @@ func NewIPSetManager(iMgrCfg *IPSetManagerCfg, ioShim *common.IOShim) *IPSetMana } /* - Reconcile removes empty/unreferenced sets from the cache. - For ApplyAllIPSets mode, those sets are added to the toDeleteCache. - We can't delete from kernel immediately unless we lock iMgr during policy CRUD. +Reconcile removes empty/unreferenced sets from the cache. +For ApplyAllIPSets mode, those sets are added to the toDeleteCache. +We can't delete from kernel immediately unless we lock iMgr during policy CRUD. */ func (iMgr *IPSetManager) Reconcile() { iMgr.Lock() diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go index 00538002af..a829520976 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_windows.go @@ -20,7 +20,7 @@ const ( donotResetIPSets = false ) -var errUnsupportedNetwork = errors.New("only 'azure' network is supported") +var errUnsupportedNetwork = errors.New("only 'azure' and 'Calico' networks are supported") type networkPolicyBuilder struct { toAddSets map[string]*hcn.SetPolicySetting @@ -242,12 +242,17 @@ func (iMgr *IPSetManager) applyIPSets() error { // networkPolicyBuild which contains the new setPolicies to be added, updated and deleted // Assumes that the dirty cache is locked (or equivalently, the ipsetmanager itself). // toAddSets: -// this function will loop through the dirty cache and adds non-existing sets to toAddSets +// +// this function will loop through the dirty cache and adds non-existing sets to toAddSets +// // toUpdateSets: -// this function will loop through the dirty cache and adds existing sets in HNS to toUpdateSets -// this function will update all existing sets in HNS with their latest goal state irrespective of any change to the object +// +// this function will loop through the dirty cache and adds existing sets in HNS to toUpdateSets +// this function will update all existing sets in HNS with their latest goal state irrespective of any change to the object +// // toDeleteSets: -// this function will loop through the dirty delete cache and adds existing set obj in HNS to toDeleteSets +// +// this function will loop through the dirty delete cache and adds existing set obj in HNS to toDeleteSets func (iMgr *IPSetManager) calculateNewSetPolicies(networkPolicies []hcn.NetworkPolicy) (*networkPolicyBuilder, error) { setPolicyBuilder := &networkPolicyBuilder{ toAddSets: map[string]*hcn.SetPolicySetting{}, @@ -307,7 +312,7 @@ func (iMgr *IPSetManager) getHCnNetwork() (*hcn.HostComputeNetwork, error) { if iMgr.iMgrCfg.NetworkName == "" { iMgr.iMgrCfg.NetworkName = util.AzureNetworkName } - if iMgr.iMgrCfg.NetworkName != util.AzureNetworkName { + if iMgr.iMgrCfg.NetworkName != util.AzureNetworkName && iMgr.iMgrCfg.NetworkName != util.CalicoNetworkName { return nil, errUnsupportedNetwork } network, err := iMgr.ioShim.Hns.GetNetworkByName(iMgr.iMgrCfg.NetworkName) diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_windows_test.go b/npm/pkg/dataplane/ipsets/ipsetmanager_windows_test.go index cd929293c9..959e36cf7e 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_windows_test.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_windows_test.go @@ -68,7 +68,7 @@ func TestGetIPsFromSelectorIPSets(t *testing.T) { } func TestAddToSetWindows(t *testing.T) { - hns := GetHNSFake(t) + hns := GetHNSFake(t, "azure") io := common.NewMockIOShimWithFakeHNS(hns) iMgr := NewIPSetManager(applyAlwaysCfg, io) @@ -102,7 +102,7 @@ func TestDestroyNPMIPSets(t *testing.T) { // create all possible SetTypes // FIXME because this can flake, commenting this out until we refactor with new windows testing framework // func TestApplyCreationsAndAdds(t *testing.T) { -// hns := GetHNSFake(t) +// hns := GetHNSFake(t, "azure") // io := common.NewMockIOShimWithFakeHNS(hns) // iMgr := NewIPSetManager(applyAlwaysCfg, io) @@ -182,7 +182,7 @@ func TestDestroyNPMIPSets(t *testing.T) { // } func TestApplyDeletions(t *testing.T) { - hns := GetHNSFake(t) + hns := GetHNSFake(t, "azure") io := common.NewMockIOShimWithFakeHNS(hns) iMgr := NewIPSetManager(applyAlwaysCfg, io) @@ -230,7 +230,7 @@ func TestApplyDeletions(t *testing.T) { // TODO test that a reconcile list is updated func TestFailureOnCreation(t *testing.T) { - hns := GetHNSFake(t) + hns := GetHNSFake(t, "azure") io := common.NewMockIOShimWithFakeHNS(hns) iMgr := NewIPSetManager(applyAlwaysCfg, io) @@ -268,7 +268,7 @@ func TestFailureOnCreation(t *testing.T) { // FIXME commenting this out until we refactor with new windows testing framework // func TestFailureOnAddToList(t *testing.T) { // // This exact scenario wouldn't occur. This error happens when the cache is out of date with the kernel. -// hns := GetHNSFake(t) +// hns := GetHNSFake(t, "azure") // io := common.NewMockIOShimWithFakeHNS(hns) // iMgr := NewIPSetManager(applyAlwaysCfg, io) @@ -319,7 +319,7 @@ func TestFailureOnCreation(t *testing.T) { // TODO test that a reconcile list is updated func TestFailureOnFlush(t *testing.T) { // This exact scenario wouldn't occur. This error happens when the cache is out of date with the kernel. - hns := GetHNSFake(t) + hns := GetHNSFake(t, "azure") io := common.NewMockIOShimWithFakeHNS(hns) iMgr := NewIPSetManager(applyAlwaysCfg, io) @@ -348,7 +348,7 @@ func TestFailureOnFlush(t *testing.T) { // TODO test that a reconcile list is updated func TestFailureOnDeletion(t *testing.T) { - hns := GetHNSFake(t) + hns := GetHNSFake(t, "azure") io := common.NewMockIOShimWithFakeHNS(hns) iMgr := NewIPSetManager(applyAlwaysCfg, io) diff --git a/npm/pkg/dataplane/ipsets/testutils_windows.go b/npm/pkg/dataplane/ipsets/testutils_windows.go index f5c780f981..f9235beffe 100644 --- a/npm/pkg/dataplane/ipsets/testutils_windows.go +++ b/npm/pkg/dataplane/ipsets/testutils_windows.go @@ -10,11 +10,11 @@ import ( "github.com/stretchr/testify/require" ) -func GetHNSFake(t *testing.T) *hnswrapper.Hnsv2wrapperFake { +func GetHNSFake(t *testing.T, networkName string) *hnswrapper.Hnsv2wrapperFake { hns := hnswrapper.NewHnsv2wrapperFake() network := &hcn.HostComputeNetwork{ Id: common.FakeHNSNetworkID, - Name: "azure", + Name: networkName, } _, err := hns.CreateNetwork(network) diff --git a/npm/pkg/dataplane/policies/policymanager_windows.go b/npm/pkg/dataplane/policies/policymanager_windows.go index dd3cb62428..6f2c61d177 100644 --- a/npm/pkg/dataplane/policies/policymanager_windows.go +++ b/npm/pkg/dataplane/policies/policymanager_windows.go @@ -12,6 +12,12 @@ import ( "k8s.io/klog" ) +const ( + // for lints + priority200 = 200 + priority65499 = 65499 +) + var ( ErrFailedMarshalACLSettings = errors.New("failed to marshal ACL settings") ErrFailedUnMarshalACLSettings = errors.New("failed to unmarshal ACL settings") @@ -19,6 +25,67 @@ var ( removeOnlyGivenPolicy shouldResetAllACLs = false ) +// baseACLsForCalicoCNI is a list of base ACLs that are required for connectivity on Calico CNI. +// Note: these ACLs have an ID with only one dash after the prefix so that they can't conflict with ACLs of a policy (see aclPolicyID()). +var baseACLsForCalicoCNI = []*NPMACLPolSettings{ + { + Id: fmt.Sprintf("%s-baseazurewireserver", policyIDPrefix), + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + Priority: priority200, + RemoteAddresses: "168.63.129.16/32", + RemotePorts: "80", + Protocols: "6", + RuleType: hcn.RuleTypeSwitch, + }, + { + Id: fmt.Sprintf("%s-baseallowinswitch", policyIDPrefix), + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + Priority: priority65499, + }, + { + Id: fmt.Sprintf("%s-baseallowoutswitch", policyIDPrefix), + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + Priority: priority65499, + }, + { + Id: fmt.Sprintf("%s-baseallowinhost", policyIDPrefix), + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + // unsupported for NPMACLPolSettings + // InternalPort: 0, + LocalAddresses: "", + // unsupported for NPMACLPolSettings (note no 's') + // LocalPort: "0", + Priority: 0, + // unsupported for NPMACLPolSettings (note no 's') + // Protocol: "256", + RemoteAddresses: "", + // unsupported for NPMACLPolSettings (note no 's') + // RemotePort: "0", + RuleType: hcn.RuleTypeHost, + }, + { + Id: fmt.Sprintf("%s-baseallowouthost", policyIDPrefix), + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + // unsupported for NPMACLPolSettings + // InternalPort: 0, + LocalAddresses: "", + // unsupported for NPMACLPolSettings (note no 's') + // LocalPort: "0", + Priority: 0, + // unsupported for NPMACLPolSettings (note no 's') + // Protocol: "256", + RemoteAddresses: "", + // unsupported for NPMACLPolSettings (note no 's') + // RemotePort: "0", + RuleType: hcn.RuleTypeHost, + }, +} + type staleChains struct{} // unused in Windows type shouldResetAllACLs bool @@ -57,6 +124,19 @@ func (pMgr *PolicyManager) reconcile() { // not implemented } +// AddBaseACLsForCalicoCNI attempts to add base ACLs for Calico CNI. +func (pMgr *PolicyManager) AddBaseACLsForCalicoCNI(epID string) { + epPolicyRequest, err := getEPPolicyReqFromACLSettings(baseACLsForCalicoCNI) + if err != nil { + klog.Errorf("failed to get policy request for base ACLs for Calico CNI. endpoint: %s. err: %v", epID, err) + return + } + + if err := pMgr.applyPoliciesToEndpointID(epID, epPolicyRequest); err != nil { + klog.Errorf("failed to apply base ACLs for Calico CNI. endpoint: %s. err: %v", epID, err) + } +} + // addPolicy will add the policy for each specified endpoint if the policy doesn't exist on the endpoint yet, // and will add the endpoint to the PodEndpoints of the policy if successful. // addPolicy may modify the endpointList input. diff --git a/npm/pkg/dataplane/policies/policymanager_windows_test.go b/npm/pkg/dataplane/policies/policymanager_windows_test.go index 812c3116e7..65b3f74f1d 100644 --- a/npm/pkg/dataplane/policies/policymanager_windows_test.go +++ b/npm/pkg/dataplane/policies/policymanager_windows_test.go @@ -183,7 +183,7 @@ func TestRemovePoliciesEndpointNotFound(t *testing.T) { // Helper functions for UTS func getPMgr(t *testing.T) (*PolicyManager, *hnswrapper.Hnsv2wrapperFake) { - hns := ipsets.GetHNSFake(t) + hns := ipsets.GetHNSFake(t, "azure") io := common.NewMockIOShimWithFakeHNS(hns) dptestutils.AddIPsToHNS(t, hns, endPointIDList) diff --git a/npm/util/const.go b/npm/util/const.go index f94ecf4935..ff10cc53d6 100644 --- a/npm/util/const.go +++ b/npm/util/const.go @@ -230,8 +230,12 @@ const ( ErrorValue float64 = 1 ) -// AzureNetworkName is the default network Azure CNI creates -const AzureNetworkName = "azure" +const ( + // AzureNetworkName is the default network Azure CNI creates + AzureNetworkName = "azure" + // CalicoNetworkName is the default network Calico CNI creates + CalicoNetworkName = "Calico" +) // These ID represents where did the error log generate from. // It's for better query purpose. In Kusto these value are used in diff --git a/npm/windows.Dockerfile b/npm/windows.Dockerfile index 1f9a3b3d23..c2c4fa006c 100644 --- a/npm/windows.Dockerfile +++ b/npm/windows.Dockerfile @@ -10,5 +10,6 @@ RUN GOOS=windows CGO_ENABLED=1 go build -v -o /usr/local/bin/azure-npm.exe -ldfl FROM mcr.microsoft.com/windows/servercore:${OS_VERSION} COPY --from=builder /usr/local/src/npm/examples/windows/kubeconfigtemplate.yaml kubeconfigtemplate.yaml COPY --from=builder /usr/local/src/npm/examples/windows/setkubeconfigpath.ps1 setkubeconfigpath.ps1 +COPY --from=builder /usr/local/src/npm/examples/windows/setkubeconfigpath-capz.ps1 setkubeconfigpath-capz.ps1 COPY --from=builder /usr/local/bin/azure-npm.exe azure-npm.exe CMD ["azure-npm.exe", "start" "--kubeconfig=.\\kubeconfig"]