diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml index 456dfdfaa1..2994d9972b 100644 --- a/.pipelines/e2e-job-template.yaml +++ b/.pipelines/e2e-job-template.yaml @@ -9,19 +9,22 @@ parameters: jobs: - job: ${{ parameters.name }} dependsOn: unit_tests + timeoutInMinutes: 90 pool: name: Networking-ContainerNetworking demands: agent.os -equals Linux container: image: ${{ parameters.pipelineBuildImage }} variables: - GOPATH: "$(System.DefaultWorkingDirectory)/gopath" + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path GOBIN: "$(GOPATH)/bin" # Go binaries path modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine" acnPath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ] CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ] - + CLEANUP_ON_EXIT: true + CLEANUP_IF_FAIL: true + AKS_ENGINE_VERSION: v0.41.3 steps: - template: e2e-step-template.yaml parameters: diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml index ff27d0013d..904a9100b5 100644 --- a/.pipelines/e2e-step-template.yaml +++ b/.pipelines/e2e-step-template.yaml @@ -29,11 +29,15 @@ steps: - bash: | ls -lah - export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"' + export CNI_URL=https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }} export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }} echo CNI type is $CNI_TYPE - sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}' - sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}' + #sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}' + # sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}' + cat '${{ parameters.clusterDefinition }}' | jq --arg cnikey $CNI_TYPE --arg cniurl $CNI_URL '.properties.orchestratorProfile.kubernetesConfig[$cnikey]= $cniurl' > '${{ parameters.clusterDefinition }}'.tmp # + # sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}' + #sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}' + cat '${{ parameters.clusterDefinition }}'.tmp | jq --arg tag $(Tag) '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = $tag' > '${{ parameters.clusterDefinition }}' echo "Running E2E tests against a cluster built with the following API model:" cat '${{ parameters.clusterDefinition }}' cp ${{ parameters.clusterDefinition }} clusterDefinition.json @@ -61,6 +65,10 @@ steps: mkdir -p $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }} cp -r _output/k*/kubeconfig/kubeconfig.$REGIONS.json $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }} cp -r _output/kubernetes-*-ssh $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }} + echo "Deleting work directory" + sudo rm -rf ./ + echo "Deleting file in GOPATH" + sudo rm -rf '$(GOPATH)' name: DeployAKSEngine displayName: Deploy AKS-Engine workingDirectory: "$(modulePath)" diff --git a/cni/telemetry/service/telemetrymain.go b/cni/telemetry/service/telemetrymain.go index 7d93bf12ad..07cc5e08a3 100644 --- a/cni/telemetry/service/telemetrymain.go +++ b/cni/telemetry/service/telemetrymain.go @@ -123,6 +123,10 @@ func main() { log.Logf("read config returned %+v", config) + // Cleaning up orphan socket if present + tbtemp := telemetry.NewTelemetryBuffer("") + tbtemp.Cleanup(telemetry.FdName) + for { tb = telemetry.NewTelemetryBuffer("") diff --git a/npm/nwpolicy.go b/npm/nwpolicy.go index b2f8aa912e..ae3f3bf630 100644 --- a/npm/nwpolicy.go +++ b/npm/nwpolicy.go @@ -3,8 +3,8 @@ package npm import ( - "github.com/Azure/azure-container-networking/npm/iptm" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/npm/iptm" "github.com/Azure/azure-container-networking/npm/util" networkingv1 "k8s.io/api/networking/v1" ) @@ -33,7 +33,7 @@ func (npMgr *NetworkPolicyManager) AddNetworkPolicy(npObj *networkingv1.NetworkP ns *namespace ) - npNs, npName := "ns-" + npObj.ObjectMeta.Namespace, npObj.ObjectMeta.Name + npNs, npName := "ns-"+npObj.ObjectMeta.Namespace, npObj.ObjectMeta.Name log.Printf("NETWORK POLICY CREATING: %v", npObj) var exists bool @@ -148,7 +148,7 @@ func (npMgr *NetworkPolicyManager) DeleteNetworkPolicy(npObj *networkingv1.Netwo ns *namespace ) - npNs, npName := "ns-" + npObj.ObjectMeta.Namespace, npObj.ObjectMeta.Name + npNs, npName := "ns-"+npObj.ObjectMeta.Namespace, npObj.ObjectMeta.Name log.Printf("NETWORK POLICY DELETING: %v", npObj) var exists bool @@ -178,14 +178,14 @@ func (npMgr *NetworkPolicyManager) DeleteNetworkPolicy(npObj *networkingv1.Netwo if err != nil { log.Printf("Error deducting policy %s from %s", npName, oldPolicy.ObjectMeta.Name) } - + if deductedPolicy == nil { delete(ns.processedNpMap, hashedSelector) } else { ns.processedNpMap[hashedSelector] = deductedPolicy } } - + if npMgr.canCleanUpNpmChains() { if err = iptMgr.UninitNpmChains(); err != nil { log.Errorf("Error: failed to uninitialize azure-npm chains.") diff --git a/npm/plugin/main.go b/npm/plugin/main.go index 501b271354..1c6f8c0ebe 100644 --- a/npm/plugin/main.go +++ b/npm/plugin/main.go @@ -62,8 +62,6 @@ func main() { go npMgr.SendNpmTelemetry() - time.Sleep(time.Second * waitForTelemetryInSeconds) - if err = npMgr.Start(wait.NeverStop); err != nil { log.Logf("npm failed with error %v.", err) panic(err.Error) diff --git a/npm/pod.go b/npm/pod.go index 427e01d44a..5e30c34abc 100644 --- a/npm/pod.go +++ b/npm/pod.go @@ -10,10 +10,7 @@ import ( ) func isValidPod(podObj *corev1.Pod) bool { - return podObj.Status.Phase != corev1.PodPhase(util.KubePodStatusFailedFlag) && - podObj.Status.Phase != corev1.PodPhase(util.KubePodStatusSucceededFlag) && - podObj.Status.Phase != corev1.PodPhase(util.KubePodStatusUnknownFlag) && - len(podObj.Status.PodIP) > 0 + return len(podObj.Status.PodIP) > 0 } func isSystemPod(podObj *corev1.Pod) bool { @@ -142,7 +139,7 @@ func (npMgr *NetworkPolicyManager) DeletePod(podObj *corev1.Pod) error { if err = ipsMgr.DeleteFromSet(podLabelKey, podIP); err != nil { log.Errorf("Error: failed to delete pod from label ipset.") return err - } + } label := podLabelKey + ":" + podLabelVal log.Printf("Deleting pod %s from ipset %s", podIP, label) diff --git a/npm/translatePolicy.go b/npm/translatePolicy.go index d8a86e84c4..84c7a4308f 100644 --- a/npm/translatePolicy.go +++ b/npm/translatePolicy.go @@ -964,9 +964,9 @@ func getAllowKubeSystemEntries(ns string, targetSelector metav1.LabelSelector) [ // 3. iptables entries generated from the input network policy object. func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []*iptm.IptEntry) { var ( - resultSets []string - resultLists []string - entries []*iptm.IptEntry + resultSets []string + resultLists []string + entries []*iptm.IptEntry hasIngress, hasEgress bool ) @@ -1010,7 +1010,15 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []* resultSets = append(resultSets, ingressSets...) resultLists = append(resultLists, ingressLists...) entries = append(entries, ingressEntries...) - hasIngress = true + + if npObj.Spec.Ingress != nil && + len(npObj.Spec.Ingress) == 1 && + len(npObj.Spec.Ingress[0].Ports) == 0 && + len(npObj.Spec.Ingress[0].From) == 0 { + hasIngress = false + } else { + hasIngress = true + } } if ptype == networkingv1.PolicyTypeEgress { @@ -1018,12 +1026,20 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []* resultSets = append(resultSets, egressSets...) resultLists = append(resultLists, egressLists...) entries = append(entries, egressEntries...) - hasEgress = true + + if npObj.Spec.Egress != nil && + len(npObj.Spec.Egress) == 1 && + len(npObj.Spec.Egress[0].Ports) == 0 && + len(npObj.Spec.Egress[0].To) == 0 { + hasEgress = false + } else { + hasEgress = true + } } } entries = append(entries, getDefaultDropEntries(npNs, npObj.Spec.PodSelector, hasIngress, hasEgress)...) - + log.Printf("Translating Policy: %+v", npObj) resultSets, resultLists = util.UniqueStrSlice(resultSets), util.UniqueStrSlice(resultLists) return resultSets, resultLists, entries diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go index 71f2bf14a9..c8aa4f5e93 100644 --- a/npm/translatePolicy_test.go +++ b/npm/translatePolicy_test.go @@ -1271,7 +1271,7 @@ func TestTranslatePolicy(t *testing.T) { }, } expectedIptEntries = append(expectedIptEntries, nonKubeSystemEntries...) - expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("testnamespace", targetSelector, true, false)...) + expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("testnamespace", targetSelector, false, false)...) if !reflect.DeepEqual(iptEntries, expectedIptEntries) { t.Errorf("translatedPolicy failed @ ALLOW-all-TO-app:frontend-FROM-all-namespaces-policy policy comparison") marshalledIptEntries, _ := json.Marshal(iptEntries) @@ -2018,7 +2018,7 @@ func TestTranslatePolicy(t *testing.T) { } expectedIptEntries = append(expectedIptEntries, nonKubeSystemEntries...) - expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("dangerous", targetSelector, true, false)...) + expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("dangerous", targetSelector, false, false)...) if !reflect.DeepEqual(iptEntries, expectedIptEntries) { t.Errorf("translatedPolicy failed @ ALLOW-ALL-TO-app:backdoor-policy policy comparison") marshalledIptEntries, _ := json.Marshal(iptEntries) @@ -2325,7 +2325,6 @@ func TestTranslatePolicy(t *testing.T) { PolicyTypes: []networkingv1.PolicyType{ networkingv1.PolicyTypeEgress, }, - Egress: []networkingv1.NetworkPolicyEgressRule{}, }, } @@ -2361,6 +2360,90 @@ func TestTranslatePolicy(t *testing.T) { t.Errorf("expectedIptEntries: %s", marshalledExpectedIptEntries) } + targetSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "backend", + }, + } + + ////// + /// This policy tests the case where pods should have unlimited egress traffic + ////// + allowAllEgress := &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ALLOW-all-FROM-app:backend-policy", + Namespace: "testnamespace", + }, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: targetSelector, + PolicyTypes: []networkingv1.PolicyType{ + networkingv1.PolicyTypeEgress, + }, + Egress: []networkingv1.NetworkPolicyEgressRule{networkingv1.NetworkPolicyEgressRule{}}, + }, + } + + sets, lists, iptEntries = translatePolicy(allowAllEgress) + + expectedSets = []string{ + "app:backend", + } + if !reflect.DeepEqual(sets, expectedSets) { + t.Errorf("translatedPolicy failed @ ALLOW-all-FROM-app:backend-policy sets comparison") + t.Errorf("sets: %v", sets) + t.Errorf("expectedSets: %v", expectedSets) + } + + expectedLists = []string{ + util.KubeAllNamespacesFlag, + } + if !reflect.DeepEqual(lists, expectedLists) { + t.Errorf("translatedPolicy failed @ ALLOW-all-FROM-app:backend-policy lists comparison") + t.Errorf("lists: %v", lists) + t.Errorf("expectedLists: %v", expectedLists) + } + + expectedIptEntries = []*iptm.IptEntry{} + expectedIptEntries = append( + expectedIptEntries, + getAllowKubeSystemEntries("testnamespace", targetSelector)..., + ) + + nonKubeSystemEntries = []*iptm.IptEntry{ + &iptm.IptEntry{ + Chain: util.IptablesAzureEgressPortChain, + Specs: []string{ + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName("app:backend"), + util.IptablesSrcFlag, + util.IptablesModuleFlag, + util.IptablesSetModuleFlag, + util.IptablesMatchSetFlag, + util.GetHashedName(util.KubeAllNamespacesFlag), + util.IptablesDstFlag, + util.IptablesJumpFlag, + util.IptablesAccept, + util.IptablesModuleFlag, + util.IptablesCommentModuleFlag, + util.IptablesCommentFlag, + "ALLOW-ALL-FROM-app:backend-TO-" + + util.KubeAllNamespacesFlag, + }, + }, + } + expectedIptEntries = append(expectedIptEntries, nonKubeSystemEntries...) + // has egress, but empty map means allow all + expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("testnamespace", targetSelector, false, false)...) + if !reflect.DeepEqual(iptEntries, expectedIptEntries) { + t.Errorf("translatedPolicy failed @ ALLOW-all-FROM-app:backend-policy policy comparison") + marshalledIptEntries, _ := json.Marshal(iptEntries) + marshalledExpectedIptEntries, _ := json.Marshal(expectedIptEntries) + t.Errorf("iptEntries: %s", marshalledIptEntries) + t.Errorf("expectedIptEntries: %s", marshalledExpectedIptEntries) + } + targetSelector = metav1.LabelSelector{} denyAllFromNsUnsafePolicy := &networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ @@ -2835,8 +2918,8 @@ func TestAllowPrecedenceOverDeny(t *testing.T) { } denyAllPolicy := &networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ - Name: "default-deny", - Namespace: "default", + Name: "default-deny", + Namespace: "default", }, Spec: networkingv1.NetworkPolicySpec{ PodSelector: targetSelector,