From 99d4ef41e6a7e8327f8aa3b7cbbd81e8055d9543 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Thu, 7 Mar 2019 16:54:49 -0800 Subject: [PATCH 01/15] update dockerfile --- npm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npm/Dockerfile b/npm/Dockerfile index b04bb47da5..1b33d8c416 100644 --- a/npm/Dockerfile +++ b/npm/Dockerfile @@ -1,5 +1,5 @@ # Use a minimal image as a parent image -FROM ubuntu:16.04 +FROM ubuntu:18.10 ARG NPM_BUILD_DIR # Install dependencies. From 66821dbffd5603e406baac423fd6ed7162adb161 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Mon, 11 Mar 2019 14:11:42 -0700 Subject: [PATCH 02/15] change to src flag --- npm/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npm/parse.go b/npm/parse.go index 83ecdb708b..977256c6a8 100644 --- a/npm/parse.go +++ b/npm/parse.go @@ -573,7 +573,7 @@ func parseEgress(ns string, targetSets []string, rules []networkingv1.NetworkPol util.IptablesSetFlag, util.IptablesMatchSetFlag, hashedTargetSetName, - util.IptablesDstFlag, + util.IptablesSrcFlag, util.IptablesJumpFlag, util.IptablesAzureEgressToNsChain, }, From 45569269f080877209d3a16b2ef2f49e6f1d3c3a Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Mon, 11 Mar 2019 15:58:36 -0700 Subject: [PATCH 03/15] make azure-npm wait for iptables lock --- npm/iptm/iptm.go | 4 ++-- npm/util/const.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/npm/iptm/iptm.go b/npm/iptm/iptm.go index 8344f39c6e..32fd561beb 100644 --- a/npm/iptm/iptm.go +++ b/npm/iptm/iptm.go @@ -320,7 +320,7 @@ func (iptMgr *IptablesManager) Delete(entry *IptEntry) error { // Run execute an iptables command to update iptables. func (iptMgr *IptablesManager) Run(entry *IptEntry) (int, error) { cmdName := util.Iptables - cmdArgs := append([]string{iptMgr.OperationFlag, entry.Chain}, entry.Specs...) + cmdArgs := append([]string{util.IptablesWaitFlag, iptMgr.OperationFlag, entry.Chain}, entry.Specs...) cmdOut, err := exec.Command(cmdName, cmdArgs...).Output() log.Printf("%s\n", string(cmdOut)) @@ -369,7 +369,7 @@ func (iptMgr *IptablesManager) Restore(configFile string) error { } // open the config file for reading - f, err := os.Open(configFile) + f, err := os.OpenFile(configFile, 0755, os.ModeExclusive) if err != nil { log.Printf("Error opening file: %s.", configFile) return err diff --git a/npm/util/const.go b/npm/util/const.go index 02b2005c6b..1024f257d5 100644 --- a/npm/util/const.go +++ b/npm/util/const.go @@ -31,6 +31,7 @@ const ( IptablesCheckFlag string = "-C" IptablesDestroyFlag string = "-X" IptablesJumpFlag string = "-j" + IptablesWaitFlag string = "-w" IptablesAccept string = "ACCEPT" IptablesReject string = "REJECT" IptablesDrop string = "DROP" From ea245d3cbe0693b3b601a31e28c1eb9bfb3e2e2c Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Mon, 11 Mar 2019 17:05:54 -0700 Subject: [PATCH 04/15] get clusterState by querying apiServer every 5 mins --- npm/namespace.go | 4 ---- npm/npm.go | 22 +++++++++++++++++++++- npm/nwpolicy.go | 4 ---- npm/pod.go | 4 ---- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/npm/namespace.go b/npm/namespace.go index 9bb855ff20..9ce7548811 100644 --- a/npm/namespace.go +++ b/npm/namespace.go @@ -121,8 +121,6 @@ func (npMgr *NetworkPolicyManager) AddNamespace(nsObj *corev1.Namespace) error { } npMgr.nsMap[nsName] = ns - npMgr.clusterState.NsCount++ - return nil } @@ -203,7 +201,5 @@ func (npMgr *NetworkPolicyManager) DeleteNamespace(nsObj *corev1.Namespace) erro delete(npMgr.nsMap, nsName) - npMgr.clusterState.NsCount-- - return nil } diff --git a/npm/npm.go b/npm/npm.go index 32c31ba8f8..e3eccaff41 100644 --- a/npm/npm.go +++ b/npm/npm.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-container-networking/telemetry" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" @@ -49,6 +50,25 @@ type NetworkPolicyManager struct { // GetClusterState returns current cluster state. func (npMgr *NetworkPolicyManager) GetClusterState() telemetry.ClusterState { + pods, err := npMgr.clientset.CoreV1().Pods("").List(metav1.ListOptions{}) + if err != nil { + log.Printf("Error Listing pods in GetClusterState") + } + + namespaces, err := npMgr.clientset.CoreV1().Namespaces().List(metav1.ListOptions{}) + if err != nil { + log.Printf("Error Listing namespaces in GetClusterState") + } + + networkpolicies, err := npMgr.clientset.NetworkingV1().NetworkPolicies("").List(metav1.ListOptions{}) + if err != nil { + log.Printf("Error Listing networkpolicies in GetClusterState") + } + + npMgr.clusterState.PodCount = len(pods.Items) + npMgr.clusterState.NsCount = len(namespaces.Items) + npMgr.clusterState.NwPolicyCount = len(networkpolicies.Items) + return npMgr.clusterState } @@ -108,7 +128,7 @@ func (npMgr *NetworkPolicyManager) RunReportManager() { log.Printf("Error sending NPM telemetry report") } - time.Sleep(1 * time.Minute) + time.Sleep(5 * time.Minute) } } diff --git a/npm/nwpolicy.go b/npm/nwpolicy.go index 2568bb6838..77b009b75e 100644 --- a/npm/nwpolicy.go +++ b/npm/nwpolicy.go @@ -72,8 +72,6 @@ func (npMgr *NetworkPolicyManager) AddNetworkPolicy(npObj *networkingv1.NetworkP allNs.npMap[npName] = npObj - npMgr.clusterState.NwPolicyCount++ - ns, err := newNs(npNs) if err != nil { log.Printf("Error creating namespace %s\n", npNs) @@ -141,8 +139,6 @@ func (npMgr *NetworkPolicyManager) DeleteNetworkPolicy(npObj *networkingv1.Netwo delete(allNs.npMap, npName) - npMgr.clusterState.NwPolicyCount-- - if len(allNs.npMap) == 0 { if err = iptMgr.UninitNpmChains(); err != nil { log.Printf("Error uninitialize azure-npm chains.\n") diff --git a/npm/pod.go b/npm/pod.go index 5dba4e8882..5cb4f67f85 100644 --- a/npm/pod.go +++ b/npm/pod.go @@ -72,8 +72,6 @@ func (npMgr *NetworkPolicyManager) AddPod(podObj *corev1.Pod) error { labelKeys = append(labelKeys, labelKey) } - npMgr.clusterState.PodCount++ - ns, err := newNs(podNs) if err != nil { log.Printf("Error creating namespace %s\n", podNs) @@ -172,7 +170,5 @@ func (npMgr *NetworkPolicyManager) DeletePod(podObj *corev1.Pod) error { } } - npMgr.clusterState.PodCount-- - return nil } From 6012b6ccc0a8741a30ba3f8149740ab6f8fd41e6 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Tue, 12 Mar 2019 15:58:24 -0700 Subject: [PATCH 05/15] disable telemetry for npm --- npm/namespace_test.go | 9 ++++--- npm/npm.go | 55 ++++++++++++++++++++++++++++++++++++------- npm/nwpolicy_test.go | 9 ++++--- npm/plugin/main.go | 2 ++ npm/pod_test.go | 9 ++++--- 5 files changed, 67 insertions(+), 17 deletions(-) diff --git a/npm/namespace_test.go b/npm/namespace_test.go index bc466cdb97..9cc69176dd 100644 --- a/npm/namespace_test.go +++ b/npm/namespace_test.go @@ -45,7 +45,8 @@ func TestAllNsList(t *testing.T) { func TestAddNamespace(t *testing.T) { npMgr := &NetworkPolicyManager{ - nsMap: make(map[string]*namespace), + nsMap: make(map[string]*namespace), + TelemetryEnabled: false, reportManager: &telemetry.ReportManager{ HostNetAgentURL: hostNetAgentURLForNpm, ContentType: contentType, @@ -86,7 +87,8 @@ func TestAddNamespace(t *testing.T) { func TestUpdateNamespace(t *testing.T) { npMgr := &NetworkPolicyManager{ - nsMap: make(map[string]*namespace), + nsMap: make(map[string]*namespace), + TelemetryEnabled: false, reportManager: &telemetry.ReportManager{ HostNetAgentURL: hostNetAgentURLForNpm, ContentType: contentType, @@ -140,7 +142,8 @@ func TestUpdateNamespace(t *testing.T) { func TestDeleteNamespace(t *testing.T) { npMgr := &NetworkPolicyManager{ - nsMap: make(map[string]*namespace), + nsMap: make(map[string]*namespace), + TelemetryEnabled: false, reportManager: &telemetry.ReportManager{ HostNetAgentURL: hostNetAgentURLForNpm, ContentType: contentType, diff --git a/npm/npm.go b/npm/npm.go index e3eccaff41..aa7819eb15 100644 --- a/npm/npm.go +++ b/npm/npm.go @@ -23,9 +23,10 @@ import ( "k8s.io/client-go/tools/cache" ) -var ( - hostNetAgentURLForNpm = "http://168.63.129.16/machine/plugins?comp=netagent&type=npmreport" - contentType = "application/json" +const ( + hostNetAgentURLForNpm = "http://168.63.129.16/machine/plugins?comp=netagent&type=npmreport" + contentType = "application/json" + retryWaitTimeInSeconds = 60 ) // NetworkPolicyManager contains informers for pod, namespace and networkpolicy. @@ -45,7 +46,8 @@ type NetworkPolicyManager struct { clusterState telemetry.ClusterState reportManager *telemetry.ReportManager - serverVersion *version.Info + serverVersion *version.Info + TelemetryEnabled bool } // GetClusterState returns current cluster state. @@ -75,6 +77,13 @@ func (npMgr *NetworkPolicyManager) GetClusterState() telemetry.ClusterState { // UpdateAndSendReport updates the npm report then send it. // This function should only be called when npMgr is locked. func (npMgr *NetworkPolicyManager) UpdateAndSendReport(err error, eventMsg string) error { + npMgr.Lock() + defer npMgr.Unlock() + + if !npMgr.TelemetryEnabled { + return nil + } + clusterState := npMgr.GetClusterState() v := reflect.ValueOf(npMgr.reportManager.Report).Elem().FieldByName("ClusterState") if v.CanSet() { @@ -89,7 +98,10 @@ func (npMgr *NetworkPolicyManager) UpdateAndSendReport(err error, eventMsg strin reflect.ValueOf(npMgr.reportManager.Report).Elem().FieldByName("EventMessage").SetString(err.Error()) } - return npMgr.reportManager.SendReport(nil) + var telemetryBuffer *telemetry.TelemetryBuffer + connectToTelemetryServer(telemetryBuffer) + + return npMgr.reportManager.SendReport(telemetryBuffer) } // Run starts shared informers and waits for the shared informer cache to sync. @@ -113,8 +125,33 @@ func (npMgr *NetworkPolicyManager) Run(stopCh <-chan struct{}) error { return nil } +func connectToTelemetryServer(telemetryBuffer *telemetry.TelemetryBuffer) { + for { + telemetryBuffer = telemetry.NewTelemetryBuffer("") + err := telemetryBuffer.StartServer() + if err == nil || telemetryBuffer.FdExists { + connErr := telemetryBuffer.Connect() + if connErr == nil { + break + } + + log.Printf("[NPM-Telemetry] Failed to establish telemetry manager connection.") + time.Sleep(time.Second * retryWaitTimeInSeconds) + } + } +} + // RunReportManager starts NPMReportManager and send telemetry periodically. func (npMgr *NetworkPolicyManager) RunReportManager() { + if npMgr.TelemetryEnabled { + return + } + + var telemetryBuffer *telemetry.TelemetryBuffer + connectToTelemetryServer(telemetryBuffer) + + go telemetryBuffer.BufferAndPushData(time.Duration(0)) + for { clusterState := npMgr.GetClusterState() v := reflect.ValueOf(npMgr.reportManager.Report).Elem().FieldByName("ClusterState") @@ -124,8 +161,9 @@ func (npMgr *NetworkPolicyManager) RunReportManager() { v.FieldByName("NwPolicyCount").SetInt(int64(clusterState.NwPolicyCount)) } - if err := npMgr.reportManager.SendReport(nil); err != nil { - log.Printf("Error sending NPM telemetry report") + if err := npMgr.reportManager.SendReport(telemetryBuffer); err != nil { + log.Printf("[NPM-Telemetry] Error sending NPM telemetry report") + connectToTelemetryServer(telemetryBuffer) } time.Sleep(5 * time.Minute) @@ -170,7 +208,8 @@ func NewNetworkPolicyManager(clientset *kubernetes.Clientset, informerFactory in ContentType: contentType, Report: &telemetry.NPMReport{}, }, - serverVersion: serverVersion, + serverVersion: serverVersion, + TelemetryEnabled: true, } clusterID := util.GetClusterID(npMgr.nodeName) diff --git a/npm/nwpolicy_test.go b/npm/nwpolicy_test.go index d17a1bbbae..1c01d8cc5b 100644 --- a/npm/nwpolicy_test.go +++ b/npm/nwpolicy_test.go @@ -18,7 +18,8 @@ import ( func TestAddNetworkPolicy(t *testing.T) { npMgr := &NetworkPolicyManager{ - nsMap: make(map[string]*namespace), + nsMap: make(map[string]*namespace), + TelemetryEnabled: false, reportManager: &telemetry.ReportManager{ HostNetAgentURL: hostNetAgentURLForNpm, ContentType: contentType, @@ -97,7 +98,8 @@ func TestAddNetworkPolicy(t *testing.T) { func TestUpdateNetworkPolicy(t *testing.T) { npMgr := &NetworkPolicyManager{ - nsMap: make(map[string]*namespace), + nsMap: make(map[string]*namespace), + TelemetryEnabled: false, reportManager: &telemetry.ReportManager{ HostNetAgentURL: hostNetAgentURLForNpm, ContentType: contentType, @@ -204,7 +206,8 @@ func TestUpdateNetworkPolicy(t *testing.T) { func TestDeleteNetworkPolicy(t *testing.T) { npMgr := &NetworkPolicyManager{ - nsMap: make(map[string]*namespace), + nsMap: make(map[string]*namespace), + TelemetryEnabled: false, reportManager: &telemetry.ReportManager{ HostNetAgentURL: hostNetAgentURLForNpm, ContentType: contentType, diff --git a/npm/plugin/main.go b/npm/plugin/main.go index 795eeac7fe..7c3cd9c16e 100644 --- a/npm/plugin/main.go +++ b/npm/plugin/main.go @@ -63,6 +63,8 @@ func main() { panic(err.Error) } + // Disable Azure-NPM telemetry for now since it might throttle wireserver. + npMgr.TelemetryEnabled = false go npMgr.RunReportManager() select {} diff --git a/npm/pod_test.go b/npm/pod_test.go index 351b459666..6330c1e4ce 100644 --- a/npm/pod_test.go +++ b/npm/pod_test.go @@ -37,7 +37,8 @@ func TestisSystemPod(t *testing.T) { func TestAddPod(t *testing.T) { npMgr := &NetworkPolicyManager{ - nsMap: make(map[string]*namespace), + nsMap: make(map[string]*namespace), + TelemetryEnabled: false, reportManager: &telemetry.ReportManager{ HostNetAgentURL: hostNetAgentURLForNpm, ContentType: contentType, @@ -81,7 +82,8 @@ func TestAddPod(t *testing.T) { func TestUpdatePod(t *testing.T) { npMgr := &NetworkPolicyManager{ - nsMap: make(map[string]*namespace), + nsMap: make(map[string]*namespace), + TelemetryEnabled: false, reportManager: &telemetry.ReportManager{ HostNetAgentURL: hostNetAgentURLForNpm, ContentType: contentType, @@ -143,7 +145,8 @@ func TestUpdatePod(t *testing.T) { func TestDeletePod(t *testing.T) { npMgr := &NetworkPolicyManager{ - nsMap: make(map[string]*namespace), + nsMap: make(map[string]*namespace), + TelemetryEnabled: false, reportManager: &telemetry.ReportManager{ HostNetAgentURL: hostNetAgentURLForNpm, ContentType: contentType, From a4ceee6cf44887be7c30e167d34907b82acac53c Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Wed, 13 Mar 2019 13:51:27 -0700 Subject: [PATCH 06/15] disable telemetry --- npm/iptm/iptm.go | 2 +- npm/npm.go | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/npm/iptm/iptm.go b/npm/iptm/iptm.go index 32fd561beb..85341f54fb 100644 --- a/npm/iptm/iptm.go +++ b/npm/iptm/iptm.go @@ -369,7 +369,7 @@ func (iptMgr *IptablesManager) Restore(configFile string) error { } // open the config file for reading - f, err := os.OpenFile(configFile, 0755, os.ModeExclusive) + f, err := os.Open(configFile) if err != nil { log.Printf("Error opening file: %s.", configFile) return err diff --git a/npm/npm.go b/npm/npm.go index aa7819eb15..93783e2f6a 100644 --- a/npm/npm.go +++ b/npm/npm.go @@ -77,9 +77,6 @@ func (npMgr *NetworkPolicyManager) GetClusterState() telemetry.ClusterState { // UpdateAndSendReport updates the npm report then send it. // This function should only be called when npMgr is locked. func (npMgr *NetworkPolicyManager) UpdateAndSendReport(err error, eventMsg string) error { - npMgr.Lock() - defer npMgr.Unlock() - if !npMgr.TelemetryEnabled { return nil } @@ -143,7 +140,7 @@ func connectToTelemetryServer(telemetryBuffer *telemetry.TelemetryBuffer) { // RunReportManager starts NPMReportManager and send telemetry periodically. func (npMgr *NetworkPolicyManager) RunReportManager() { - if npMgr.TelemetryEnabled { + if !npMgr.TelemetryEnabled { return } From e23e3f0812a29f6b38c67a0c0546a2d4590bdfea Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Wed, 13 Mar 2019 15:01:52 -0700 Subject: [PATCH 07/15] use iptables v1.8.2 --- Makefile | 2 ++ npm/Dockerfile | 7 ++++--- npm/iptm/iptm.go | 2 +- scripts/install-npm-dependencies.sh | 25 +++++++++++++++++++++++++ 4 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 scripts/install-npm-dependencies.sh diff --git a/Makefile b/Makefile index 0e1e284670..86078ee285 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,7 @@ CNI_IPAM_DIR = cni/ipam/plugin CNI_TELEMETRY_DIR = cni/telemetry/service CNS_DIR = cns/service NPM_DIR = npm/plugin +SCRIPTS_DIR = scripts OUTPUT_DIR = output BUILD_DIR = $(OUTPUT_DIR)/$(GOOS)_$(GOARCH) CNM_BUILD_DIR = $(BUILD_DIR)/cnm @@ -231,6 +232,7 @@ ifeq ($(GOOS),linux) -f npm/Dockerfile \ -t $(AZURE_NPM_IMAGE):$(AZURE_NPM_VERSION) \ --build-arg NPM_BUILD_DIR=$(NPM_BUILD_DIR) \ + --build-arg SCRIPTS_DIR=$(SCRIPTS_DIR) \ . docker save $(AZURE_NPM_IMAGE):$(AZURE_NPM_VERSION) | gzip -c > $(NPM_BUILD_DIR)/$(NPM_ARCHIVE_NAME) endif diff --git a/npm/Dockerfile b/npm/Dockerfile index 1b33d8c416..7d725bc291 100644 --- a/npm/Dockerfile +++ b/npm/Dockerfile @@ -1,11 +1,12 @@ # Use a minimal image as a parent image FROM ubuntu:18.10 ARG NPM_BUILD_DIR +ARG SCRIPTS_DIR # Install dependencies. -RUN apt-get update -RUN apt-get install -y iptables -RUN apt-get install -y ipset +COPY $SCRIPTS_DIR/install-npm-dependencies.sh . +RUN chmod +x ./install-npm-dependencies.sh +RUN ./install-npm-dependencies.sh > /dev/null # Install plugin. COPY $NPM_BUILD_DIR/azure-npm /usr/bin diff --git a/npm/iptm/iptm.go b/npm/iptm/iptm.go index 85341f54fb..e64db0a71b 100644 --- a/npm/iptm/iptm.go +++ b/npm/iptm/iptm.go @@ -376,7 +376,7 @@ func (iptMgr *IptablesManager) Restore(configFile string) error { } defer f.Close() - cmd := exec.Command(util.IptablesRestore) + cmd := exec.Command(util.IptablesRestore, util.IptablesWaitFlag) cmd.Stdin = f if err := cmd.Start(); err != nil { log.Printf("Error running iptables-restore.\n") diff --git a/scripts/install-npm-dependencies.sh b/scripts/install-npm-dependencies.sh new file mode 100644 index 0000000000..afb47916b7 --- /dev/null +++ b/scripts/install-npm-dependencies.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +IPTABLES_ZIP_URL=http://www.netfilter.org/projects/iptables/files/ +IPTABLES_ZIP_FILE_NAME=iptables-1.8.2 + +apt-get update +apt-get install -y ipset + +wget ${IPTABLES_ZIP_URL}${IPTABLES_ZIP_FILE_NAME}.tar.bz2 +tar xjf ${IPTABLES_ZIP_FILE_NAME}.tar.bz2 && cd ${IPTABLES_ZIP_FILE_NAME} +./configure --prefix=/usr \ + --sbindir=/sbin \ + --disable-nftables \ + --enable-libipq \ + --with-xtlibdir=/lib/xtables && make + +make install < /dev/null +ln -sfv ../../sbin/xtables-legacy-multi /usr/bin/iptables-xml +for file in ip4tc ip6tc ipq iptc xtables +do + mv -v /usr/lib/lib${file}.so.* /lib && + ln -sfv ../../lib/$(readlink /usr/lib/lib${file}.so) /usr/lib/lib${file}.so +done + +rm -rf ../${IPTABLES_ZIP_FILE_NAME}* \ No newline at end of file From d039f98a4cd3d82eb0ed50bae8034810dede87e8 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Wed, 13 Mar 2019 16:04:37 -0700 Subject: [PATCH 08/15] update dependency script --- scripts/install-npm-dependencies.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/install-npm-dependencies.sh b/scripts/install-npm-dependencies.sh index afb47916b7..5905cf3c1a 100644 --- a/scripts/install-npm-dependencies.sh +++ b/scripts/install-npm-dependencies.sh @@ -5,6 +5,9 @@ IPTABLES_ZIP_FILE_NAME=iptables-1.8.2 apt-get update apt-get install -y ipset +apt-get install -y wget +apt-get install -y make +apt-get install -y gcc wget ${IPTABLES_ZIP_URL}${IPTABLES_ZIP_FILE_NAME}.tar.bz2 tar xjf ${IPTABLES_ZIP_FILE_NAME}.tar.bz2 && cd ${IPTABLES_ZIP_FILE_NAME} @@ -22,4 +25,8 @@ do ln -sfv ../../lib/$(readlink /usr/lib/lib${file}.so) /usr/lib/lib${file}.so done -rm -rf ../${IPTABLES_ZIP_FILE_NAME}* \ No newline at end of file +rm -rf ../${IPTABLES_ZIP_FILE_NAME}* + +apt-get purge -y wget +apt-get purge -y make +apt-get purge -y gcc \ No newline at end of file From 49047b99677eead47efd642d15183470a602efd5 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Wed, 13 Mar 2019 16:44:50 -0700 Subject: [PATCH 09/15] update npm script --- scripts/install-npm-dependencies.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/install-npm-dependencies.sh b/scripts/install-npm-dependencies.sh index 5905cf3c1a..dd9086ddf0 100644 --- a/scripts/install-npm-dependencies.sh +++ b/scripts/install-npm-dependencies.sh @@ -4,10 +4,10 @@ IPTABLES_ZIP_URL=http://www.netfilter.org/projects/iptables/files/ IPTABLES_ZIP_FILE_NAME=iptables-1.8.2 apt-get update -apt-get install -y ipset -apt-get install -y wget -apt-get install -y make -apt-get install -y gcc +apt-get install -y ipset \ +wget \ +make \ +gcc wget ${IPTABLES_ZIP_URL}${IPTABLES_ZIP_FILE_NAME}.tar.bz2 tar xjf ${IPTABLES_ZIP_FILE_NAME}.tar.bz2 && cd ${IPTABLES_ZIP_FILE_NAME} @@ -25,8 +25,8 @@ do ln -sfv ../../lib/$(readlink /usr/lib/lib${file}.so) /usr/lib/lib${file}.so done -rm -rf ../${IPTABLES_ZIP_FILE_NAME}* +apt-get purge -y wget \ +make \ +gcc -apt-get purge -y wget -apt-get purge -y make -apt-get purge -y gcc \ No newline at end of file +rm -rf ../${IPTABLES_ZIP_FILE_NAME}* \ No newline at end of file From 9831bae38e9a8642edd7945be53dcf20519a66b7 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Wed, 13 Mar 2019 18:33:41 -0700 Subject: [PATCH 10/15] update dockerfile --- npm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npm/Dockerfile b/npm/Dockerfile index 7d725bc291..43c45cc36a 100644 --- a/npm/Dockerfile +++ b/npm/Dockerfile @@ -6,7 +6,7 @@ ARG SCRIPTS_DIR # Install dependencies. COPY $SCRIPTS_DIR/install-npm-dependencies.sh . RUN chmod +x ./install-npm-dependencies.sh -RUN ./install-npm-dependencies.sh > /dev/null +RUN ./install-npm-dependencies.sh # Install plugin. COPY $NPM_BUILD_DIR/azure-npm /usr/bin From aa2b1c56da3f07cedd99fd9c0d71e415abac8f11 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Thu, 14 Mar 2019 16:16:06 -0700 Subject: [PATCH 11/15] use lock for iptables-restore --- npm/iptm/iptm.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++- npm/util/const.go | 1 + 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/npm/iptm/iptm.go b/npm/iptm/iptm.go index e64db0a71b..96a2e8e3d9 100644 --- a/npm/iptm/iptm.go +++ b/npm/iptm/iptm.go @@ -1,12 +1,23 @@ +/* + +Part of this file is modified from iptables package from Kuberenetes. +https://github.com/kubernetes/kubernetes/blob/master/pkg/util/iptables + +*/ package iptm import ( + "fmt" "os" "os/exec" "syscall" + "time" + + "golang.org/x/sys/unix" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/npm/util" + "k8s.io/apimachinery/pkg/util/wait" ) // IptEntry represents an iptables rule. @@ -354,7 +365,7 @@ func (iptMgr *IptablesManager) Save(configFile string) error { cmd := exec.Command(util.IptablesSave) cmd.Stdout = f if err := cmd.Start(); err != nil { - log.Printf("Error running iptables-save.\n") + log.Printf("Error running iptables-save.") return err } cmd.Wait() @@ -363,6 +374,7 @@ func (iptMgr *IptablesManager) Save(configFile string) error { } // Restore restores iptables configuration from /var/log/iptables.conf +/* func (iptMgr *IptablesManager) Restore(configFile string) error { if len(configFile) == 0 { configFile = util.IptablesConfigFile @@ -378,6 +390,43 @@ func (iptMgr *IptablesManager) Restore(configFile string) error { cmd := exec.Command(util.IptablesRestore, util.IptablesWaitFlag) cmd.Stdin = f + if err := cmd.Start(); err != nil { + log.Printf("Error running iptables-restore.") + return err + } + cmd.Wait() + + return nil +} +*/ + +// Restore restores iptables configuration from /var/log/iptables.conf +func (iptMgr *IptablesManager) Restore(configFile string) error { + if len(configFile) == 0 { + configFile = util.IptablesConfigFile + } + + l, err := grabIptablesLocks() + if err != nil { + return err + } + + defer func(l *os.File) { + if err = l.Close(); err != nil { + fmt.Printf("Failed to close iptables locks") + } + }(l) + + // open the config file for reading + f, err := os.Open(configFile) + if err != nil { + log.Printf("Error opening file: %s.", configFile) + return err + } + defer f.Close() + + cmd := exec.Command(util.IptablesRestore) + cmd.Stdin = f if err := cmd.Start(); err != nil { log.Printf("Error running iptables-restore.\n") return err @@ -386,3 +435,41 @@ func (iptMgr *IptablesManager) Restore(configFile string) error { return nil } + +// grabs iptables v1.6 xtable lock +func grabIptablesLocks() (*os.File, error) { + var success bool + + l := &os.File{} + defer func(l *os.File) { + // Clean up immediately on failure + if !success { + l.Close() + } + }(l) + + // Grab 1.6.x style lock. + l, err := os.OpenFile(util.IptablesLockFile, os.O_CREATE, 0600) + if err != nil { + log.Printf("failed to open iptables lock") + return nil, err + } + + if err := wait.PollImmediate(200*time.Millisecond, 2*time.Second, func() (bool, error) { + if err := grabIptablesFileLock(l); err != nil { + return false, nil + } + + return true, nil + }); err != nil { + log.Printf("failed to acquire new iptables lock: %v", err) + return nil, err + } + + success = true + return l, nil +} + +func grabIptablesFileLock(f *os.File) error { + return unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) +} diff --git a/npm/util/const.go b/npm/util/const.go index 1024f257d5..880b893d2f 100644 --- a/npm/util/const.go +++ b/npm/util/const.go @@ -23,6 +23,7 @@ const ( IptablesRestore string = "iptables-restore" IptablesConfigFile string = "/var/log/iptables.conf" IptablesTestConfigFile string = "/var/log/iptables-test.conf" + IptablesLockFile string = "/run/xtables.lock" IptablesChainCreationFlag string = "-N" IptablesInsertionFlag string = "-I" IptablesAppendFlag string = "-A" From 73a872d8b52b773086139dc49c476dee81419ced Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Thu, 14 Mar 2019 16:26:12 -0700 Subject: [PATCH 12/15] use existing iptables --- Makefile | 2 -- npm/Dockerfile | 9 ++++---- scripts/install-npm-dependencies.sh | 32 ----------------------------- 3 files changed, 4 insertions(+), 39 deletions(-) delete mode 100644 scripts/install-npm-dependencies.sh diff --git a/Makefile b/Makefile index 86078ee285..0e1e284670 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,6 @@ CNI_IPAM_DIR = cni/ipam/plugin CNI_TELEMETRY_DIR = cni/telemetry/service CNS_DIR = cns/service NPM_DIR = npm/plugin -SCRIPTS_DIR = scripts OUTPUT_DIR = output BUILD_DIR = $(OUTPUT_DIR)/$(GOOS)_$(GOARCH) CNM_BUILD_DIR = $(BUILD_DIR)/cnm @@ -232,7 +231,6 @@ ifeq ($(GOOS),linux) -f npm/Dockerfile \ -t $(AZURE_NPM_IMAGE):$(AZURE_NPM_VERSION) \ --build-arg NPM_BUILD_DIR=$(NPM_BUILD_DIR) \ - --build-arg SCRIPTS_DIR=$(SCRIPTS_DIR) \ . docker save $(AZURE_NPM_IMAGE):$(AZURE_NPM_VERSION) | gzip -c > $(NPM_BUILD_DIR)/$(NPM_ARCHIVE_NAME) endif diff --git a/npm/Dockerfile b/npm/Dockerfile index 43c45cc36a..b04bb47da5 100644 --- a/npm/Dockerfile +++ b/npm/Dockerfile @@ -1,12 +1,11 @@ # Use a minimal image as a parent image -FROM ubuntu:18.10 +FROM ubuntu:16.04 ARG NPM_BUILD_DIR -ARG SCRIPTS_DIR # Install dependencies. -COPY $SCRIPTS_DIR/install-npm-dependencies.sh . -RUN chmod +x ./install-npm-dependencies.sh -RUN ./install-npm-dependencies.sh +RUN apt-get update +RUN apt-get install -y iptables +RUN apt-get install -y ipset # Install plugin. COPY $NPM_BUILD_DIR/azure-npm /usr/bin diff --git a/scripts/install-npm-dependencies.sh b/scripts/install-npm-dependencies.sh deleted file mode 100644 index dd9086ddf0..0000000000 --- a/scripts/install-npm-dependencies.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -IPTABLES_ZIP_URL=http://www.netfilter.org/projects/iptables/files/ -IPTABLES_ZIP_FILE_NAME=iptables-1.8.2 - -apt-get update -apt-get install -y ipset \ -wget \ -make \ -gcc - -wget ${IPTABLES_ZIP_URL}${IPTABLES_ZIP_FILE_NAME}.tar.bz2 -tar xjf ${IPTABLES_ZIP_FILE_NAME}.tar.bz2 && cd ${IPTABLES_ZIP_FILE_NAME} -./configure --prefix=/usr \ - --sbindir=/sbin \ - --disable-nftables \ - --enable-libipq \ - --with-xtlibdir=/lib/xtables && make - -make install < /dev/null -ln -sfv ../../sbin/xtables-legacy-multi /usr/bin/iptables-xml -for file in ip4tc ip6tc ipq iptc xtables -do - mv -v /usr/lib/lib${file}.so.* /lib && - ln -sfv ../../lib/$(readlink /usr/lib/lib${file}.so) /usr/lib/lib${file}.so -done - -apt-get purge -y wget \ -make \ -gcc - -rm -rf ../${IPTABLES_ZIP_FILE_NAME}* \ No newline at end of file From 004c8bd0cbf3c52974fbae67a0cfc65cbcd6304c Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Fri, 15 Mar 2019 10:02:09 -0700 Subject: [PATCH 13/15] update iptables-save --- npm/iptm/iptm.go | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/npm/iptm/iptm.go b/npm/iptm/iptm.go index 96a2e8e3d9..1b85e492b3 100644 --- a/npm/iptm/iptm.go +++ b/npm/iptm/iptm.go @@ -354,51 +354,35 @@ func (iptMgr *IptablesManager) Save(configFile string) error { configFile = util.IptablesConfigFile } - // create the config file for writing - f, err := os.Create(configFile) + l, err := grabIptablesLocks() if err != nil { - log.Printf("Error opening file: %s.", configFile) - return err - } - defer f.Close() - - cmd := exec.Command(util.IptablesSave) - cmd.Stdout = f - if err := cmd.Start(); err != nil { - log.Printf("Error running iptables-save.") return err } - cmd.Wait() - return nil -} - -// Restore restores iptables configuration from /var/log/iptables.conf -/* -func (iptMgr *IptablesManager) Restore(configFile string) error { - if len(configFile) == 0 { - configFile = util.IptablesConfigFile - } + defer func(l *os.File) { + if err = l.Close(); err != nil { + fmt.Printf("Failed to close iptables locks") + } + }(l) - // open the config file for reading - f, err := os.Open(configFile) + // create the config file for writing + f, err := os.Create(configFile) if err != nil { log.Printf("Error opening file: %s.", configFile) return err } defer f.Close() - cmd := exec.Command(util.IptablesRestore, util.IptablesWaitFlag) - cmd.Stdin = f + cmd := exec.Command(util.IptablesSave) + cmd.Stdout = f if err := cmd.Start(); err != nil { - log.Printf("Error running iptables-restore.") + log.Printf("Error running iptables-save.") return err } cmd.Wait() return nil } -*/ // Restore restores iptables configuration from /var/log/iptables.conf func (iptMgr *IptablesManager) Restore(configFile string) error { From ec18a05474f42180cec0bc520faae4ef12f7db5d Mon Sep 17 00:00:00 2001 From: Yongli Chen <12708785@qq.com> Date: Tue, 19 Mar 2019 11:52:11 -0700 Subject: [PATCH 14/15] use log instead of fmt --- npm/iptm/iptm.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/npm/iptm/iptm.go b/npm/iptm/iptm.go index 1b85e492b3..db797f1d01 100644 --- a/npm/iptm/iptm.go +++ b/npm/iptm/iptm.go @@ -7,7 +7,6 @@ https://github.com/kubernetes/kubernetes/blob/master/pkg/util/iptables package iptm import ( - "fmt" "os" "os/exec" "syscall" @@ -361,7 +360,7 @@ func (iptMgr *IptablesManager) Save(configFile string) error { defer func(l *os.File) { if err = l.Close(); err != nil { - fmt.Printf("Failed to close iptables locks") + log.Printf("Failed to close iptables locks") } }(l) @@ -397,7 +396,7 @@ func (iptMgr *IptablesManager) Restore(configFile string) error { defer func(l *os.File) { if err = l.Close(); err != nil { - fmt.Printf("Failed to close iptables locks") + log.Printf("Failed to close iptables locks") } }(l) From 997c8db46671f182c7fa0d7fdd19caa9fc49a30a Mon Sep 17 00:00:00 2001 From: Yongli Chen <12708785@qq.com> Date: Tue, 19 Mar 2019 11:59:50 -0700 Subject: [PATCH 15/15] rename telemetryRetryWaitTimeInSeconds --- npm/npm.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/npm/npm.go b/npm/npm.go index 93783e2f6a..57f2e307ac 100644 --- a/npm/npm.go +++ b/npm/npm.go @@ -24,9 +24,9 @@ import ( ) const ( - hostNetAgentURLForNpm = "http://168.63.129.16/machine/plugins?comp=netagent&type=npmreport" - contentType = "application/json" - retryWaitTimeInSeconds = 60 + hostNetAgentURLForNpm = "http://168.63.129.16/machine/plugins?comp=netagent&type=npmreport" + contentType = "application/json" + telemetryRetryWaitTimeInSeconds = 60 ) // NetworkPolicyManager contains informers for pod, namespace and networkpolicy. @@ -133,7 +133,7 @@ func connectToTelemetryServer(telemetryBuffer *telemetry.TelemetryBuffer) { } log.Printf("[NPM-Telemetry] Failed to establish telemetry manager connection.") - time.Sleep(time.Second * retryWaitTimeInSeconds) + time.Sleep(time.Second * telemetryRetryWaitTimeInSeconds) } } }