feat: [NPM] Restructure code and add deploy manifests

Tiltfile

@@ -0,0 +1,11 @@
+allow_k8s_contexts(k8s_context())
+default_registry('ttl.sh/nitishm-12390')
+docker_build('azure-npm', '.', dockerfile='npm/Dockerfile', build_args = {
+	"VERSION": "v1.4.14-101-gf900e319-dirty",
+	"NPM_AI_PATH": "github.com/Azure/azure-container-networking/npm.aiMetadata",
+	"NPM_AI_ID": "014c22bd-4107-459e-8475-67909e96edcb"
+})
+# watch_file('npm')
+k8s_yaml('npm/deploy/manifests/controller/azure-npm.yaml')
+k8s_yaml('npm/deploy/manifests/daemon/azure-npm.yaml', allow_duplicates=True)
+

docs/npm.md

@@ -12,7 +12,7 @@ Azure-NPM serves as a distributed firewall for the Kubernetes cluster, and it ca
 
 Running the command below will bring up one azure-npm instance on each Kubernetes node.
 ```
-kubectl apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/azure-npm.yaml
+kubectl apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/deploy/npm/azure-npm.yaml
 ```
 Now you can secure your Kubernetes cluster with Azure-NPM by applying Kubernetes network policies.
 

npm/azure-npm.yaml

@@ -97,10 +97,10 @@ spec:
             - name: NPM_CONFIG
               value: /etc/azure-npm/azure-npm.json
           volumeMounts:
-          - name: xtables-lock
-            mountPath: /run/xtables.lock
           - name: log
             mountPath: /var/log
+          - name: xtables-lock
+            mountPath: /run/xtables.lock
           - name: protocols
             mountPath: /etc/protocols
           - name: azure-npm-config
@@ -156,4 +156,4 @@ data:
             "EnableV2NPM":             false,
             "PlaceAzureChainFirst":    false
         }
-    }
+    }

npm/cmd/root.go

@@ -14,7 +14,13 @@ func NewRootCmd() *cobra.Command {
 		},
 	}
 
-	rootCmd.AddCommand(newStartNPMCmd())
+	startCmd := newStartNPMCmd()
+
+	startCmd.AddCommand(newStartNPMControlplaneCmd())
+	startCmd.AddCommand(newStartNPMDaemonCmd())
+
+	rootCmd.AddCommand(startCmd)
+
 	rootCmd.AddCommand(newDebugCmd())
 
 	return rootCmd

npm/cmd/start.go

@@ -18,6 +18,7 @@ import (
 	"github.com/Azure/azure-container-networking/npm/pkg/dataplane"
 	"github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets"
 	"github.com/Azure/azure-container-networking/npm/pkg/dataplane/policies"
+	"github.com/Azure/azure-container-networking/npm/pkg/models"
 	"github.com/Azure/azure-container-networking/npm/util"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -85,11 +86,6 @@ func newStartNPMCmd() *cobra.Command {
 
 	startNPMCmd.Flags().String(flagKubeConfigPath, flagDefaults[flagKubeConfigPath], "path to kubeconfig")
 
-	// The controlplane subcommand starts the NPM controller's controlplane component in the decomposed mode
-	startNPMCmd.AddCommand(newStartNPMControlplaneCmd())
-	// The daemon subcommand starts the NPM controller's datapath component in the daemon mode
-	startNPMCmd.AddCommand(newStartNPMDaemonCmd())
-
 	return startNPMCmd
 }
 
@@ -149,7 +145,7 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error {
 		} else {
 			npmV2DataplaneCfg.IPSetMode = ipsets.ApplyAllIPSets
 		}
-		dp, err = dataplane.NewDataPlane(npm.GetNodeName(), common.NewIOShim(), npmV2DataplaneCfg, stopChannel)
+		dp, err = dataplane.NewDataPlane(models.GetNodeName(), common.NewIOShim(), npmV2DataplaneCfg, stopChannel)
 		if err != nil {
 			return fmt.Errorf("failed to create dataplane with error %w", err)
 		}

npm/cmd/start_daemon.go

@@ -7,10 +7,11 @@ import (
 	"strconv"
 
 	"github.com/Azure/azure-container-networking/common"
-	"github.com/Azure/azure-container-networking/npm"
 	npmconfig "github.com/Azure/azure-container-networking/npm/config"
+	"github.com/Azure/azure-container-networking/npm/daemon"
 	"github.com/Azure/azure-container-networking/npm/pkg/controlplane/goalstateprocessor"
 	"github.com/Azure/azure-container-networking/npm/pkg/dataplane"
+	"github.com/Azure/azure-container-networking/npm/pkg/models"
 	"github.com/Azure/azure-container-networking/npm/pkg/transport"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -48,7 +49,7 @@ func startDaemon(config npmconfig.Config) error {
 	pod := os.Getenv(podNameEnv)
 	node := os.Getenv(nodeNameEnv)
 
-	addr := config.Transport.Address + ":" + strconv.Itoa(config.Transport.Port)
+	addr := config.Transport.Address + ":" + strconv.Itoa(config.Transport.ServicePort)
 	ctx := context.Background()
 	err := initLogging()
 	if err != nil {
@@ -58,7 +59,7 @@ func startDaemon(config npmconfig.Config) error {
 
 	var dp dataplane.GenericDataplane
 
-	dp, err = dataplane.NewDataPlane(npm.GetNodeName(), common.NewIOShim(), npmV2DataplaneCfg, wait.NeverStop)
+	dp, err = dataplane.NewDataPlane(models.GetNodeName(), common.NewIOShim(), npmV2DataplaneCfg, wait.NeverStop)
 	if err != nil {
 		klog.Errorf("failed to create dataplane: %v", err)
 		return fmt.Errorf("failed to create dataplane with error %w", err)
@@ -76,7 +77,7 @@ func startDaemon(config npmconfig.Config) error {
 		return fmt.Errorf("failed to create goalstate processor: %w", err)
 	}
 
-	n, err := npm.NewNetworkPolicyDaemon(ctx, config, dp, gsp, client, version)
+	n, err := daemon.NewNetworkPolicyDaemon(ctx, config, dp, gsp, client, version)
 	if err != nil {
 		klog.Errorf("failed to create dataplane : %v", err)
 		return fmt.Errorf("failed to create dataplane: %w", err)

npm/cmd/start_server.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/Azure/azure-container-networking/npm"
 	npmconfig "github.com/Azure/azure-container-networking/npm/config"
+	"github.com/Azure/azure-container-networking/npm/controller"
 	restserver "github.com/Azure/azure-container-networking/npm/http/server"
 	"github.com/Azure/azure-container-networking/npm/metrics"
 	"github.com/Azure/azure-container-networking/npm/pkg/dataplane"
@@ -105,7 +106,7 @@ func startControlplane(config npmconfig.Config, flags npmconfig.Flags) error {
 		return fmt.Errorf("failed to create dataplane with error: %w", err)
 	}
 
-	npMgr, err := npm.NewNetworkPolicyServer(config, factory, mgr, dp, version, k8sServerVersion)
+	npMgr, err := controller.NewNetworkPolicyServer(config, factory, mgr, dp, version, k8sServerVersion)
 	if err != nil {
 		klog.Errorf("failed to create NPM controlplane manager with error: %v", err)
 		return fmt.Errorf("failed to create NPM controlplane manager: %w", err)

npm/config/config.go

@@ -1,9 +1,10 @@
 package npmconfig
 
 const (
-	defaultResyncPeriod  = 15
-	defaultListeningPort = 10091
-	defaultGrpcPort      = 10092
+	defaultResyncPeriod    = 15
+	defaultListeningPort   = 10091
+	defaultGrpcPort        = 10092
+	defaultGrpcServicePort = 9002
 	// ConfigEnvPath is what's used by viper to load config path
 	ConfigEnvPath = "NPM_CONFIG"
 )
@@ -16,8 +17,9 @@ var DefaultConfig = Config{
 	ListeningAddress: "0.0.0.0",
 
 	Transport: GrpcServerConfig{
-		Address: "0.0.0.0",
-		Port:    defaultGrpcPort,
+		Address:     "0.0.0.0",
+		Port:        defaultGrpcPort,
+		ServicePort: defaultGrpcServicePort,
 	},
 
 	Toggles: Toggles{
@@ -35,6 +37,8 @@ type GrpcServerConfig struct {
 	Address string `json:"Address,omitempty"`
 	// Port is the port on which the gRPC server will listen
 	Port int `json:"Port,omitempty"`
+	// ServicePort is the service port for the client to connect to the gRPC server
+	ServicePort int `json:"ServicePort,omitempty"`
 }
 
 type Config struct {

npm/server.go → npm/controller/server.go

@@ -1,6 +1,6 @@
 // Copyright 2018 Microsoft. All rights reserved.
 // MIT License
-package npm
+package controller
 
 import (
 	"encoding/json"
@@ -9,6 +9,7 @@ import (
 	npmconfig "github.com/Azure/azure-container-networking/npm/config"
 	controllersv2 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v2"
 	"github.com/Azure/azure-container-networking/npm/pkg/dataplane"
+	"github.com/Azure/azure-container-networking/npm/pkg/models"
 	"github.com/Azure/azure-container-networking/npm/pkg/transport"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/version"
@@ -17,6 +18,8 @@ import (
 	"k8s.io/klog"
 )
 
+var aiMetadata string //nolint // aiMetadata is set in Makefile
+
 type NetworkPolicyServer struct {
 	config npmconfig.Config
 
@@ -25,20 +28,20 @@ type NetworkPolicyServer struct {
 
 	// Informers are the Kubernetes Informer
 	// https://pkg.go.dev/k8s.io/client-go/informers
-	Informers
+	models.Informers
 
 	// Controllers for handling Kubernetes resource watcher events
-	K8SControllersV2
+	models.K8SControllersV2
 
 	// Azure-specific variables
-	AzureConfig
+	models.AzureConfig
 }
 
 var (
-	ErrInformerFactoryNil  = errors.New("informer factory is nil")
-	ErrTransportManagerNil = errors.New("transport manager is nil")
-	ErrK8SServerVersionNil = errors.New("k8s server version is nil")
-	ErrInformerSyncFailure = errors.New("informer sync failure")
+	ErrInformerFactoryNil      = errors.New("informer factory is nil")
+	ErrTransportManagerNil     = errors.New("transport manager is nil")
+	ErrK8SServerVersionNil     = errors.New("k8s server version is nil")
+	ErrDataplaneNotInitialized = errors.New("dataplane is not initialized")
 )
 
 func NewNetworkPolicyServer(
@@ -70,87 +73,87 @@ func NewNetworkPolicyServer(
 	n := &NetworkPolicyServer{
 		config: config,
 		tm:     mgr,
-		Informers: Informers{
-			informerFactory: informerFactory,
-			podInformer:     informerFactory.Core().V1().Pods(),
-			nsInformer:      informerFactory.Core().V1().Namespaces(),
-			npInformer:      informerFactory.Networking().V1().NetworkPolicies(),
+		Informers: models.Informers{
+			InformerFactory: informerFactory,
+			PodInformer:     informerFactory.Core().V1().Pods(),
+			NsInformer:      informerFactory.Core().V1().Namespaces(),
+			NpInformer:      informerFactory.Networking().V1().NetworkPolicies(),
 		},
-		AzureConfig: AzureConfig{
-			k8sServerVersion: k8sServerVersion,
-			NodeName:         GetNodeName(),
-			version:          npmVersion,
+		AzureConfig: models.AzureConfig{
+			K8sServerVersion: k8sServerVersion,
+			NodeName:         models.GetNodeName(),
+			Version:          npmVersion,
 			TelemetryEnabled: true,
 		},
 	}
 
-	n.npmNamespaceCacheV2 = &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*controllersv2.Namespace)}
-	n.podControllerV2 = controllersv2.NewPodController(n.podInformer, dp, n.npmNamespaceCacheV2)
-	n.namespaceControllerV2 = controllersv2.NewNamespaceController(n.nsInformer, dp, n.npmNamespaceCacheV2)
-	n.netPolControllerV2 = controllersv2.NewNetworkPolicyController(n.npInformer, dp)
+	n.NpmNamespaceCacheV2 = &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*controllersv2.Namespace)}
+	n.PodControllerV2 = controllersv2.NewPodController(n.PodInformer, dp, n.NpmNamespaceCacheV2)
+	n.NamespaceControllerV2 = controllersv2.NewNamespaceController(n.NsInformer, dp, n.NpmNamespaceCacheV2)
+	n.NetPolControllerV2 = controllersv2.NewNetworkPolicyController(n.NpInformer, dp)
 
 	return n, nil
 }
 
 func (n *NetworkPolicyServer) MarshalJSON() ([]byte, error) {
-	m := map[CacheKey]json.RawMessage{}
+	m := map[models.CacheKey]json.RawMessage{}
 
 	var npmNamespaceCacheRaw []byte
 	var err error
-	npmNamespaceCacheRaw, err = json.Marshal(n.npmNamespaceCacheV2)
+	npmNamespaceCacheRaw, err = json.Marshal(n.NpmNamespaceCacheV2)
 
 	if err != nil {
-		return nil, errors.Errorf("%s: %v", errMarshalNPMCache, err)
+		return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err)
 	}
-	m[NsMap] = npmNamespaceCacheRaw
+	m[models.NsMap] = npmNamespaceCacheRaw
 
 	var podControllerRaw []byte
-	podControllerRaw, err = json.Marshal(n.podControllerV2)
+	podControllerRaw, err = json.Marshal(n.PodControllerV2)
 
 	if err != nil {
-		return nil, errors.Errorf("%s: %v", errMarshalNPMCache, err)
+		return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err)
 	}
-	m[PodMap] = podControllerRaw
+	m[models.PodMap] = podControllerRaw
 
 	nodeNameRaw, err := json.Marshal(n.NodeName)
 	if err != nil {
-		return nil, errors.Errorf("%s: %v", errMarshalNPMCache, err)
+		return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err)
 	}
-	m[NodeName] = nodeNameRaw
+	m[models.NodeName] = nodeNameRaw
 
 	npmCacheRaw, err := json.Marshal(m)
 	if err != nil {
-		return nil, errors.Errorf("%s: %v", errMarshalNPMCache, err)
+		return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err)
 	}
 
 	return npmCacheRaw, nil
 }
 
 func (n *NetworkPolicyServer) GetAppVersion() string {
-	return n.version
+	return n.Version
 }
 
 func (n *NetworkPolicyServer) Start(config npmconfig.Config, stopCh <-chan struct{}) error {
-	// Starts all informers manufactured by n's informerFactory.
-	n.informerFactory.Start(stopCh)
+	// Starts all informers manufactured by n's InformerFactory.
+	n.InformerFactory.Start(stopCh)
 
 	// Wait for the initial sync of local cache.
-	if !cache.WaitForCacheSync(stopCh, n.podInformer.Informer().HasSynced) {
-		return fmt.Errorf("Pod informer error: %w", ErrInformerSyncFailure)
+	if !cache.WaitForCacheSync(stopCh, n.PodInformer.Informer().HasSynced) {
+		return fmt.Errorf("Pod informer error: %w", models.ErrInformerSyncFailure)
 	}
 
-	if !cache.WaitForCacheSync(stopCh, n.nsInformer.Informer().HasSynced) {
-		return fmt.Errorf("Namespace informer error: %w", ErrInformerSyncFailure)
+	if !cache.WaitForCacheSync(stopCh, n.NsInformer.Informer().HasSynced) {
+		return fmt.Errorf("Namespace informer error: %w", models.ErrInformerSyncFailure)
 	}
 
-	if !cache.WaitForCacheSync(stopCh, n.npInformer.Informer().HasSynced) {
-		return fmt.Errorf("NetworkPolicy informer error: %w", ErrInformerSyncFailure)
+	if !cache.WaitForCacheSync(stopCh, n.NpInformer.Informer().HasSynced) {
+		return fmt.Errorf("NetworkPolicy informer error: %w", models.ErrInformerSyncFailure)
 	}
 
 	// start v2 NPM controllers after synced
-	go n.podControllerV2.Run(stopCh)
-	go n.namespaceControllerV2.Run(stopCh)
-	go n.netPolControllerV2.Run(stopCh)
+	go n.PodControllerV2.Run(stopCh)
+	go n.NamespaceControllerV2.Run(stopCh)
+	go n.NetPolControllerV2.Run(stopCh)
 
 	// start the transport layer (gRPC) server
 	// We block the main thread here until the server is stopped.

npm/daemon.go → npm/daemon/daemon.go

@@ -1,9 +1,10 @@
 // Copyright 2018 Microsoft. All rights reserved.
 // MIT License
-package npm
+package daemon
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	npmconfig "github.com/Azure/azure-container-networking/npm/config"
@@ -12,6 +13,10 @@ import (
 	"github.com/Azure/azure-container-networking/npm/pkg/transport"
 )
 
+var aiMetadata string //nolint // aiMetadata is set in Makefile
+
+var ErrDataplaneNotInitialized = errors.New("dataplane is not initialized")
+
 type NetworkPolicyDaemon struct {
 	ctx     context.Context
 	config  npmconfig.Config