Add options to disable hyper internal haproxy

Author: feiskyer

cmd/kube-proxy/app/options/options.go

@@ -61,6 +61,7 @@ func NewProxyConfig() *ProxyServerConfig {
 // AddFlags adds flags for a specific ProxyServer to the specified FlagSet
 func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
 	fs.Var(componentconfig.IPVar{&s.BindAddress}, "bind-address", "The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)")
+	fs.BoolVar(&s.DisableHyperInternalService, "disable-hyper-internal-service", s.DisableHyperInternalService, "Disable the internal haproxy service in Hyper pods")
 	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
 	fs.IntVar(&s.HealthzPort, "healthz-port", s.HealthzPort, "The port to bind the health check server. Use 0 to disable.")
 	fs.Var(componentconfig.IPVar{&s.HealthzBindAddress}, "healthz-bind-address", "The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)")

cmd/kube-proxy/app/server.go

@@ -217,7 +217,7 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err
 		userspace.CleanupLeftovers(iptInterface)
 	case proxyModeHaproxy:
 		glog.V(2).Info("Using pod-buildin-haproxy proxy.")
-		proxierBuildin, err := haproxy.NewProxier(config.ConfigSyncPeriod, client)
+		proxierBuildin, err := haproxy.NewProxier(config.ConfigSyncPeriod, client, config.DisableHyperInternalService)
 		if err != nil {
 			glog.Fatalf("Unable to create proxier: %v", err)
 		}

cmd/kubelet/app/options/options.go

@@ -139,6 +139,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&s.HTTPCheckFrequency.Duration, "http-check-frequency", s.HTTPCheckFrequency.Duration, "Duration between checking http for new data")
 	fs.StringVar(&s.ManifestURL, "manifest-url", s.ManifestURL, "URL for accessing the container manifest")
 	fs.StringVar(&s.ManifestURLHeader, "manifest-url-header", s.ManifestURLHeader, "HTTP header to use when accessing the manifest URL, with the key separated from the value with a ':', as in 'key:value'")
+	fs.BoolVar(&s.DisableHyperInternalService, "disable-hyper-internal-service", s.DisableHyperInternalService, "Disable the internal haproxy service in Hyper pods")
 	fs.BoolVar(&s.EnableServer, "enable-server", s.EnableServer, "Enable the Kubelet's server")
 	fs.Var(componentconfig.IPVar{&s.Address}, "address", "The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces)")
 	fs.UintVar(&s.Port, "port", s.Port, "The port for the Kubelet to serve on.")

cmd/kubelet/app/server.go

@@ -195,57 +195,58 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
 	}
 
 	return &KubeletConfig{
-		Address:                   net.ParseIP(s.Address),
-		AllowPrivileged:           s.AllowPrivileged,
-		Auth:                      nil, // default does not enforce auth[nz]
-		CAdvisorInterface:         nil, // launches background processes, not set here
-		VolumeStatsAggPeriod:      s.VolumeStatsAggPeriod.Duration,
-		CgroupRoot:                s.CgroupRoot,
-		CinderConfig:              s.CinderConfig,
-		Cloud:                     nil, // cloud provider might start background processes
-		ClusterDNS:                net.ParseIP(s.ClusterDNS),
-		ClusterDomain:             s.ClusterDomain,
-		ConfigFile:                s.Config,
-		ConfigureCBR0:             s.ConfigureCBR0,
-		ContainerManager:          nil,
-		ContainerRuntime:          s.ContainerRuntime,
-		CPUCFSQuota:               s.CPUCFSQuota,
-		DiskSpacePolicy:           diskSpacePolicy,
-		DockerClient:              dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
-		RuntimeCgroups:            s.RuntimeCgroups,
-		DockerExecHandler:         dockerExecHandler,
-		EnableCustomMetrics:       s.EnableCustomMetrics,
-		EnableDebuggingHandlers:   s.EnableDebuggingHandlers,
-		EnableServer:              s.EnableServer,
-		EventBurst:                s.EventBurst,
-		EventRecordQPS:            s.EventRecordQPS,
-		FileCheckFrequency:        s.FileCheckFrequency.Duration,
-		HostnameOverride:          s.HostnameOverride,
-		HostNetworkSources:        hostNetworkSources,
-		HostPIDSources:            hostPIDSources,
-		HostIPCSources:            hostIPCSources,
-		HTTPCheckFrequency:        s.HTTPCheckFrequency.Duration,
-		ImageGCPolicy:             imageGCPolicy,
-		KubeClient:                nil,
-		ManifestURL:               s.ManifestURL,
-		ManifestURLHeader:         manifestURLHeader,
-		MasterServiceNamespace:    s.MasterServiceNamespace,
-		MaxContainerCount:         s.MaxContainerCount,
-		MaxOpenFiles:              s.MaxOpenFiles,
-		MaxPerPodContainerCount:   s.MaxPerPodContainerCount,
-		MaxPods:                   s.MaxPods,
-		MinimumGCAge:              s.MinimumGCAge.Duration,
-		Mounter:                   mounter,
-		NetworkPluginName:         networkPluginName,
-		NetworkPlugins:            networkPlugins,
-		NodeLabels:                s.NodeLabels,
-		NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency.Duration,
-		NonMasqueradeCIDR:         s.NonMasqueradeCIDR,
-		OOMAdjuster:               oom.NewOOMAdjuster(),
-		OSInterface:               kubecontainer.RealOS{},
-		PodCIDR:                   s.PodCIDR,
-		ReconcileCIDR:             s.ReconcileCIDR,
-		PodInfraContainerImage:    s.PodInfraContainerImage,
+		Address:                     net.ParseIP(s.Address),
+		AllowPrivileged:             s.AllowPrivileged,
+		Auth:                        nil, // default does not enforce auth[nz]
+		CAdvisorInterface:           nil, // launches background processes, not set here
+		VolumeStatsAggPeriod:        s.VolumeStatsAggPeriod.Duration,
+		CgroupRoot:                  s.CgroupRoot,
+		CinderConfig:                s.CinderConfig,
+		Cloud:                       nil, // cloud provider might start background processes
+		ClusterDNS:                  net.ParseIP(s.ClusterDNS),
+		ClusterDomain:               s.ClusterDomain,
+		ConfigFile:                  s.Config,
+		ConfigureCBR0:               s.ConfigureCBR0,
+		ContainerManager:            nil,
+		ContainerRuntime:            s.ContainerRuntime,
+		CPUCFSQuota:                 s.CPUCFSQuota,
+		DiskSpacePolicy:             diskSpacePolicy,
+		DockerClient:                dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
+		RuntimeCgroups:              s.RuntimeCgroups,
+		DockerExecHandler:           dockerExecHandler,
+		EnableCustomMetrics:         s.EnableCustomMetrics,
+		EnableDebuggingHandlers:     s.EnableDebuggingHandlers,
+		EnableServer:                s.EnableServer,
+		EventBurst:                  s.EventBurst,
+		EventRecordQPS:              s.EventRecordQPS,
+		FileCheckFrequency:          s.FileCheckFrequency.Duration,
+		HostnameOverride:            s.HostnameOverride,
+		HostNetworkSources:          hostNetworkSources,
+		HostPIDSources:              hostPIDSources,
+		HostIPCSources:              hostIPCSources,
+		HTTPCheckFrequency:          s.HTTPCheckFrequency.Duration,
+		ImageGCPolicy:               imageGCPolicy,
+		KubeClient:                  nil,
+		ManifestURL:                 s.ManifestURL,
+		ManifestURLHeader:           manifestURLHeader,
+		MasterServiceNamespace:      s.MasterServiceNamespace,
+		MaxContainerCount:           s.MaxContainerCount,
+		MaxOpenFiles:                s.MaxOpenFiles,
+		MaxPerPodContainerCount:     s.MaxPerPodContainerCount,
+		MaxPods:                     s.MaxPods,
+		MinimumGCAge:                s.MinimumGCAge.Duration,
+		Mounter:                     mounter,
+		NetworkPluginName:           networkPluginName,
+		NetworkPlugins:              networkPlugins,
+		NodeLabels:                  s.NodeLabels,
+		NodeStatusUpdateFrequency:   s.NodeStatusUpdateFrequency.Duration,
+		NonMasqueradeCIDR:           s.NonMasqueradeCIDR,
+		OOMAdjuster:                 oom.NewOOMAdjuster(),
+		OSInterface:                 kubecontainer.RealOS{},
+		PodCIDR:                     s.PodCIDR,
+		ReconcileCIDR:               s.ReconcileCIDR,
+		PodInfraContainerImage:      s.PodInfraContainerImage,
+		DisableHyperInternalService: s.DisableHyperInternalService,
 		Port:                           s.Port,
 		ReadOnlyPort:                   s.ReadOnlyPort,
 		RegisterNode:                   s.RegisterNode,
@@ -720,6 +721,7 @@ type KubeletConfig struct {
 	ContainerManager               cm.ContainerManager
 	ContainerRuntime               string
 	CPUCFSQuota                    bool
+	DisableHyperInternalService    bool
 	DiskSpacePolicy                kubelet.DiskSpacePolicy
 	DockerClient                   dockertools.DockerInterface
 	RuntimeCgroups                 string
@@ -879,6 +881,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.HairpinMode,
 		kc.BabysitDaemons,
 		kc.Options,
+		kc.DisableHyperInternalService,
 	)
 
 	if err != nil {

docs/admin/kube-proxy.md

@@ -30,6 +30,7 @@ kube-proxy
       --config-sync-period=15m0s: How often configuration from the apiserver is refreshed.  Must be greater than 0.
       --conntrack-max=262144: Maximum number of NAT connections to track (0 to leave as-is)
       --conntrack-tcp-timeout-established=24h0m0s: Idle timeout for established TCP connections (0 to leave as-is)
+      --disable-hyper-internal-service[=false]: Disable the internal haproxy service in Hyper pods
       --google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication.
       --healthz-bind-address=127.0.0.1: The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
       --healthz-port=10249: The port to bind the health check server. Use 0 to disable.
@@ -43,25 +44,13 @@ kube-proxy
       --masquerade-all[=false]: If using the pure iptables proxy, SNAT everything
       --master="": The address of the Kubernetes API server (overrides any value in kubeconfig)
       --oom-score-adj=-999: The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]
-<<<<<<< b1fd15c08dee9971bd6d691aba1c4676fa90fa39
       --proxy-mode=: Which proxy mode to use: 'userspace' (older) or 'iptables' (faster). If blank, look at the Node object on the Kubernetes API and respect the 'net.experimental.kubernetes.io/proxy-mode' annotation if provided.  Otherwise use the best-available proxy (currently iptables).  If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.
-=======
-      --proxy-mode="haproxy": Which proxy mode to use: 'userspace' (older, stable) , 'iptables' (experimental) or haproxy (for hyper only). If blank, look at the Node object on the Kubernetes API and respect the 'net.experimental.kubernetes.io/proxy-mode' annotation if provided.  Otherwise use the best-available proxy (currently userspace, but may change in future versions).  If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.
->>>>>>> Rebased with kubernetes upstream
       --proxy-port-range=: Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.
       --udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxy-mode=userspace
 ```
 
 ###### Auto generated by spf13/cobra on 7-Feb-2016
 
-
-
-
-<!-- BEGIN MUNGE: IS_VERSIONED -->
-<!-- TAG IS_VERSIONED -->
-<!-- END MUNGE: IS_VERSIONED -->
-
-
 <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/admin/kube-proxy.md?pixel)]()
 <!-- END MUNGE: GENERATED_ANALYTICS -->

docs/admin/kubelet.md

@@ -81,6 +81,7 @@ kubelet
       --container-runtime="docker": The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.
       --containerized[=false]: Experimental support for running kubelet in a container.  Intended for testing. [default=false]
       --cpu-cfs-quota[=true]: Enable CPU CFS quota enforcement for containers that specify CPU limits
+      --disable-hyper-internal-service[=false]: Disable the internal haproxy service in Hyper pods
       --docker-endpoint="": If non-empty, use this for the docker endpoint to communicate with
       --docker-exec-handler="native": Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.
       --enable-custom-metrics[=false]: Support for gathering custom metrics.

hack/verify-flags/known-flags.txt

@@ -84,6 +84,7 @@ disable-filter
 docker-email
 docker-endpoint
 docker-exec-handler
+disable-hyper-internal-service
 docker-password
 docker-server
 docker-username

pkg/apis/componentconfig/types.go

@@ -24,6 +24,8 @@ type KubeProxyConfiguration struct {
 	// bindAddress is the IP address for the proxy server to serve on (set to 0.0.0.0
 	// for all interfaces)
 	BindAddress string `json:"bindAddress"`
+	// disableHyperInternalService disables haproxy proxy in Hyper Pod
+	DisableHyperInternalService bool `json:"DisableHyperInternalService"`
 	// healthzBindAddress is the IP address for the health check server to serve on,
 	// defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
 	HealthzBindAddress string `json:"healthzBindAddress"`
@@ -102,6 +104,8 @@ type KubeletConfiguration struct {
 	Config string `json:"config"`
 	// cinderConfig is the config file for openstack cinder
 	CinderConfig string `json:""CinderConfig`
+	// disableHyperInternalService disables haproxy in Hyper pod
+	DisableHyperInternalService bool `json:"DisableHyperInternalService"`
 	// syncFrequency is the max period between synchronizing running
 	// containers and config
 	SyncFrequency unversioned.Duration `json:"syncFrequency"`

pkg/kubelet/hyper/hyper.go

@@ -74,6 +74,9 @@ type runtime struct {
 	imagePuller         kubecontainer.ImagePuller
 	version             kubecontainer.Version
 
+	// Disable the internal haproxy service in Hyper pods
+	disableHyperInternalService bool
+
 	// Runner of lifecycle events.
 	runner kubecontainer.HandlerRunner
 }
@@ -95,6 +98,7 @@ func New(generator kubecontainer.RunContainerOptionsGenerator,
 	imageBackOff *util.Backoff,
 	serializeImagePulls bool,
 	httpClient kubetypes.HttpGetter,
+	disableHyperInternalService bool,
 ) (kubecontainer.Runtime, error) {
 	// check hyper has already installed
 	hyperBinAbsPath, err := exec.LookPath(hyperBinName)
@@ -104,16 +108,17 @@ func New(generator kubecontainer.RunContainerOptionsGenerator,
 	}
 
 	hyper := &runtime{
-		hyperBinAbsPath:     hyperBinAbsPath,
-		dockerKeyring:       credentialprovider.NewDockerKeyring(),
-		containerRefManager: containerRefManager,
-		generator:           generator,
-		livenessManager:     livenessManager,
-		recorder:            recorder,
-		networkPlugin:       networkPlugin,
-		volumeGetter:        volumeGetter,
-		hyperClient:         NewHyperClient(),
-		kubeClient:          kubeClient,
+		hyperBinAbsPath:             hyperBinAbsPath,
+		dockerKeyring:               credentialprovider.NewDockerKeyring(),
+		containerRefManager:         containerRefManager,
+		generator:                   generator,
+		livenessManager:             livenessManager,
+		recorder:                    recorder,
+		networkPlugin:               networkPlugin,
+		volumeGetter:                volumeGetter,
+		hyperClient:                 NewHyperClient(),
+		kubeClient:                  kubeClient,
+		disableHyperInternalService: disableHyperInternalService,
 	}
 
 	if serializeImagePulls {
@@ -454,17 +459,19 @@ func (r *runtime) buildHyperPod(pod *api.Pod, restartCount int, pullSecrets []ap
 
 	glog.V(4).Infof("Hyper volumes: %v", volumes)
 
-	services := r.buildHyperPodServices(pod)
-	if services == nil {
-		// services can't be null for kubernetes, so fake one if it is null
-		services = []HyperService{
-			{
-				ServiceIP:   "127.0.0.2",
-				ServicePort: 65534,
-			},
+	if !r.disableHyperInternalService {
+		services := r.buildHyperPodServices(pod)
+		if services == nil {
+			// services can't be null for kubernetes, so fake one if it is null
+			services = []HyperService{
+				{
+					ServiceIP:   "127.0.0.2",
+					ServicePort: 65534,
+				},
+			}
 		}
+		specMap["services"] = services
 	}
-	specMap["services"] = services
 
 	// build hyper containers spec
 	var containers []map[string]interface{}

pkg/kubelet/hyper/hyperclient.go

@@ -39,7 +39,7 @@ const (
 	HYPER_PROTO       = "unix"
 	HYPER_ADDR        = "/var/run/hyper.sock"
 	HYPER_SCHEME      = "http"
-	HYPER_MINVERSION  = "0.4.0"
+	HYPER_MINVERSION  = "0.5.0"
 	DEFAULT_IMAGE_TAG = "latest"
 
 	KEY_COMMAND        = "command"

pkg/kubelet/kubelet.go

@@ -218,6 +218,7 @@ func NewMainKubelet(
 	hairpinMode string,
 	babysitDaemons bool,
 	kubeOptions []Option,
+	disableHyperInternalService bool,
 ) (*Kubelet, error) {
 	if rootDirectory == "" {
 		return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
@@ -341,6 +342,7 @@ func NewMainKubelet(
 		reservation:                  reservation,
 		enableCustomMetrics:          enableCustomMetrics,
 		babysitDaemons:               babysitDaemons,
+		disableHyperInternalService:  disableHyperInternalService,
 	}
 	// TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency
 	klet.resourceAnalyzer = stats.NewResourceAnalyzer(klet, volumeStatsAggPeriod)
@@ -444,6 +446,7 @@ func NewMainKubelet(
 			imageBackOff,
 			serializeImagePulls,
 			klet.httpClient,
+			klet.disableHyperInternalService,
 		)
 		if err != nil {
 			return nil, err
@@ -775,6 +778,9 @@ type Kubelet struct {
 
 	// handlers called during the tryUpdateNodeStatus cycle
 	setNodeStatusFuncs []func(*api.Node) error
+
+	// Disable the internal haproxy service in Hyper pods
+	disableHyperInternalService bool
 }
 
 // Validate given node IP belongs to the current host

pkg/proxy/haproxy/proxier.go

@@ -68,8 +68,9 @@ type Proxier struct {
 	haveReceivedEndpointsUpdate bool // true once we've seen an OnEndpointsUpdate event
 
 	// These are effectively const and do not need the mutex to be held.
-	syncPeriod    time.Duration
-	masqueradeAll bool
+	syncPeriod                  time.Duration
+	masqueradeAll               bool
+	disableHyperInternalService bool
 }
 
 type localPort struct {
@@ -91,7 +92,7 @@ type closeable interface {
 var _ proxy.ProxyProvider = &Proxier{}
 
 // NewProxier returns a new Proxier given an pod-buildin-haproxy Interface instance.
-func NewProxier(syncPeriod time.Duration, kubeClient *kubeclient.Client) (*Proxier, error) {
+func NewProxier(syncPeriod time.Duration, kubeClient *kubeclient.Client, disableHyperInternalService bool) (*Proxier, error) {
 	client := hyper.NewHyperClient()
 	_, err := client.Version()
 	if err != nil {
@@ -100,11 +101,12 @@ func NewProxier(syncPeriod time.Duration, kubeClient *kubeclient.Client) (*Proxi
 	}
 
 	return &Proxier{
-		serviceMap:  make(map[proxy.ServicePortName]*serviceInfo),
-		portsMap:    make(map[localPort]closeable),
-		syncPeriod:  syncPeriod,
-		hyperClient: client,
-		kubeClient:  kubeClient,
+		serviceMap:                  make(map[proxy.ServicePortName]*serviceInfo),
+		portsMap:                    make(map[localPort]closeable),
+		syncPeriod:                  syncPeriod,
+		hyperClient:                 client,
+		kubeClient:                  kubeClient,
+		disableHyperInternalService: disableHyperInternalService,
 	}, nil
 }
 
@@ -352,6 +354,10 @@ func flattenValidEndpoints(endpoints []hostPortPair) []string {
 // This is where all of haproxy-setting calls happen.
 // assumes proxier.mu is held
 func (proxier *Proxier) syncProxyRules() {
+	if proxier.disableHyperInternalService {
+		return
+	}
+
 	// don't sync rules till we've received services and endpoints
 	if !proxier.haveReceivedEndpointsUpdate || !proxier.haveReceivedServiceUpdate {
 		glog.V(2).Info("Not syncing proxy rules until Services and Endpoints have been received from master")