-
Notifications
You must be signed in to change notification settings - Fork 38.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Initial work on running windows containers on Kubernetes #31707
Changes from all commits
0928586
2441525
9ef3528
a659ac9
66a1ef2
9e6815e
6daab26
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -196,7 +196,7 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { | |
if obj.DockerExecHandlerName == "" { | ||
obj.DockerExecHandlerName = "native" | ||
} | ||
if obj.DockerEndpoint == "" { | ||
if obj.DockerEndpoint == "" && runtime.GOOS != "windows" { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It seems like a better answer is conditionalize this on !windows |
||
obj.DockerEndpoint = "unix:///var/run/docker.sock" | ||
} | ||
if obj.EventBurst == 0 { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
// +build !cgo !linux | ||
// +build !linux,!windows | ||
|
||
/* | ||
Copyright 2015 The Kubernetes Authors. | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
// +build windows | ||
|
||
/* | ||
Copyright 2015 The Kubernetes Authors. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package cadvisor | ||
|
||
import ( | ||
"github.com/google/cadvisor/events" | ||
cadvisorapi "github.com/google/cadvisor/info/v1" | ||
cadvisorapiv2 "github.com/google/cadvisor/info/v2" | ||
) | ||
|
||
type cadvisorClient struct { | ||
} | ||
|
||
var _ Interface = new(cadvisorClient) | ||
|
||
// New creates a cAdvisor and exports its API on the specified port if port > 0. | ||
func New(port uint, runtime string, rootPath string) (Interface, error) { | ||
return &cadvisorClient{}, nil | ||
} | ||
|
||
func (cu *cadvisorClient) Start() error { | ||
return nil | ||
} | ||
|
||
func (cu *cadvisorClient) DockerContainer(name string, req *cadvisorapi.ContainerInfoRequest) (cadvisorapi.ContainerInfo, error) { | ||
return cadvisorapi.ContainerInfo{}, nil | ||
} | ||
|
||
func (cu *cadvisorClient) ContainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) { | ||
return &cadvisorapi.ContainerInfo{}, nil | ||
} | ||
|
||
func (cu *cadvisorClient) ContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) { | ||
return make(map[string]cadvisorapiv2.ContainerInfo), nil | ||
} | ||
|
||
func (cu *cadvisorClient) SubcontainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (map[string]*cadvisorapi.ContainerInfo, error) { | ||
return nil, nil | ||
} | ||
|
||
func (cu *cadvisorClient) MachineInfo() (*cadvisorapi.MachineInfo, error) { | ||
return &cadvisorapi.MachineInfo{}, nil | ||
} | ||
|
||
func (cu *cadvisorClient) VersionInfo() (*cadvisorapi.VersionInfo, error) { | ||
return &cadvisorapi.VersionInfo{}, nil | ||
} | ||
|
||
func (cu *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) { | ||
return cadvisorapiv2.FsInfo{}, nil | ||
} | ||
|
||
func (cu *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) { | ||
return cadvisorapiv2.FsInfo{}, nil | ||
} | ||
|
||
func (cu *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) { | ||
return &events.EventChannel{}, nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
// +build !linux | ||
// +build !linux,!windows | ||
|
||
/* | ||
Copyright 2015 The Kubernetes Authors. | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
// +build !linux | ||
// +build !linux,!windows | ||
|
||
/* | ||
Copyright 2015 The Kubernetes Authors. | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
// +build windows | ||
|
||
/* | ||
Copyright 2015 The Kubernetes Authors. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package cm | ||
|
||
import ( | ||
"github.com/golang/glog" | ||
|
||
"k8s.io/kubernetes/pkg/api" | ||
"k8s.io/kubernetes/pkg/kubelet/cadvisor" | ||
"k8s.io/kubernetes/pkg/util/mount" | ||
) | ||
|
||
type containerManagerImpl struct { | ||
containerManagerStub | ||
} | ||
|
||
var _ ContainerManager = &containerManagerImpl{} | ||
|
||
func (cm *containerManagerImpl) Start(_ *api.Node) error { | ||
glog.V(2).Infof("Starting Windows stub container manager") | ||
return nil | ||
} | ||
|
||
func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.Interface, nodeConfig NodeConfig) (ContainerManager, error) { | ||
return &containerManagerImpl{}, nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -28,6 +28,7 @@ import ( | |
"os/exec" | ||
"path" | ||
"path/filepath" | ||
"runtime" | ||
"strconv" | ||
"strings" | ||
"sync" | ||
|
@@ -57,7 +58,7 @@ import ( | |
"k8s.io/kubernetes/pkg/kubelet/types" | ||
"k8s.io/kubernetes/pkg/kubelet/util/cache" | ||
"k8s.io/kubernetes/pkg/kubelet/util/format" | ||
"k8s.io/kubernetes/pkg/runtime" | ||
kruntime "k8s.io/kubernetes/pkg/runtime" | ||
"k8s.io/kubernetes/pkg/security/apparmor" | ||
"k8s.io/kubernetes/pkg/securitycontext" | ||
kubetypes "k8s.io/kubernetes/pkg/types" | ||
|
@@ -341,16 +342,7 @@ var ( | |
// that the container passed is the infrastructure container of a pod and the responsibility | ||
// of the caller to ensure that the correct container is passed. | ||
func (dm *DockerManager) determineContainerIP(podNamespace, podName string, container *dockertypes.ContainerJSON) (string, error) { | ||
result := "" | ||
|
||
if container.NetworkSettings != nil { | ||
result = container.NetworkSettings.IPAddress | ||
|
||
// Fall back to IPv6 address if no IPv4 address is present | ||
if result == "" { | ||
result = container.NetworkSettings.GlobalIPv6Address | ||
} | ||
} | ||
result := getContainerIP(container) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is probably something we should clear up for Linux too; instead of having the !network.DefaultPluginName check just below, we could have the default plugin just do the equivalent of this block. I forget if docker adds the host IP address to the container properties for HostNetwork mode though? |
||
|
||
networkMode := getDockerNetworkMode(container) | ||
isHostNetwork := networkMode == namespaceModeHost | ||
|
@@ -431,7 +423,7 @@ func (dm *DockerManager) inspectContainer(id string, podName, podNamespace strin | |
// Container that are running, restarting and paused | ||
status.State = kubecontainer.ContainerStateRunning | ||
status.StartedAt = startedAt | ||
if containerName == PodInfraContainerName { | ||
if containerProvidesPodIP(dockerName) { | ||
ip, err = dm.determineContainerIP(podNamespace, podName, iResult) | ||
// Kubelet doesn't handle the network error scenario | ||
if err != nil { | ||
|
@@ -615,7 +607,7 @@ func (dm *DockerManager) runContainer( | |
// TODO: This is kind of hacky, we should really just encode the bits we need. | ||
// TODO: This is hacky because the Kubelet should be parameterized to encode a specific version | ||
// and needs to be able to migrate this whenever we deprecate v1. Should be a member of DockerManager. | ||
if data, err := runtime.Encode(api.Codecs.LegacyCodec(unversioned.GroupVersion{Group: api.GroupName, Version: "v1"}), pod); err == nil { | ||
if data, err := kruntime.Encode(api.Codecs.LegacyCodec(unversioned.GroupVersion{Group: api.GroupName, Version: "v1"}), pod); err == nil { | ||
labels[kubernetesPodLabel] = string(data) | ||
} else { | ||
glog.Errorf("Failed to encode pod: %s for prestop hook", pod.Name) | ||
|
@@ -686,6 +678,12 @@ func (dm *DockerManager) runContainer( | |
SecurityOpt: fmtSecurityOpts, | ||
} | ||
|
||
// There is no /etc/resolv.conf in Windows, DNS and DNSSearch options would have to be passed to Docker runtime instead | ||
if runtime.GOOS == "windows" { | ||
hc.DNS = opts.DNS | ||
hc.DNSSearch = opts.DNSSearch | ||
} | ||
|
||
// Set sysctls if requested | ||
sysctls, unsafeSysctls, err := api.SysctlsFromPodAnnotations(pod.Annotations) | ||
if err != nil { | ||
|
@@ -1122,23 +1120,6 @@ func (dm *DockerManager) fmtDockerOpts(opts []dockerOpt) ([]string, error) { | |
return fmtOpts, nil | ||
} | ||
|
||
func (dm *DockerManager) getSecurityOpts(pod *api.Pod, ctrName string) ([]dockerOpt, error) { | ||
var securityOpts []dockerOpt | ||
if seccompOpts, err := dm.getSeccompOpts(pod, ctrName); err != nil { | ||
return nil, err | ||
} else { | ||
securityOpts = append(securityOpts, seccompOpts...) | ||
} | ||
|
||
if appArmorOpts, err := dm.getAppArmorOpts(pod, ctrName); err != nil { | ||
return nil, err | ||
} else { | ||
securityOpts = append(securityOpts, appArmorOpts...) | ||
} | ||
|
||
return securityOpts, nil | ||
} | ||
|
||
type dockerOpt struct { | ||
// The key-value pair passed to docker. | ||
key, value string | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you restore this function to this location so that we minimize the diff? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. (ah, nm you moved this out to a different file...) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah, needs different impls for now. |
||
|
@@ -1612,7 +1593,7 @@ func containerAndPodFromLabels(inspect *dockertypes.ContainerJSON) (pod *api.Pod | |
// the pod data may not be set | ||
if body, found := labels[kubernetesPodLabel]; found { | ||
pod = &api.Pod{} | ||
if err = runtime.DecodeInto(api.Codecs.UniversalDecoder(), []byte(body), pod); err == nil { | ||
if err = kruntime.DecodeInto(api.Codecs.UniversalDecoder(), []byte(body), pod); err == nil { | ||
name := labels[types.KubernetesContainerNameLabel] | ||
for ix := range pod.Spec.Containers { | ||
if pod.Spec.Containers[ix].Name == name { | ||
|
@@ -2321,7 +2302,14 @@ func (dm *DockerManager) tryContainerStart(container *api.Container, pod *api.Po | |
restartCount = containerStatus.RestartCount + 1 | ||
} | ||
|
||
_, err = dm.runContainerInPod(pod, container, namespaceMode, namespaceMode, pidMode, podIP, restartCount) | ||
// Allow override of networking mode for specific platforms (e.g. Windows) | ||
netMode := getNetworkingMode() | ||
if netMode == "" { | ||
// If not overriden, use the namespace mode | ||
netMode = namespaceMode | ||
} | ||
|
||
_, err = dm.runContainerInPod(pod, container, netMode, namespaceMode, pidMode, podIP, restartCount) | ||
if err != nil { | ||
// TODO(bburns) : Perhaps blacklist a container after N failures? | ||
return kubecontainer.ErrRunContainer, err.Error() | ||
|
@@ -2618,7 +2606,7 @@ func (dm *DockerManager) GetPodStatus(uid kubetypes.UID, name, namespace string) | |
} | ||
} | ||
containerStatuses = append(containerStatuses, result) | ||
if ip != "" { | ||
if containerProvidesPodIP(dockerName) && ip != "" { | ||
podStatus.IP = ip | ||
} | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
// +build linux | ||
|
||
/* | ||
Copyright 2015 The Kubernetes Authors. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package dockertools | ||
|
||
import ( | ||
dockertypes "github.com/docker/engine-api/types" | ||
"k8s.io/kubernetes/pkg/api" | ||
) | ||
|
||
func getContainerIP(container *dockertypes.ContainerJSON) string { | ||
result := "" | ||
if container.NetworkSettings != nil { | ||
result = container.NetworkSettings.IPAddress | ||
|
||
// Fall back to IPv6 address if no IPv4 address is present | ||
if result == "" { | ||
result = container.NetworkSettings.GlobalIPv6Address | ||
} | ||
} | ||
return result | ||
} | ||
|
||
// We don't want to override the networking mode on Linux. | ||
func getNetworkingMode() string { return "" } | ||
|
||
// Returns true if the container name matches the infrastructure's container name | ||
func containerProvidesPodIP(name *KubeletContainerName) bool { | ||
return name.ContainerName == PodInfraContainerName | ||
} | ||
|
||
// Returns Seccomp and AppArmor Security options | ||
func (dm *DockerManager) getSecurityOpts(pod *api.Pod, ctrName string) ([]dockerOpt, error) { | ||
var securityOpts []dockerOpt | ||
if seccompOpts, err := dm.getSeccompOpts(pod, ctrName); err != nil { | ||
return nil, err | ||
} else { | ||
securityOpts = append(securityOpts, seccompOpts...) | ||
} | ||
|
||
if appArmorOpts, err := dm.getAppArmorOpts(pod, ctrName); err != nil { | ||
return nil, err | ||
} else { | ||
securityOpts = append(securityOpts, appArmorOpts...) | ||
} | ||
|
||
return securityOpts, nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
doesn't seem like this should be removed?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
or at least conditionalize this on windows?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
My question here is, where exactly does this runs, the kubelet? I was under the impression this validation would happen at the API server and having that said, just couldn't conditionalize on Windows, e.g.
if GOOS=="windows"
.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
ah, good point. Can you instead make this check deeper?
e.g. `[A-Za-z]:' is ok, anything else isn't. Also if there are any forward slashes '/' then there better not be any colons.
Basically heuristically guess if it is a windows or linux path and then do the ':' validation if it looks like a Linux path.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
On it.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Actually, it seems that:
c:/my/path
works with Windows Containers (there's some translation in place)/my:path
is valid Unix pathPlease, advise.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
As agreed on Slack between me and @brendandburns, this is without effect.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ok, yeah, this is fine with me given that /foo:bar/baz is apparently a valid unix path. It's unclear why this check was there to begin with.