Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Changes to support OpenEBS integration. #529

Merged
merged 4 commits into from Dec 6, 2019
Merged
Changes from all commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -47,6 +47,7 @@ RUN apt-get update && apt-get install -q -y --allow-downgrades bridge-utils \
nfs-common \
jq \
conntrack \
open-iscsi \
strace ; \
apt-get -t testing install -y lvm2; \
apt-get -y autoclean; apt-get -y clean
@@ -32,6 +32,14 @@ RUN locale-gen \
# Exit codes < 0 are failures
RUN systemctl set-default multi-user.target; if [ "$?" -lt 0 ]; then exit $?; fi;

# TODO(r0mant): Disable *iscsi* services cause they may be running on host
# In the future we will need to enable them conditionally to
# be able to support OpenEBS cStor engine out of the box
RUN systemctl mask iscsi.service && \
systemctl mask iscsid.service && \
systemctl mask open-iscsi.service && \
systemctl mask systemd-udevd.service

ENV LANGUAGE en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
@@ -9,6 +9,7 @@ EnvironmentFile=/run/dns.env
EnvironmentFile=-/run/kubernetes-reserved.env
ExecStartPre=/bin/systemctl is-active docker.service
ExecStart=/usr/bin/kubelet \
--root-dir=/var/lib/gravity/planet/kubelet \
--hostname-override=${KUBE_NODE_NAME} \
--logtostderr=true \
--cluster-dns=${DNS_ADDRESSES} \
@@ -27,6 +27,7 @@ all:
rm -rf $(ROOTFS)/var/log/*
rm -rf $(ROOTFS)/var/cache
rm -rf $(ROOTFS)/lib/systemd/system/sysinit.target.wants/proc-sys-fs-binfmt_misc.automount
rm -rf $(ROOTFS)/lib/modules-load.d/open-iscsi.conf
$(foreach unit,$(units),rm -f $(ROOTFS)/lib/systemd/system/multi-user.target.wants/$(unit);)
$(foreach unit,$(units),rm -f $(ROOTFS)/etc/systemd/system/multi-user.target.wants/$(unit);)
$(foreach unit,$(units),rm -f $(ROOTFS)/etc/systemd/system/sysinit.target.wants/$(unit);)
@@ -314,7 +314,7 @@ func getLibcontainerConfig(containerID, rootfs string, cfg Config) (*configs.Con
Source: "sysfs",
Destination: "/sys",
Device: "sysfs",
Flags: defaultMountFlags | syscall.MS_RDONLY,
Flags: defaultMountFlags,
},
{
Source: "devpts",
@@ -340,6 +340,23 @@ func getLibcontainerConfig(containerID, rootfs string, cfg Config) (*configs.Con
Destination: "/dev/kmsg",
Flags: syscall.MS_BIND,
},
// /run has to be mounted explicitly as tmpfs in order to be able
// to mount /run/udev below
{
Source: "tmpfs",
Destination: "/run",
Device: "tmpfs",
Flags: syscall.MS_NOSUID | syscall.MS_NODEV,
Data: "mode=755",
},
// /run/udev is used by OpenEBS node device manager to detect
// added and removed block devices
{
Device: "bind",
Source: "/run/udev",

This comment has been minimized.

Copy link
@a-palchikov

a-palchikov Dec 6, 2019

Contributor

Hmm, I've seen errors from systemd-udevd related to this in robotest logs:

-- Logs begin at Fri 2019-12-06 04:36:50 CET, end at Fri 2019-12-06 04:38:43 CET. --
Dez 06 04:36:56 robotest-52efe7d3-node-0 systemd[1]: Starting udev Kernel Device Manager...
Dez 06 04:36:56 robotest-52efe7d3-node-0 systemd[1]: Started udev Kernel Device Manager.
Dez 06 04:36:56 robotest-52efe7d3-node-0 systemd-udevd[571]: could not unlink /run/udev/queue: No such file or directory
Dez 06 04:36:56 robotest-52efe7d3-node-0 systemd-udevd[571]: could not unlink /run/udev/queue: No such file or directory
Dez 06 04:36:56 robotest-52efe7d3-node-0 systemd-udevd[571]: could not unlink /run/udev/queue: No such file or directory
Dez 06 04:36:56 robotest-52efe7d3-node-0 systemd-udevd[571]: could not unlink /run/udev/queue: No such file or directory
Dez 06 04:36:56 robotest-52efe7d3-node-0 systemd-udevd[571]: could not unlink /run/udev/queue: No such file or directory
Dez 06 04:36:56 robotest-52efe7d3-node-0 systemd-udevd[571]: could not unlink /run/udev/queue: No such file or directory
Dez 06 04:36:57 robotest-52efe7d3-node-0 systemd-udevd[571]: could not unlink /run/udev/queue: No such file or directory
Dez 06 04:36:57 robotest-52efe7d3-node-0 systemd-udevd[571]: could not unlink /run/udev/queue: No such file or directory
Dez 06 04:36:57 robotest-52efe7d3-node-0 systemd-udevd[571]: could not unlink /run/udev/queue: No such file or directory
Dez 06 04:36:57 robotest-52efe7d3-node-0 systemd-udevd[571]: could not unlink /run/udev/queue: No such file or directory
...

Not sure if these are related though - need more time to look into this.

This comment has been minimized.

Copy link
@r0mant

r0mant Dec 6, 2019

Author Contributor

I don't think these affect anything (I get the same warnings on host) but this actually reminded me that udevd should not be running inside planet. Previously it wasn't launched b/c /sys was mounted as read-only, but now it's read-write cause OpenEBS needs it. I've masked systemd-udevd service like iscsi services above.

Destination: "/run/udev",
Flags: syscall.MS_BIND,
},
},
Cgroups: &configs.Cgroup{
Name: fmt.Sprintf("planet-%v", containerID),
@@ -391,10 +408,11 @@ func getLibcontainerConfig(containerID, rootfs string, cfg Config) (*configs.Con
targetPath = match
}
mount := &configs.Mount{
Device: "bind",
Source: match,
Destination: targetPath,
Flags: syscall.MS_BIND,
Device: "bind",
Source: match,
Destination: targetPath,
Flags: syscall.MS_BIND,
PropagationFlags: []int{syscall.MS_SHARED},
}
if mountSpec.Readonly {
mount.Flags |= syscall.MS_RDONLY
@@ -407,7 +407,7 @@ const (
DefaultVxlanPort = 8472

// DefaultFeatureGates is the default set of component feature gates
DefaultFeatureGates = "AllAlpha=true,APIResponseCompression=false,BoundServiceAccountTokenVolume=false,CSIMigration=false,KubeletPodResources=false,EndpointSlice=false,IPv6DualStack=false"
DefaultFeatureGates = "AllAlpha=true,APIResponseCompression=false,BoundServiceAccountTokenVolume=false,CSIMigration=false,KubeletPodResources=false,EndpointSlice=false,IPv6DualStack=false,RemoveSelfLink=false"

// DefaultServiceNodePortRange defines the default IP range for services with NodePort visibility
DefaultServiceNodePortRange = "30000-32767"
@@ -37,6 +37,7 @@ func newUdevListener(rootfs, socketPath string) (*udevListener, error) {
doneC := make(chan struct{})

monitor.FilterAddMatchSubsystemDevtype("block", "disk")
monitor.FilterAddMatchSubsystemDevtype("block", "partition")
monitor.FilterAddMatchTag("systemd")

recvC, err := monitor.DeviceChan(doneC)
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.