Skip to content

Commit

Permalink
k8s_atomic: Run all syscontainer with podman
Browse files Browse the repository at this point in the history 
Using the atomic cli to install kubelet breaks mount
propagation of secrets, configmaps and so on. Using podman
in a systemd unit works.

Additionally, with this change all atomic commands are dropped,
containers are pulled from gcr.io (ofiicial kubernetes containers).

Finally, after this patch only by starting the heat-agent with
ignition, we can use fedora coreos as a drop-in replacement.

* Drop del of docker0
  This command to remove docker0 is carried from
  earlier versions of docker. This is not an issue
  anymore.

story: 2006459
task: 36871

Change-Id: I2ed8e02f5295e48d371ac9e1aff2ad5d30d0c2bd
Signed-off-by: Spyros Trigazis <spyridon.trigazi@cern.ch>
  • Loading branch information
@strigazi
Spyros Trigazis authored and strigazi committed Oct 8, 2019
1 parent 2f72fdf commit 3674b36
Show file tree
Hide file tree
Showing 8 changed files with 417 additions and 103 deletions.
111 changes: 82 additions & 29 deletions magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh
View file
Expand Up @@ -50,12 +50,30 @@ if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then

fi

_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
$ssh_cmd atomic install \
--system-package no \
--system \
--storage ostree \
--name=etcd ${_prefix}etcd:${ETCD_TAG}
cat > /etc/systemd/system/etcd.service <<EOF
[Unit]
Description=Etcd server
After=network-online.target
Wants=network-online.target
[Service]
ExecStartPre=mkdir -p /var/lib/etcd
ExecStartPre=-/bin/podman rm etcd
ExecStart=/bin/podman run \\
--name etcd \\
--volume /etc/pki/ca-trust/extracted/pem:/etc/ssl/certs:ro,z \\
--volume /etc/etcd:/etc/etcd:ro,z \\
--volume /var/lib/etcd:/var/lib/etcd:rshared,z \\
--net=host \\
${CONTAINER_INFRA_PREFIX:-"k8s.gcr.io/"}etcd:${ETCD_TAG} \\
/usr/local/bin/etcd \\
--config-file /etc/etcd/etcd.conf.yaml
ExecStop=/bin/podman stop etcd
[Install]
WantedBy=multi-user.target
EOF


if [ -z "$KUBE_NODE_IP" ]; then
# FIXME(yuanying): Set KUBE_NODE_IP correctly
Expand All @@ -70,34 +88,69 @@ if [ "$TLS_DISABLED" = "True" ]; then
protocol="http"
fi

cat > /etc/etcd/etcd.conf <<EOF
ETCD_NAME="$myip"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
ETCD_LISTEN_PEER_URLS="$protocol://$myip:2380"
cat > /etc/etcd/etcd.conf.yaml <<EOF
# This is the configuration file for the etcd server.
ETCD_ADVERTISE_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="$protocol://$myip:2380"
ETCD_DISCOVERY="$ETCD_DISCOVERY_URL"
EOF
# Human-readable name for this member.
name: "${INSTANCE_NAME}"
if [ "$TLS_DISABLED" = "False" ]; then
# Path to the data directory.
data-dir: /var/lib/etcd/default.etcd
# List of comma separated URLs to listen on for peer traffic.
listen-peer-urls: "$protocol://$myip:2380"
# List of comma separated URLs to listen on for client traffic.
listen-client-urls: "$protocol://$myip:2379,http://127.0.0.1:2379"
# List of this member's peer URLs to advertise to the rest of the cluster.
# The URLs needed to be a comma-separated list.
initial-advertise-peer-urls: "$protocol://$myip:2380"
# List of this member's client URLs to advertise to the public.
# The URLs needed to be a comma-separated list.
advertise-client-urls: "$protocol://$myip:2379,http://127.0.0.1:2379"
# Discovery URL used to bootstrap the cluster.
discovery: "$ETCD_DISCOVERY_URL"
cat >> /etc/etcd/etcd.conf <<EOF
ETCD_CA_FILE=$cert_dir/ca.crt
ETCD_TRUSTED_CA_FILE=$cert_dir/ca.crt
ETCD_CERT_FILE=$cert_dir/server.crt
ETCD_KEY_FILE=$cert_dir/server.key
ETCD_CLIENT_CERT_AUTH=true
ETCD_PEER_CA_FILE=$cert_dir/ca.crt
ETCD_PEER_TRUSTED_CA_FILE=$cert_dir/ca.crt
ETCD_PEER_CERT_FILE=$cert_dir/server.crt
ETCD_PEER_KEY_FILE=$cert_dir/server.key
ETCD_PEER_CLIENT_CERT_AUTH=true
EOF

if [ -n "$HTTP_PROXY" ]; then
cat >> /etc/etcd/etcd.conf.yaml <<EOF
# HTTP proxy to use for traffic to discovery service.
discovery-proxy: $HTTP_PROXY
EOF
fi

if [ -n "$HTTP_PROXY" ]; then
echo "ETCD_DISCOVERY_PROXY=$HTTP_PROXY" >> /etc/etcd/etcd.conf
if [ "$TLS_DISABLED" = "False" ]; then

cat >> /etc/etcd/etcd.conf.yaml <<EOF
client-transport-security:
# Path to the client server TLS cert file.
cert-file: $cert_dir/server.crt
# Path to the client server TLS key file.
key-file: $cert_dir/server.key
# Enable client cert authentication.
client-cert-auth: true
# Path to the client server TLS trusted CA cert file.
trusted-ca-file: $cert_dir/ca.crt
peer-transport-security:
# Path to the peer server TLS cert file.
cert-file: $cert_dir/server.crt
# Path to the peer server TLS key file.
key-file: $cert_dir/server.key
# Enable peer client cert authentication.
client-cert-auth: true
# Path to the peer server TLS trusted CA cert file.
trusted-ca-file: $cert_dir/ca.crt
EOF
fi
206 changes: 190 additions & 16 deletions magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
View file
Expand Up @@ -21,14 +21,11 @@ if [ ! -z "$NO_PROXY" ]; then
export NO_PROXY
fi

_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}

$ssh_cmd rm -rf /etc/cni/net.d/*
$ssh_cmd rm -rf /var/lib/cni/*
$ssh_cmd rm -rf /opt/cni/*
$ssh_cmd mkdir -p /opt/cni
$ssh_cmd mkdir -p /opt/cni/bin
$ssh_cmd mkdir -p /etc/cni/net.d/
_addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]},{"type":"bind","source":"/var/lib/docker","destination":"/var/lib/docker","options":["bind","rw","slave","mode=755"]}'

if [ "$NETWORK_DRIVER" = "calico" ]; then
echo "net.ipv4.conf.all.rp_filter = 1" >> /etc/sysctl.conf
Expand All @@ -49,16 +46,193 @@ fi


mkdir -p /srv/magnum/kubernetes/
cat > /srv/magnum/kubernetes/install-kubernetes.sh <<EOF
#!/bin/bash -x
atomic install --storage ostree --system --set=ADDTL_MOUNTS='${_addtl_mounts}' --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG}
cat > /etc/kubernetes/config <<EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=3"
KUBE_MASTER="--master=http://127.0.0.1:8080"
EOF
cat > /etc/kubernetes/kubelet <<EOF
KUBELET_ARGS="--fail-swap-on=false"
EOF

cat > /etc/kubernetes/apiserver <<EOF
KUBE_API_ADDRESS="--insecure-bind-address=127.0.0.1"
KUBE_ETCD_SERVERS="--etcd-servers=http://127.0.0.1:2379,http://127.0.0.1:4001"
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
KUBE_API_ARGS=""
EOF

cat > /etc/kubernetes/controller-manager <<EOF
KUBE_CONTROLLER_MANAGER_ARGS=""
EOF
cat > /etc/kubernetes/scheduler<<EOF
KUBE_SCHEDULER_ARGS=""
EOF
cat > /etc/kubernetes/proxy <<EOF
KUBE_PROXY_ARGS=""
EOF


cat > /etc/systemd/system/kube-apiserver.service <<EOF
[Unit]
Description=kube-apiserver via Hyperkube
[Service]
EnvironmentFile=/etc/sysconfig/heat-params
EnvironmentFile=/etc/kubernetes/config
EnvironmentFile=/etc/kubernetes/apiserver
ExecStartPre=/bin/mkdir -p /etc/kubernetes/
ExecStartPre=-/usr/bin/podman rm kube-apiserver
ExecStartPre=-/bin/bash -c '/usr/bin/podman run --privileged --user root --net host --rm --volume /usr/local/bin:/host/usr/local/bin \${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} /bin/sh -c "cp /usr/local/bin/kubectl /host/usr/local/bin/kubectl"'
ExecStart=/bin/bash -c '/usr/bin/podman run --name kube-apiserver \\
--net host \\
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
--volume /usr/lib/os-release:/etc/os-release:ro \\
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
--volume /run:/run \\
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
/hyperkube kube-apiserver \\
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBE_ETCD_SERVERS \$KUBE_API_ADDRESS \$KUBE_API_PORT \$KUBELET_PORT \$KUBE_SERVICE_ADDRESSES \$KUBE_ADMISSION_CONTROL \$KUBE_API_ARGS'
ExecStop=-/usr/bin/podman stop kube-apiserver
Delegate=yes
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF

cat > /etc/systemd/system/kube-controller-manager.service <<EOF
[Unit]
Description=kube-controller-manager via Hyperkube
[Service]
EnvironmentFile=/etc/sysconfig/heat-params
EnvironmentFile=/etc/kubernetes/config
EnvironmentFile=/etc/kubernetes/controller-manager
ExecStartPre=/bin/mkdir -p /etc/kubernetes/
ExecStartPre=-/usr/bin/podman rm kube-controller-manager
ExecStart=/bin/bash -c '/usr/bin/podman run --name kube-controller-manager \\
--net host \\
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
--volume /usr/lib/os-release:/etc/os-release:ro \\
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
--volume /run:/run \\
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
/hyperkube kube-controller-manager \\
--secure-port=0 \\
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBE_MASTER \$KUBE_CONTROLLER_MANAGER_ARGS'
ExecStop=-/usr/bin/podman stop kube-controller-manager
Delegate=yes
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF

cat > /etc/systemd/system/kube-scheduler.service <<EOF
[Unit]
Description=kube-scheduler via Hyperkube
[Service]
EnvironmentFile=/etc/sysconfig/heat-params
EnvironmentFile=/etc/kubernetes/config
EnvironmentFile=/etc/kubernetes/scheduler
ExecStartPre=/bin/mkdir -p /etc/kubernetes/
ExecStartPre=-/usr/bin/podman rm kube-scheduler
ExecStart=/bin/bash -c '/usr/bin/podman run --name kube-scheduler \\
--net host \\
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
--volume /usr/lib/os-release:/etc/os-release:ro \\
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
--volume /run:/run \\
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
/hyperkube kube-scheduler \\
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBE_MASTER \$KUBE_SCHEDULER_ARGS'
ExecStop=-/usr/bin/podman stop kube-scheduler
Delegate=yes
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF



cat > /etc/systemd/system/kubelet.service <<EOF
[Unit]
Description=Kubelet via Hyperkube (System Container)
[Service]
EnvironmentFile=/etc/sysconfig/heat-params
EnvironmentFile=/etc/kubernetes/config
EnvironmentFile=/etc/kubernetes/kubelet
ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
ExecStartPre=/bin/mkdir -p /var/lib/calico
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
ExecStartPre=/bin/mkdir -p /opt/cni/bin
ExecStartPre=-/usr/bin/podman rm kubelet
ExecStart=/bin/bash -c '/usr/bin/podman run --name kubelet \\
--privileged \\
--pid host \\
--network host \\
--volume /etc/cni/net.d:/etc/cni/net.d:ro,z \\
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
--volume /usr/lib/os-release:/etc/os-release:ro \\
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
--volume /lib/modules:/lib/modules:ro \\
--volume /run:/run \\
--volume /sys/fs/cgroup:/sys/fs/cgroup:ro \\
--volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \\
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
--volume /var/lib/calico:/var/lib/calico \\
--volume /var/lib/docker:/var/lib/docker \\
--volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \\
--volume /var/log:/var/log \\
--volume /var/run:/var/run \\
--volume /var/run/lock:/var/run/lock:z \\
--volume /opt/cni/bin:/opt/cni/bin:z \\
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
/hyperkube kubelet \\
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBELET_API_SERVER \$KUBELET_ADDRESS \$KUBELET_PORT \$KUBELET_HOSTNAME \$KUBELET_ARGS'
ExecStop=-/usr/bin/podman stop kubelet
Delegate=yes
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF

cat > /etc/systemd/system/kube-proxy.service <<EOF
[Unit]
Description=kube-proxy via Hyperkube
[Service]
EnvironmentFile=/etc/sysconfig/heat-params
EnvironmentFile=/etc/kubernetes/config
EnvironmentFile=/etc/kubernetes/proxy
ExecStartPre=/bin/mkdir -p /etc/kubernetes/
ExecStartPre=-/usr/bin/podman rm kube-proxy
ExecStart=/bin/bash -c '/usr/bin/podman run --name kube-proxy \\
--privileged \\
--net host \\
--volume /etc/kubernetes:/etc/kubernetes:ro,z \\
--volume /usr/lib/os-release:/etc/os-release:ro \\
--volume /etc/ssl/certs:/etc/ssl/certs:ro \\
--volume /run:/run \\
--volume /sys/fs/cgroup:/sys/fs/cgroup:ro \\
--volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \\
--volume /lib/modules:/lib/modules:ro \\
--volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \\
\${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} \\
/hyperkube kube-proxy \\
\$KUBE_LOGTOSTDERR \$KUBE_LOG_LEVEL \$KUBE_MASTER \$KUBE_PROXY_ARGS'
ExecStop=-/usr/bin/podman stop kube-proxy
Delegate=yes
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF
chmod +x /srv/magnum/kubernetes/install-kubernetes.sh
$ssh_cmd "/srv/magnum/kubernetes/install-kubernetes.sh"


CERT_DIR=/etc/kubernetes/certs
Expand Down Expand Up @@ -199,7 +373,7 @@ sed -i '
sed -i '/^KUBE_SCHEDULER_ARGS=/ s/=.*/="--leader-elect=true"/' /etc/kubernetes/scheduler

$ssh_cmd mkdir -p /etc/kubernetes/manifests
KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --cadvisor-port=0 --hostname-override=${INSTANCE_NAME}"
KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${INSTANCE_NAME}"
KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}pause:3.0"
KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
Expand Down Expand Up @@ -281,7 +455,7 @@ fi
KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"

sed -i '
/^KUBELET_ADDRESS=/ s/=.*/="--address=${KUBE_NODE_IP}"/
/^KUBELET_ADDRESS=/ s/=.*/=""/
/^KUBELET_HOSTNAME=/ s/=.*/=""/
/^KUBELET_ARGS=/ s|=.*|="'"\$(/etc/kubernetes/get_require_kubeconfig.sh) ${KUBELET_ARGS}"'"|
/^KUBELET_ARGS=/ s|=.*|="'"${KUBELET_ARGS}"'"|
' /etc/kubernetes/kubelet

0 comments on commit 3674b36

Please sign in to comment.