Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1444 from celebdor/kubelet-nodeip-fix
Bug 1798788: Set Kubelet node IP to non-vip
- Loading branch information
Showing
5 changed files
with
199 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
filesystem: "root" | ||
mode: 0755 | ||
path: "/usr/local/bin/nodeip-finder" | ||
contents: | ||
inline: | | ||
#!/usr/libexec/platform-python | ||
# /* vim: set filetype=python : */ | ||
"""Writes Kubelet and CRI-O configuration to choose the right IP address | ||
For kubelet, a systemd environment file with a KUBELET_NODE_IP setting | ||
For CRI-O it drops a config file in /etc/crio/crio.conf.d""" | ||
from importlib import util as iutil | ||
from importlib import machinery as imachinery | ||
from types import ModuleType | ||
import os | ||
import pathlib | ||
import socket | ||
import sys | ||
loader = imachinery.SourceFileLoader( | ||
'non_virtual_ip', | ||
os.path.join(os.path.dirname(os.path.realpath(__file__)), 'non_virtual_ip')) | ||
spec = iutil.spec_from_loader('non_virtual_ip', loader) | ||
non_virtual_ip = iutil.module_from_spec(spec) | ||
loader.exec_module(non_virtual_ip) | ||
KUBELET_WORKAROUND_PATH = '/etc/systemd/system/kubelet.service.d/20-nodenet.conf' | ||
CRIO_WORKAROUND_PATH = '/etc/systemd/system/crio.service.d/20-nodenet.conf' | ||
def first_candidate_addr(api_vip: str) -> non_virtual_ip.Address: | ||
filters = (non_virtual_ip.non_host_scope, | ||
non_virtual_ip.non_deprecated, | ||
non_virtual_ip.non_secondary) | ||
iface_addrs = list(non_virtual_ip.interface_addrs(filters)) | ||
subnet, candidates = non_virtual_ip.vip_subnet_and_addrs_in_it(api_vip, iface_addrs) | ||
sys.stderr.write('VIP Subnet %s\n' % subnet.cidr) | ||
for addr in candidates: | ||
return addr | ||
raise non_virtual_ip.AddressNotFoundException() | ||
def main() -> None: | ||
if len(sys.argv) > 1: | ||
api_vip = sys.argv[1] | ||
else: | ||
api_int_name = os.getenv('API_INT') | ||
try: | ||
sstream_tuple = socket.getaddrinfo(api_int_name, None)[0] | ||
_, _, _, _, sockaddr = sstream_tuple | ||
api_vip = sockaddr[0] | ||
sys.stderr.write(f'Found {api_int_name} to resolve to {api_vip}\n') | ||
except socket.gaierror: | ||
sys.stderr.write(f'api-int VIP not provided and failed to resolve {api_int_name}\n') | ||
sys.exit(1) | ||
try: | ||
first: non_virtual_ip.Address = first_candidate_addr(api_vip) | ||
prefixless = first.cidr.split('/')[0] | ||
# Kubelet | ||
with open(KUBELET_WORKAROUND_PATH, 'w') as kwf: | ||
print(f'[Service]\nEnvironment="KUBELET_NODE_IP={prefixless}"', file=kwf) | ||
# CRI-O | ||
crio_confd = pathlib.Path(CRIO_WORKAROUND_PATH).parent | ||
crio_confd.mkdir(parents=True, exist_ok=True) | ||
with open(CRIO_WORKAROUND_PATH, 'w') as cwf: | ||
print(f'[Service]\nEnvironment="CONTAINER_STREAM_ADDRESS={prefixless}"', file=cwf) | ||
except (non_virtual_ip.AddressNotFoundException, non_virtual_ip.SubnetNotFoundException): | ||
sys.stderr.write('Failed to find suitable node ip') | ||
sys.exit(1) | ||
if __name__ == '__main__': | ||
main() |
20 changes: 20 additions & 0 deletions
20
templates/common/baremetal/units/nodeip-configuration.service
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
name: "nodeip-configuration.service" | ||
enabled: true | ||
contents: | | ||
[Unit] | ||
Description=Writes IP address configuration so that kubelet and crio services select a valid node IP | ||
# This only applies to VIP managing environments where the kubelet and crio IP | ||
# address picking logic is flawed and may end up selecting an address from a | ||
# different subnet or a deprecated address | ||
Wants=network-online.target | ||
After=network-online.target ignition-firstboot-complete.service | ||
Before=kubelet.service crio.service | ||
|
||
[Service] | ||
# Need oneshot to delay kubelet | ||
Type=oneshot | ||
ExecStart=/usr/local/bin/nodeip-finder {{.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIP }} | ||
|
||
[Install] | ||
WantedBy=multi-user.target | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
38 changes: 38 additions & 0 deletions
38
templates/worker/01-worker-kubelet/baremetal/units/kubelet.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
name: "kubelet.service" | ||
enabled: true | ||
contents: | | ||
[Unit] | ||
Description=Kubernetes Kubelet | ||
Wants=rpc-statd.service network-online.target crio.service | ||
After=network-online.target crio.service | ||
[Service] | ||
Type=notify | ||
ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests | ||
ExecStartPre=/bin/rm -f /var/lib/kubelet/cpu_manager_state | ||
Environment="KUBELET_LOG_LEVEL=3" | ||
EnvironmentFile=/etc/os-release | ||
EnvironmentFile=-/etc/kubernetes/kubelet-workaround | ||
EnvironmentFile=-/etc/kubernetes/kubelet-env | ||
ExecStart=/usr/bin/hyperkube \ | ||
kubelet \ | ||
--config=/etc/kubernetes/kubelet.conf \ | ||
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ | ||
--kubeconfig=/var/lib/kubelet/kubeconfig \ | ||
--container-runtime=remote \ | ||
--container-runtime-endpoint=/var/run/crio/crio.sock \ | ||
--node-labels=node-role.kubernetes.io/worker,node.openshift.io/os_id=${ID} \ | ||
--node-ip="${KUBELET_NODE_IP}" \ | ||
--address="${KUBELET_NODE_IP}" \ | ||
--minimum-container-ttl-duration=6m0s \ | ||
--volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec \ | ||
--cloud-provider={{cloudProvider .}} \ | ||
{{cloudConfigFlag . }} \ | ||
--v=${KUBELET_LOG_LEVEL} | ||
Restart=always | ||
RestartSec=10 | ||
[Install] | ||
WantedBy=multi-user.target |