export node's IP addresses as env vars

README.md

@@ -100,10 +100,29 @@ We store a bunch of k8s meta data inside the unit in a `[X-kubernetes]` section.
 know a pod state systemk will query systemd and read the unit file back. This way we know the status
 and have access to all the meta data, like pod UID and if the unit is an init container.
 
-### Limitations
+### Addresses
+
+Addressses are configured with one the systemk commandline flags: `--node-ip` and
+`--node-external-ip`, these may be IPv4 or IPv6. In the future this may get expanded into allow both
+(i.e. dual stack support). The primary Pod address will be the value from `--node-external-ip`.
+
+If `--node-ip` is not given, systemk will try to find a RFC 1918 address on the interfaces use the
+first one found.
+
+If `--node-external-ip` is not given, systemk will try to find a non-RFC 1918 address on the
+interfaces and use the first one found.
+
+If after all this one of the values is still not found, the other existing value will be copied, i.e
+internal == external in that case. If both were empty systemk exists with a fatal error.
+
+### Environment Variables
+
+The following environment variables are made available to the units:
+
+* `HOSTNAME`, `KUBERNETES_SERVICE_PORT` and `KUBERNETES_SERVICE_HOST` (same as the kubelet).
+* `SYSTEMK_NODE_INTERNAL_IP` the internal IP address.
+* `SYSTEMK_NODE_EXTERNAL_IP` the external IP address.
 
-By using systemd and the hosts network we have weak isolation between pods, i.e. no more than
-process isolation. Starting two pods that use the same port is guaranteed to fail for one.
 
 ### Using username in securityContext
 
@@ -118,6 +137,12 @@ The primary group will be found by systemk and both a `User` and `Group` will be
 unit file. The files created on disk for the configMap/secrets/emptyDir will be made of the same
 user/group.
 
+
+### Limitations
+
+By using systemd and the hosts network we have weak isolation between pods, i.e. no more than
+process isolation. Starting two pods that use the same port is guaranteed to fail for one.
+
 ## Use with K3S
 
 Download k3s from it's releases on GitHub, you just need the `k3s` binary. Use the `k3s/k3s` shell

k3s/prometheus.yaml

@@ -54,11 +54,19 @@ spec:
     windowsOptions:
       runAsUserName: "prometheus"
   serviceAccountName: prometheus
+  initContainers:
+  - name: init-prometheus-config
+    image: bash
+    command: ["bash", "-c"]
+    args: ["envsubst '$SYSTEMK_NODE_INTERNAL_IP' < /tmp/init-prometheus/prometheus.yaml.orig > /tmp/init-prometheus/prometheus.yaml"]
+    volumeMounts:
+    - name: prometheus-config-volume
+      mountPath: /tmp/init-prometheus
   containers:
   - name: prometheus
     image: deb://www.miek.nl/prometheus_2.23.0-0~20.040_amd64.deb
     args:
-    - "--config.file=/etc/prometheus/prometheus.yml"
+    - "--config.file=/etc/prometheus/prometheus.yaml"
     - "--storage.tsdb.path=/tmp/prometheus"
     ports:
     - containerPort: 9090
@@ -82,7 +90,7 @@ metadata:
   name: prometheus-server-conf
   namespace: monitoring
 data:
-  prometheus.yml: |-
+  prometheus.yaml.orig: |-
     scrape_configs:
     - job_name: 'kubernetes-pods'
       kubernetes_sd_configs:
@@ -97,4 +105,11 @@ data:
       - source_labels: [__meta_kubernetes_pod_name]
         action: replace
         target_label: kubernetes_pod_name
----
+      - source_labels: [__address__]
+        regex: '.*:(\d+)'
+        action: replace
+        target_label: __address__
+        replacement: '${SYSTEMK_NODE_INTERNAL_IP}:$1'
+      - source_labels: [__meta_kubernetes_pod_container_init]
+        regex: 'true'
+        action: drop

main.go

@@ -40,10 +40,15 @@ func main() {
 	var (
 		certFile string
 		keyFile  string
+		nodeIP   string
+		nodeEIP  string
 	)
 	flags := pflag.NewFlagSet("client", pflag.ContinueOnError)
+	// these options need to be make inline with k3s or make clear they afffect logs, certfile and keyfile is too short
 	flags.StringVar(&certFile, "certfile", "", "certfile")
 	flags.StringVar(&keyFile, "keyfile", "", "keyfile")
+	flags.StringVarP(&nodeIP, "node-ip", "i", "", "IP address to advertise for node")
+	flags.StringVar(&nodeEIP, "node-external-ip", "", "External IP address to advertise for node")
 
 	ctx := cli.ContextWithCancelOnSignal(context.Background())
 
@@ -54,7 +59,6 @@ func main() {
 	o.Provider = "systemd"
 	o.Version = strings.Join([]string{k8sVersion, "vk-systemd", buildVersion}, "-")
 	o.NodeName = system.Hostname()
-
 	node, err := cli.New(ctx,
 		cli.WithBaseOpts(o),
 		cli.WithPersistentFlags(flags),
@@ -66,8 +70,11 @@ func main() {
 			if err != nil {
 				return p, err
 			}
+			p.SetNodeIPs(nodeIP, nodeEIP)
+			klog.Infof("Using internal/external IP addresses: %s/%s", p.NodeInternalIP.Address, p.NodeExternalIP.Address)
+
 			if certFile == "" || keyFile == "" {
-				klog.Info("Not certificates found, disabling GetContainerLogs")
+				klog.Info("No certificates found, disabling GetContainerLogs")
 				return p, nil
 			}
 

pkg/manager/manager.go

@@ -198,7 +198,7 @@ func (m *UnitManager) States(prefix string) (map[string]*unit.State, error) {
 	if err != nil {
 		return nil, err
 	}
-	klog.Infof("%d statusses for prefix %q returned", len(dbusStatuses), prefix)
+	klog.Infof("%d statuses returned", len(dbusStatuses))
 
 	states := make(map[string]*unit.State)
 	for _, dus := range dbusStatuses {

pkg/system/system.go

@@ -14,7 +14,6 @@ import (
 var (
 	// this is a variable so it can be overridden during unit-testing.
 	osReleaseFilePath = "/etc/os-release"
-
 )
 
 // Memory returns the amount of memory in the system.

systemd/env.go

@@ -2,6 +2,8 @@ package systemd
 
 import (
 	"fmt"
+	"net"
+	"net/url"
 
 	"github.com/virtual-kubelet/systemk/pkg/system"
 )
@@ -10,8 +12,28 @@ import (
 // It returns a list of strings VAR=VALUE.
 func (p *P) defaultEnvironment() []string {
 	env := []string{}
+
+	host := "127.0.0.1"
+	port := "6444"
+	if p.Host != "" {
+		url, _ := url.Parse(p.Host)
+		host, port, _ = net.SplitHostPort(url.Host)
+	}
+
 	env = append(env, fmt.Sprintf("HOSTNAME=%s", system.Hostname()))
-	env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_PORT=%d", 6444))        // get from provider/flag?
-	env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_HOST=%s", "127.0.0.1")) // get from provider/flag?
+	env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_PORT=%s", port))
+	env = append(env, fmt.Sprintf("KUBERNETES_SERVICE_HOST=%s", host))
+
+	// These are systemk spefific environment variables. TODO(miek): should this be done at all?
+	// SYSTEMK_NODE_INTERNAL_IP: internal address of the node
+	// SYSTEMK_NODE_EXTERNAL_IP: external address of the node
+	env = append(env, mkEnvVar("NODE_INTERNAL_IP", p.NodeInternalIP.Address))
+	env = append(env, mkEnvVar("NODE_EXTERNAL_IP", p.NodeExternalIP.Address))
+
 	return env
 }
+
+func mkEnvVar(name, value string) string {
+	const s = "SYSTEMK_"
+	return s + name + "=" + value
+}

systemd/env_test.go

@@ -0,0 +1,28 @@
+package systemd
+
+import (
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+func TestProviderIPEnvironment(t *testing.T) {
+	p := new(P)
+	p.NodeInternalIP = &corev1.NodeAddress{Address: "192.168.1.1", Type: corev1.NodeInternalIP}
+	p.NodeExternalIP = &corev1.NodeAddress{Address: "172.16.0.1", Type: corev1.NodeExternalIP}
+
+	env := p.defaultEnvironment()
+	found := 0
+	for _, e := range env {
+		println(e)
+		if e == "SYSTEMK_NODE_INTERNAL_IP=192.168.1.1" {
+			found++
+		}
+		if e == "SYSTEMK_NODE_EXTERNAL_IP=172.16.0.1" {
+			found++
+		}
+	}
+	if found != 2 {
+		t.Errorf("failed to find SYSTEMK_NODE_INTERNAL_IP or SYSTEMK_NODE_EXTERNAL_IP")
+	}
+}

systemd/node.go

@@ -10,17 +10,17 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const(
-	defaultOS="Linux"
+const (
+	defaultOS = "Linux"
 )
 
 // ConfigureNode enables a provider to configure the node object that will be used for Kubernetes.
 func (p *P) ConfigureNode(ctx context.Context, node *corev1.Node) {
-	node.Status.Capacity = p.capacity()
-	node.Status.Allocatable = p.capacity()
-	node.Status.Conditions = p.nodeConditions()
-	node.Status.Addresses = p.nodeAddresses()
-	node.Status.DaemonEndpoints = p.nodeDaemonEndpoints()
+	node.Status.Capacity = capacity()
+	node.Status.Allocatable = capacity()
+	node.Status.Conditions = nodeConditions()
+	node.Status.Addresses = append([]corev1.NodeAddress{*p.NodeInternalIP}, *p.NodeExternalIP)
+	node.Status.DaemonEndpoints = nodeDaemonEndpoints(p.DaemonPort)
 	node.Status.NodeInfo.OperatingSystem = defaultOS
 	node.Status.NodeInfo.KernelVersion = system.Kernel()
 	node.Status.NodeInfo.OSImage = system.Image()
@@ -35,56 +35,51 @@ func (p *P) ConfigureNode(ctx context.Context, node *corev1.Node) {
 			corev1.LabelZoneRegionStable:        system.Hostname(),
 		},
 	}
-	p.Addresses = node.Status.Addresses
 }
 
-// nodeAddresses returns a list of addresses for the node status within Kubernetes.
-func (p *P) nodeAddresses() []corev1.NodeAddress {
+// nodeAddresses finds the internal and external address of the node (if found).
+func nodeAddresses() (internal, external *corev1.NodeAddress) {
 	cidrs := []string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"}
 	rfc1918 := make([]*net.IPNet, len(cidrs))
 	for i, cidr := range cidrs {
 		_, block, _ := net.ParseCIDR(cidr)
 		rfc1918[i] = block
 	}
 
-	ips := system.IPs()
-	na := make([]corev1.NodeAddress, len(ips))
-	for i, ip := range ips {
+	for _, ip := range system.IPs() {
 		for _, block := range rfc1918 {
-			na[i] = corev1.NodeAddress{
-				Address: ip.String(),
-				Type:    corev1.NodeExternalIP,
-			}
 			if block.Contains(ip) {
-				na[i].Type = corev1.NodeInternalIP
+				if internal == nil {
+					internal = &corev1.NodeAddress{Address: ip.String(), Type: corev1.NodeInternalIP}
+				}
+				continue
+			}
+
+			if external == nil {
+				external = &corev1.NodeAddress{Address: ip.String(), Type: corev1.NodeExternalIP}
 			}
 		}
 	}
-	// corev1.NodeInternalDNS??
-	return na
+	return internal, external
 }
 
 // nodeDaemonEndpoints returns NodeDaemonEndpoints for the node status within Kubernetes.
-func (p *P) nodeDaemonEndpoints() corev1.NodeDaemonEndpoints {
-	// use for logs
-	return corev1.NodeDaemonEndpoints{
-		KubeletEndpoint: corev1.DaemonEndpoint{
-			Port: p.DaemonPort,
-		},
-	}
+func nodeDaemonEndpoints(port int32) corev1.NodeDaemonEndpoints {
+	// used for logs
+	return corev1.NodeDaemonEndpoints{KubeletEndpoint: corev1.DaemonEndpoint{Port: port}}
 }
 
 // capacity returns a resource list containing the capacity limits set for Zun.
-func (p *P) capacity() corev1.ResourceList {
+func capacity() corev1.ResourceList {
 	return corev1.ResourceList{
 		"cpu":     resource.MustParse(system.CPU()),
 		"memory":  resource.MustParse(system.Memory()),
 		"pods":    resource.MustParse(system.Pid()),
-		"storage": resource.MustParse("40G"), // We're using tmpfs _a_ lot.. so
+		"storage": resource.MustParse("40G"), // needs the size of the /var FS
 	}
 }
 
-func (p *P) nodeConditions() []corev1.NodeCondition {
+func nodeConditions() []corev1.NodeCondition {
 	return []corev1.NodeCondition{
 		{
 			Type:               "Ready",
@@ -136,19 +131,3 @@ func (p *P) nodeConditions() []corev1.NodeCondition {
 		},
 	}
 }
-
-// externalOrInternalAddress prefers to return the external address of the node, if not available
-// an internal address will be returned. If neither is found a 127.0.0.1 node address is synthesized.
-func externalOrInternalAddress(addrs []corev1.NodeAddress) corev1.NodeAddress {
-	for _, a := range addrs {
-		if a.Type == corev1.NodeExternalIP {
-			return a
-		}
-	}
-	for _, a := range addrs {
-		if a.Type == corev1.NodeInternalIP {
-			return a
-		}
-	}
-	return corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: "127.0.0.1"}
-}