Add support for loadbalancer interface (#153)

* Add support for loadbalancer interface

This adds support for the `loadbalancer` interface so that cloud-native
LBs can be provided by the integrator charms. Additionally, it
simplifies the confusing way the relations between the masters and
workers change depending on whether kubeapi-load-balancer is being used
or not by making that use the same `lb-provider` endpoint and always
forwarding the API endpoint URLs via the `kube-control` relation.

Part of [lp:1897818][]
Depends on:
  * juju-solutions/loadbalancer-interface#13
  * juju-solutions/interface-kube-control#33
  * charmed-kubernetes/charm-kubernetes-worker#84
  * charmed-kubernetes/charm-kubeapi-load-balancer#11

[lp:1897818]: https://bugs.launchpad.net/charmed-kubernetes-testing/+bug/1897818

* Fix lint error

* Add test for status reporting of incomplete LB relation

* Fix lint errors in test

* Fix hook error when built with old version of kube-control interface

* Split lb-provider endpoint into separate, more explicit ones for internal / external

* Fix handling of internal LB response and missing endpoints

* Move test to new LB relation pattern and use edge charms

* Improve status reporting around the auth-webhook and skip trying to create secrets before apiserver is available

* Create a ~/.kube/config for the ubuntu user

* Fix NoneType error for old-style relation

* Fix unit test

* Revert drive-by related to auth setup status reporting; splitting to separate bug

lib/charms/layer/kubernetes_master.py

@@ -15,7 +15,7 @@
 from charmhelpers.core.templating import render
 from charmhelpers.core import unitdata
 from charmhelpers.fetch import apt_install
-from charms.reactive import endpoint_from_flag, is_flag_set
+from charms.reactive import endpoint_from_flag, endpoint_from_name, is_flag_set
 from charms.layer import kubernetes_common
 
 
@@ -32,10 +32,9 @@
 db = unitdata.kv()
 
 
-def get_external_lb_endpoints():
+def get_endpoints_from_config():
     """
-    Return a list of any external API load-balancer endpoints that have
-    been manually configured.
+    Return a list of any manually configured API endpoints.
     """
     ha_connected = is_flag_set("ha.connected")
     forced_lb_ips = hookenv.config("loadbalancer-ips").split()
@@ -54,41 +53,109 @@ def get_external_lb_endpoints():
         return []
 
 
-def get_lb_endpoints():
+def get_internal_api_endpoints(relation=None):
     """
-    Return all load-balancer endpoints, whether from manual config or via
-    relation.
+    Determine the best API endpoints for an internal client to connect to.
+
+    If a relation is given, it will try to take that into account.
+
+    May return an empty list if an endpoint is expected but not yet available.
     """
-    external_lb_endpoints = get_external_lb_endpoints()
-    loadbalancer = endpoint_from_flag("loadbalancer.available")
+    try:
+        goal_state = hookenv.goal_state()
+    except NotImplementedError:
+        goal_state = {}
+    goal_state.setdefault("relations", {})
+
+    # Config takes precedence.
+    endpoints_from_config = get_endpoints_from_config()
+    if endpoints_from_config:
+        return endpoints_from_config
+
+    # If the internal LB relation is attached, use that or nothing. If it's
+    # not attached but the external LB relation is, use that or nothing.
+    for lb_type in ("internal", "external"):
+        lb_endpoint = "loadbalancer-" + lb_type
+        request_name = "api-server-" + lb_type
+        if lb_endpoint in goal_state["relations"]:
+            lb_provider = endpoint_from_name(lb_endpoint)
+            lb_response = lb_provider.get_response(request_name)
+            if not lb_response or lb_response.error:
+                return []
+            return [(lb_response.address, STANDARD_API_PORT)]
+
+    # Support the older loadbalancer relation (public-address interface).
+    if "loadbalancer" in goal_state["relations"]:
+        loadbalancer = endpoint_from_name("loadbalancer")
+        lb_addresses = loadbalancer.get_addresses_ports()
+        return [(host.get("public-address"), host.get("port")) for host in lb_addresses]
 
-    if external_lb_endpoints:
-        return external_lb_endpoints
-    elif loadbalancer:
+    # No LBs of any kind, so fall back to ingress-address.
+    if not relation:
+        kube_control = endpoint_from_name("kube-control")
+        if not kube_control.relations:
+            return []
+        relation = kube_control.relations[0]
+    ingress_address = hookenv.ingress_address(
+        relation.relation_id, hookenv.local_unit()
+    )
+    return [(ingress_address, STANDARD_API_PORT)]
+
+
+def get_external_api_endpoints():
+    """
+    Determine the best API endpoints for an external client to connect to.
+
+    May return an empty list if an endpoint is expected but not yet available.
+    """
+    try:
+        goal_state = hookenv.goal_state()
+    except NotImplementedError:
+        goal_state = {}
+    goal_state.setdefault("relations", {})
+
+    # Config takes precedence.
+    endpoints_from_config = get_endpoints_from_config()
+    if endpoints_from_config:
+        return endpoints_from_config
+
+    # If the external LB relation is attached, use that or nothing. If it's
+    # not attached but the internal LB relation is, use that or nothing.
+    for lb_type in ("external", "internal"):
+        lb_endpoint = "loadbalancer-" + lb_type
+        lb_name = "api-server-" + lb_type
+        if lb_endpoint in goal_state["relations"]:
+            lb_provider = endpoint_from_name(lb_endpoint)
+            lb_response = lb_provider.get_response(lb_name)
+            if not lb_response or lb_response.error:
+                return []
+            return [(lb_response.address, STANDARD_API_PORT)]
+
+    # Support the older loadbalancer relation (public-address interface).
+    if "loadbalancer" in goal_state["relations"]:
+        loadbalancer = endpoint_from_flag("loadbalancer.available")
         lb_addresses = loadbalancer.get_addresses_ports()
         return [(host.get("public-address"), host.get("port")) for host in lb_addresses]
-    else:
-        return []
 
+    # No LBs of any kind, so fall back to public-address.
+    return [(hookenv.unit_public_ip(), STANDARD_API_PORT)]
 
-def get_api_endpoint(relation=None):
+
+def get_api_urls(endpoints):
+    """
+    Convert a list of API server endpoints to URLs.
     """
-    Determine the best endpoint for a client to connect to.
+    return ["https://{0}:{1}".format(*endpoint) for endpoint in endpoints]
 
-    If a relation is given, it will take that into account when choosing an
-    endpoint.
+
+def get_api_url(endpoints):
     """
-    endpoints = get_lb_endpoints()
-    if endpoints:
-        # select a single endpoint based on our local unit number
-        return endpoints[kubernetes_common.get_unit_number() % len(endpoints)]
-    elif relation:
-        ingress_address = hookenv.ingress_address(
-            relation.relation_id, hookenv.local_unit()
-        )
-        return (ingress_address, STANDARD_API_PORT)
-    else:
-        return (hookenv.unit_public_ip(), STANDARD_API_PORT)
+    Choose an API endpoint from the list and build a URL from it.
+    """
+    if not endpoints:
+        return None
+    urls = get_api_urls(endpoints)
+    return urls[kubernetes_common.get_unit_number() % len(urls)]
 
 
 def install_ceph_common():

metadata.yaml

@@ -29,6 +29,13 @@ peers:
     interface: kube-masters
 provides:
   kube-api-endpoint:
+    # Use of this relation is strongly discouraged as the API endpoints will be
+    # provided via the kube-control relation. However, it can be used to
+    # override those endpoints if you need to inject a reverse proxy between
+    # the master and workers using a charm which only supports the old MITM
+    # style relations. Note, though, that since this reverse proxy will not be
+    # visible to the master, it will not be used in any of the client or
+    # component kube config files.
     interface: http
   cluster-dns:
     # kube-dns is deprecated. Its functionality has been rolled into the
@@ -51,6 +58,8 @@ requires:
   etcd:
     interface: etcd
   loadbalancer:
+    # Use of this relation is strongly discouraged in favor of the more
+    # explicit loadbalancer-internal / loadbalancer-external relations.
     interface: public-address
   ceph-storage:
     interface: ceph-admin
@@ -70,6 +79,14 @@ requires:
     interface: keystone-credentials
   dns-provider:
     interface: kube-dns
+  loadbalancer-internal:
+    # Indicates that the LB should not be public and should use internal
+    # networks if available. Intended for control plane and other internal use.
+    interface: loadbalancer
+  loadbalancer-external:
+    # Indicates that the LB should be public facing. Intended for clients which
+    # must reach the API server via external networks.
+    interface: loadbalancer
 resources:
   core:
     type: file