From 7dacaffc0e1fa7b53b22efecc24102d2e3b0ec98 Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Tue, 6 Apr 2021 16:44:23 -0400
Subject: [PATCH 01/13] Add support for loadbalancer interface

This adds support for the `loadbalancer` interface so that cloud-native
LBs can be provided by the integrator charms. Additionally, it
simplifies the confusing way the relations between the masters and
workers change depending on whether kubeapi-load-balancer is being used
or not by making that use the same `lb-provider` endpoint and always
forwarding the API endpoint URLs via the `kube-control` relation.

Part of [lp:1897818][]
Depends on:
  * https://github.com/juju-solutions/loadbalancer-interface/pull/13
  * https://github.com/juju-solutions/interface-kube-control/pull/33
  * https://github.com/charmed-kubernetes/charm-kubernetes-worker/pull/84
  * https://github.com/charmed-kubernetes/charm-kubeapi-load-balancer/pull/11

[lp:1897818]: https://bugs.launchpad.net/charmed-kubernetes-testing/+bug/1897818
---
 lib/charms/layer/kubernetes_master.py |  6 +-
 metadata.yaml                         |  5 ++
 reactive/kubernetes_master.py         | 91 ++++++++++++++++++++-------
 wheelhouse.txt                        |  1 +
 4 files changed, 81 insertions(+), 22 deletions(-)

diff --git a/lib/charms/layer/kubernetes_master.py b/lib/charms/layer/kubernetes_master.py
index 21da2a27..b4fb74d0 100644
--- a/lib/charms/layer/kubernetes_master.py
+++ b/lib/charms/layer/kubernetes_master.py
@@ -15,7 +15,7 @@
 from charmhelpers.core.templating import render
 from charmhelpers.core import unitdata
 from charmhelpers.fetch import apt_install
-from charms.reactive import endpoint_from_flag, is_flag_set
+from charms.reactive import endpoint_from_flag, endpoint_from_name, is_flag_set
 from charms.layer import kubernetes_common
 
 
@@ -60,10 +60,14 @@ def get_lb_endpoints():
     relation.
     """
     external_lb_endpoints = get_external_lb_endpoints()
+    lb_provider = endpoint_from_name("lb-provider")
+    lb_response = lb_provider.get_response("api-server")
     loadbalancer = endpoint_from_flag("loadbalancer.available")
 
     if external_lb_endpoints:
         return external_lb_endpoints
+    elif lb_response and lb_response.address:
+        return [(lb_response.address, STANDARD_API_PORT)]
     elif loadbalancer:
         lb_addresses = loadbalancer.get_addresses_ports()
         return [(host.get("public-address"), host.get("port")) for host in lb_addresses]
diff --git a/metadata.yaml b/metadata.yaml
index c972c5ce..fa4ed6f5 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -29,6 +29,9 @@ peers:
     interface: kube-masters
 provides:
   kube-api-endpoint:
+    # kube-api-endpoint is deprecated. Its functionality has been rolled into
+    # the kube-control interface. The relation endpoint will be removed in a
+    # future release.
     interface: http
   cluster-dns:
     # kube-dns is deprecated. Its functionality has been rolled into the
@@ -70,6 +73,8 @@ requires:
     interface: keystone-credentials
   dns-provider:
     interface: kube-dns
+  lb-provider:
+    interface: loadbalancer
 resources:
   core:
     type: file
diff --git a/reactive/kubernetes_master.py b/reactive/kubernetes_master.py
index d98903f1..0737eac2 100644
--- a/reactive/kubernetes_master.py
+++ b/reactive/kubernetes_master.py
@@ -37,8 +37,8 @@
 from charms.reactive import hook
 from charms.reactive import remove_state, clear_flag
 from charms.reactive import set_state, set_flag
-from charms.reactive import is_state, is_flag_set, get_unset_flags, all_flags_set
-from charms.reactive import endpoint_from_flag
+from charms.reactive import is_state, is_flag_set, get_unset_flags
+from charms.reactive import endpoint_from_flag, endpoint_from_name
 from charms.reactive import when, when_any, when_not, when_none
 from charms.reactive import register_trigger
 from charms.reactive import data_changed, any_file_changed
@@ -800,13 +800,16 @@ def set_final_status():
         hookenv.status_set("waiting", "Waiting for cloud integration")
         return
 
-    if not is_state("kube-api-endpoint.available"):
-        if "kube-api-endpoint" in goal_state.get("relations", {}):
-            status = "waiting"
-        else:
-            status = "blocked"
-        hookenv.status_set(status, "Waiting for kube-api-endpoint relation")
-        return
+    if "kube-api-endpoint" in goal_state.get("relations", {}):
+        if not is_state("kube-api-endpoint.available"):
+            hookenv.status_set("waiting", "Waiting for kube-api-endpoint relation")
+            return
+
+    if "lb-provider" in goal_state.get("relations", {}):
+        lb_provider = endpoint_from_name("lb-provider")
+        if not lb_provider.has_response:
+            hookenv.status_set("waiting", "Waiting for lb-provider")
+            return
 
     if not is_state("kube-control.connected"):
         if "kube-control" in goal_state.get("relations", {}):
@@ -1405,7 +1408,11 @@ def create_tokens_and_sign_auth_requests():
 @when("kube-api-endpoint.available")
 def push_service_data():
     """Send configuration to the load balancer, and close access to the
-    public interface"""
+    public interface.
+
+    Note: This approach is deprecated in favor of the less complicated
+    lb-provider + kube-control approach.
+    """
     kube_api = endpoint_from_flag("kube-api-endpoint.available")
 
     external_endpoints = kubernetes_master.get_external_lb_endpoints()
@@ -1418,12 +1425,49 @@ def push_service_data():
         kube_api.configure(kubernetes_master.STANDARD_API_PORT)
 
 
-@when("certificates.available", "kube-api-endpoint.available", "cni.available")
+@when("leadership.is_leader")
+@when("endpoint.lb-provider.available")
+@when_not("kubernetes-master.sent-lb-request")
+def request_load_balancer():
+    """Request a LB from the related provider.
+    """
+    lb_provider = endpoint_from_name("lb-provider")
+    req = lb_provider.get_request("api-server")
+    req.protocol = req.protocols.tcp
+    port = kubernetes_master.STANDARD_API_PORT
+    req.port_mapping = {port: port}
+    req.public = True
+    if not req.health_checks:
+        req.add_health_check(
+            protocol=req.protocols.http,
+            port=8080,
+            path="/livez",
+        )
+    lb_provider.send_request(req)
+    set_flag("kubernetes-master.sent-lb-request")
+
+
+@when("kube-control.connected")
+def send_api_endpoints():
+    kube_control = endpoint_from_name("kube-control")
+    lb_provider = endpoint_from_name("lb-provider")
+    if lb_provider.is_available and not lb_provider.has_response:
+        # waiting for lb-provider
+        return
+    endpoints = kubernetes_master.get_lb_endpoints()
+    if not endpoints:
+        for relation in kube_control.relations:
+            endpoints.append(kubernetes_master.get_api_endpoint(relation))
+    kube_control.set_api_endpoints([
+        "https://{}:{}".format(address, port)
+        for address, port in endpoints
+    ])
+
+
+@when("certificates.available", "cni.available")
 def send_data():
     """Send the data that is required to create a server certificate for
     this server."""
-    kube_api_endpoint = endpoint_from_flag("kube-api-endpoint.available")
-
     # Use the public ip of this unit as the Common Name for the certificate.
     common_name = hookenv.unit_public_ip()
 
@@ -1438,7 +1482,8 @@ def send_data():
 
     # Get ingress address (this is probably already covered by bind_ips,
     # but list it explicitly as well just in case it's not).
-    ingress_ip = get_ingress_address(kube_api_endpoint.endpoint_name)
+    old_ingress_ip = get_ingress_address("kube-api-endpoint")
+    new_ingress_ip = get_ingress_address("kube-control")
 
     domain = hookenv.config("dns_domain")
     # Create SANs that the tls layer will add to the server cert.
@@ -1447,7 +1492,8 @@ def send_data():
             # The CN field is checked as a hostname, so if it's an IP, it
             # won't match unless also included in the SANs as an IP field.
             common_name,
-            ingress_ip,
+            old_ingress_ip,
+            new_ingress_ip,
             socket.gethostname(),
             socket.getfqdn(),
             "kubernetes",
@@ -2685,8 +2731,9 @@ def poke_network_unavailable():
     discussion about refactoring the affected code but nothing has happened
     in a while.
     """
-    local_address = get_ingress_address("kube-api-endpoint")
-    local_server = "https://{0}:{1}".format(local_address, 6443)
+    local_address = get_ingress_address("kube-control")
+    local_server = "https://{0}:{1}".format(local_address,
+                                            kubernetes_master.STANDARD_API_PORT)
 
     client_token = get_token("admin")
     http_header = ("Authorization", "Bearer {}".format(client_token))
@@ -3168,11 +3215,11 @@ def configure_hacluster():
         add_service_to_hacluster(service, daemon)
 
     # get a new cert
-    if all_flags_set("certificates.available", "kube-api-endpoint.available"):
+    if is_flag_set("certificates.available"):
         send_data()
 
     # update workers
-    if is_state("kube-api-endpoint.available"):
+    if is_state("kube-control.connected"):
         push_service_data()
 
     set_flag("hacluster-configured")
@@ -3186,10 +3233,12 @@ def remove_hacluster():
         remove_service_from_hacluster(service, daemon)
 
     # get a new cert
-    if all_flags_set("certificates.available", "kube-api-endpoint.available"):
+    if is_flag_set("certificates.available"):
         send_data()
     # update workers
-    if is_state("kube-api-endpoint.available"):
+    if is_flag_set("kube-api-endpoint.available"):
+        push_service_data()
+    if is_flag_set("kube-control.connected"):
         push_service_data()
 
     clear_flag("hacluster-configured")
diff --git a/wheelhouse.txt b/wheelhouse.txt
index 5e54df54..09fc638a 100644
--- a/wheelhouse.txt
+++ b/wheelhouse.txt
@@ -1,2 +1,3 @@
 aiohttp>=3.7.4,<4.0.0
 gunicorn>=20.0.0,<21.0.0
+loadbalancer-interface

From 9947497ce17633d8b3291ace9fa4c90f1f592d0f Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Tue, 6 Apr 2021 16:54:17 -0400
Subject: [PATCH 02/13] Fix lint error

---
 reactive/kubernetes_master.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/reactive/kubernetes_master.py b/reactive/kubernetes_master.py
index 0737eac2..c64bb047 100644
--- a/reactive/kubernetes_master.py
+++ b/reactive/kubernetes_master.py
@@ -1429,8 +1429,7 @@ def push_service_data():
 @when("endpoint.lb-provider.available")
 @when_not("kubernetes-master.sent-lb-request")
 def request_load_balancer():
-    """Request a LB from the related provider.
-    """
+    """Request a LB from the related provider."""
     lb_provider = endpoint_from_name("lb-provider")
     req = lb_provider.get_request("api-server")
     req.protocol = req.protocols.tcp
@@ -1458,10 +1457,9 @@ def send_api_endpoints():
     if not endpoints:
         for relation in kube_control.relations:
             endpoints.append(kubernetes_master.get_api_endpoint(relation))
-    kube_control.set_api_endpoints([
-        "https://{}:{}".format(address, port)
-        for address, port in endpoints
-    ])
+    kube_control.set_api_endpoints(
+        ["https://{}:{}".format(address, port) for address, port in endpoints]
+    )
 
 
 @when("certificates.available", "cni.available")
@@ -2732,8 +2730,9 @@ def poke_network_unavailable():
     in a while.
     """
     local_address = get_ingress_address("kube-control")
-    local_server = "https://{0}:{1}".format(local_address,
-                                            kubernetes_master.STANDARD_API_PORT)
+    local_server = "https://{0}:{1}".format(
+        local_address, kubernetes_master.STANDARD_API_PORT
+    )
 
     client_token = get_token("admin")
     http_header = ("Authorization", "Bearer {}".format(client_token))

From 965845b58bc7850c14ab18770080741cdfd07a9d Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Tue, 6 Apr 2021 17:25:29 -0400
Subject: [PATCH 03/13] Add test for status reporting of incomplete LB relation

---
 tests/unit/test_kubernetes_master.py | 50 ++++++++++++++++++++++++++--
 1 file changed, 48 insertions(+), 2 deletions(-)

diff --git a/tests/unit/test_kubernetes_master.py b/tests/unit/test_kubernetes_master.py
index 9ac74b1d..0e03ef82 100644
--- a/tests/unit/test_kubernetes_master.py
+++ b/tests/unit/test_kubernetes_master.py
@@ -4,8 +4,9 @@
 from reactive import kubernetes_master
 from charms.layer import kubernetes_common
 from charms.layer.kubernetes_common import get_version, kubectl
-from charms.reactive import endpoint_from_flag, set_flag, is_flag_set, clear_flag
-from charmhelpers.core import hookenv, unitdata
+from charms.reactive import endpoint_from_flag, endpoint_from_name
+from charms.reactive import set_flag, is_flag_set, clear_flag
+from charmhelpers.core import hookenv, host, unitdata
 
 
 kubernetes_common.get_networks = lambda cidrs: [
@@ -133,6 +134,51 @@ def test_status_set_on_missing_ca():
     )
 
 
+def test_stauts_set_on_incomplete_lb():
+    """Test that set_final_status() will set waiting if LB is pending."""
+    set_flag("certificates.available")
+    clear_flag("kubernetes-master.secure-storage.failed")
+    set_flag("kube-control.connected")
+    set_flag("kubernetes-master.components.started")
+    set_flag("cdk-addons.configured")
+    set_flag("kubernetes-master.system-monitoring-rbac-role.applied")
+    hookenv.config.return_value = "auto"
+    host.service_running.return_value = True
+    kubectl.side_effect = None
+    kubectl.return_value = b'{"items": []}'
+
+    # test no LB relation
+    hookenv.goal_state.return_value = {}
+    kubernetes_master.set_final_status()
+    hookenv.status_set.assert_called_with("active", mock.ANY)
+
+    # test legacy kube-api-endpoint relation
+    hookenv.goal_state.return_value = {
+        "relations": {"kube-api-endpoint": None}
+    }
+    kubernetes_master.set_final_status()
+    hookenv.status_set.assert_called_with(
+        "waiting", "Waiting for kube-api-endpoint relation"
+    )
+    set_flag("kube-api-endpoint.available")
+    kubernetes_master.set_final_status()
+    hookenv.status_set.assert_called_with("active", mock.ANY)
+
+    # test new lb-provider relation
+    clear_flag("kube-api-endpoint.available")
+    hookenv.goal_state.return_value = {
+        "relations": {"lb-provider": None}
+    }
+    endpoint_from_name.return_value.has_response = False
+    kubernetes_master.set_final_status()
+    hookenv.status_set.assert_called_with(
+        "waiting", "Waiting for lb-provider"
+    )
+    endpoint_from_name.return_value.has_response = True
+    kubernetes_master.set_final_status()
+    hookenv.status_set.assert_called_with("active", mock.ANY)
+
+
 @mock.patch("reactive.kubernetes_master.setup_tokens")
 @mock.patch("reactive.kubernetes_master.get_token")
 def test_create_token_sign_auth_requests(get_token, setup_tokens):

From cccb83de1524f68d4e65837bce0efa00528178a9 Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Tue, 6 Apr 2021 17:40:11 -0400
Subject: [PATCH 04/13] Fix lint errors in test

---
 tests/unit/test_kubernetes_master.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/tests/unit/test_kubernetes_master.py b/tests/unit/test_kubernetes_master.py
index 0e03ef82..78e6dc63 100644
--- a/tests/unit/test_kubernetes_master.py
+++ b/tests/unit/test_kubernetes_master.py
@@ -153,9 +153,7 @@ def test_stauts_set_on_incomplete_lb():
     hookenv.status_set.assert_called_with("active", mock.ANY)
 
     # test legacy kube-api-endpoint relation
-    hookenv.goal_state.return_value = {
-        "relations": {"kube-api-endpoint": None}
-    }
+    hookenv.goal_state.return_value = {"relations": {"kube-api-endpoint": None}}
     kubernetes_master.set_final_status()
     hookenv.status_set.assert_called_with(
         "waiting", "Waiting for kube-api-endpoint relation"
@@ -166,14 +164,10 @@ def test_stauts_set_on_incomplete_lb():
 
     # test new lb-provider relation
     clear_flag("kube-api-endpoint.available")
-    hookenv.goal_state.return_value = {
-        "relations": {"lb-provider": None}
-    }
+    hookenv.goal_state.return_value = {"relations": {"lb-provider": None}}
     endpoint_from_name.return_value.has_response = False
     kubernetes_master.set_final_status()
-    hookenv.status_set.assert_called_with(
-        "waiting", "Waiting for lb-provider"
-    )
+    hookenv.status_set.assert_called_with("waiting", "Waiting for lb-provider")
     endpoint_from_name.return_value.has_response = True
     kubernetes_master.set_final_status()
     hookenv.status_set.assert_called_with("active", mock.ANY)

From 228c8e090f972eef64b29cb1a220cf87d3d22f05 Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Wed, 7 Apr 2021 13:41:07 -0400
Subject: [PATCH 05/13] Fix hook error when built with old version of
 kube-control interface

---
 reactive/kubernetes_master.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/reactive/kubernetes_master.py b/reactive/kubernetes_master.py
index c64bb047..0c80ef4a 100644
--- a/reactive/kubernetes_master.py
+++ b/reactive/kubernetes_master.py
@@ -1449,6 +1449,10 @@ def request_load_balancer():
 @when("kube-control.connected")
 def send_api_endpoints():
     kube_control = endpoint_from_name("kube-control")
+    if not hasattr(kube_control, "set_api_endpoints"):
+        # built with an old version of the kube-control interface
+        # the old kube-api-endpoint relation must be used instead
+        return
     lb_provider = endpoint_from_name("lb-provider")
     if lb_provider.is_available and not lb_provider.has_response:
         # waiting for lb-provider

From 6245411bde2e03a67143e6403105134b42b3fc54 Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Thu, 15 Apr 2021 10:19:36 -0400
Subject: [PATCH 06/13] Split lb-provider endpoint into separate, more explicit
 ones for internal / external

---
 lib/charms/layer/kubernetes_master.py | 125 +++++++++++++++++++-------
 metadata.yaml                         |  20 ++++-
 reactive/kubernetes_master.py         | 117 ++++++++++++------------
 tests/unit/test_kubernetes_master.py  |  21 ++++-
 4 files changed, 180 insertions(+), 103 deletions(-)

diff --git a/lib/charms/layer/kubernetes_master.py b/lib/charms/layer/kubernetes_master.py
index b4fb74d0..ec3bd06a 100644
--- a/lib/charms/layer/kubernetes_master.py
+++ b/lib/charms/layer/kubernetes_master.py
@@ -32,10 +32,9 @@
 db = unitdata.kv()
 
 
-def get_external_lb_endpoints():
+def get_endpoints_from_config():
     """
-    Return a list of any external API load-balancer endpoints that have
-    been manually configured.
+    Return a list of any manually configured API endpoints.
     """
     ha_connected = is_flag_set("ha.connected")
     forced_lb_ips = hookenv.config("loadbalancer-ips").split()
@@ -54,45 +53,105 @@ def get_external_lb_endpoints():
         return []
 
 
-def get_lb_endpoints():
+def get_internal_api_endpoints(relation=None):
     """
-    Return all load-balancer endpoints, whether from manual config or via
-    relation.
+    Determine the best API endpoints for an internal client to connect to.
+
+    If a relation is given, it will try to take that into account.
+
+    May return an empty list if an endpoint is expected but not yet available.
     """
-    external_lb_endpoints = get_external_lb_endpoints()
-    lb_provider = endpoint_from_name("lb-provider")
-    lb_response = lb_provider.get_response("api-server")
-    loadbalancer = endpoint_from_flag("loadbalancer.available")
-
-    if external_lb_endpoints:
-        return external_lb_endpoints
-    elif lb_response and lb_response.address:
-        return [(lb_response.address, STANDARD_API_PORT)]
-    elif loadbalancer:
+    try:
+        goal_state = hookenv.goal_state()
+    except NotImplementedError:
+        goal_state = {}
+    goal_state.setdefault("relations", {})
+
+    # Config takes precedence.
+    endpoints_from_config = get_endpoints_from_config()
+    if endpoints_from_config:
+        return endpoints_from_config
+
+    # If the internal LB relation is attached, use that or nothing. If it's
+    # not attached but the external LB relation is, use that or nothing.
+    for lb_endpoint in ("loadbalancer-internal", "loadbalancer-external"):
+        if lb_endpoint in goal_state["relations"]:
+            lb_provider = endpoint_from_name(lb_endpoint)
+            lb_response = lb_provider.get_response("kube-api")
+            if not lb_response or lb_response.error:
+                return []
+            return [(lb_response.address, STANDARD_API_PORT)]
+
+    # Support the older loadbalancer relation (public-address interface).
+    if "loadbalancer" in goal_state["relations"]:
+        loadbalancer = endpoint_from_flag("loadbalancer.available")
         lb_addresses = loadbalancer.get_addresses_ports()
         return [(host.get("public-address"), host.get("port")) for host in lb_addresses]
-    else:
-        return []
 
+    # No LBs of any kind, so fall back to ingress-address.
+    if not relation:
+        kube_control = endpoint_from_name("kube-control")
+        if not kube_control.relations:
+            return []
+        relation = kube_control.relations[0]
+    ingress_address = hookenv.ingress_address(
+        relation.relation_id, hookenv.local_unit()
+    )
+    return [(ingress_address, STANDARD_API_PORT)]
 
-def get_api_endpoint(relation=None):
+
+def get_external_api_endpoints():
     """
-    Determine the best endpoint for a client to connect to.
+    Determine the best API endpoints for an external client to connect to.
 
-    If a relation is given, it will take that into account when choosing an
-    endpoint.
+    May return an empty list if an endpoint is expected but not yet available.
     """
-    endpoints = get_lb_endpoints()
-    if endpoints:
-        # select a single endpoint based on our local unit number
-        return endpoints[kubernetes_common.get_unit_number() % len(endpoints)]
-    elif relation:
-        ingress_address = hookenv.ingress_address(
-            relation.relation_id, hookenv.local_unit()
-        )
-        return (ingress_address, STANDARD_API_PORT)
-    else:
-        return (hookenv.unit_public_ip(), STANDARD_API_PORT)
+    try:
+        goal_state = hookenv.goal_state()
+    except NotImplementedError:
+        goal_state = {}
+    goal_state.setdefault("relations", {})
+
+    # Config takes precedence.
+    endpoints_from_config = get_endpoints_from_config()
+    if endpoints_from_config:
+        return endpoints_from_config
+
+    # If the external LB relation is attached, use that or nothing. If it's
+    # not attached but the internal LB relation is, use that or nothing.
+    for lb_type in ("external", "internal"):
+        lb_endpoint = "loadbalancer-" + lb_type
+        lb_name = "api-server-" + lb_type
+        if lb_endpoint in goal_state["relations"]:
+            lb_provider = endpoint_from_name(lb_endpoint)
+            lb_response = lb_provider.get_response(lb_name)
+            if not lb_response or lb_response.error:
+                return []
+            return [(lb_response.address, STANDARD_API_PORT)]
+
+    # Support the older loadbalancer relation (public-address interface).
+    if "loadbalancer" in goal_state["relations"]:
+        loadbalancer = endpoint_from_flag("loadbalancer.available")
+        lb_addresses = loadbalancer.get_addresses_ports()
+        return [(host.get("public-address"), host.get("port")) for host in lb_addresses]
+
+    # No LBs of any kind, so fall back to public-address.
+    return [(hookenv.unit_public_ip(), STANDARD_API_PORT)]
+
+
+def get_api_urls(endpoints):
+    """
+    Convert a list of API server endpoints to URLs.
+    """
+    return ["https://{0}:{1}".format(*endpoint) for endpoint in endpoints]
+
+
+def get_api_url(endpoints):
+    """
+    Choose an API endpoint from the list and build a URL from it.
+    """
+    urls = get_api_urls(endpoints)
+    return urls[kubernetes_common.get_unit_number() % len(urls)]
 
 
 def install_ceph_common():
diff --git a/metadata.yaml b/metadata.yaml
index fa4ed6f5..0d0ec9c5 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -29,9 +29,13 @@ peers:
     interface: kube-masters
 provides:
   kube-api-endpoint:
-    # kube-api-endpoint is deprecated. Its functionality has been rolled into
-    # the kube-control interface. The relation endpoint will be removed in a
-    # future release.
+    # Use of this relation is strongly discouraged as the API endpoints will be
+    # provided via the kube-control relation. However, it can be used to
+    # override those endpoints if you need to inject a reverse proxy between
+    # the master and workers using a charm which only supports the old MITM
+    # style relations. Note, though, that since this reverse proxy will not be
+    # visible to the master, it will not be used in any of the client or
+    # component kube config files.
     interface: http
   cluster-dns:
     # kube-dns is deprecated. Its functionality has been rolled into the
@@ -54,6 +58,8 @@ requires:
   etcd:
     interface: etcd
   loadbalancer:
+    # Use of this relation is strongly discouraged in favor of the more
+    # explicit loadbalancer-internal / loadbalancer-external relations.
     interface: public-address
   ceph-storage:
     interface: ceph-admin
@@ -73,7 +79,13 @@ requires:
     interface: keystone-credentials
   dns-provider:
     interface: kube-dns
-  lb-provider:
+  loadbalancer-internal:
+    # Indicates that the LB should not be public and should use internal
+    # networks if available. Intended for control plane and other internal use.
+    interface: loadbalancer
+  loadbalancer-external:
+    # Indicates that the LB should be public facing. Intended for clients which
+    # must reach the API server via external networks.
     interface: loadbalancer
 resources:
   core:
diff --git a/reactive/kubernetes_master.py b/reactive/kubernetes_master.py
index 0c80ef4a..ab189e7e 100644
--- a/reactive/kubernetes_master.py
+++ b/reactive/kubernetes_master.py
@@ -805,11 +805,12 @@ def set_final_status():
             hookenv.status_set("waiting", "Waiting for kube-api-endpoint relation")
             return
 
-    if "lb-provider" in goal_state.get("relations", {}):
-        lb_provider = endpoint_from_name("lb-provider")
-        if not lb_provider.has_response:
-            hookenv.status_set("waiting", "Waiting for lb-provider")
-            return
+    for lb_endpoint in ("loadbalancer-internal", "loadbalancer-external"):
+        if lb_endpoint in goal_state.get("relations", {}):
+            lb_provider = endpoint_from_name(lb_endpoint)
+            if not lb_provider.has_response:
+                hookenv.status_set("waiting", "Waiting for " + lb_endpoint)
+                return
 
     if not is_state("kube-control.connected"):
         if "kube-control" in goal_state.get("relations", {}):
@@ -1409,61 +1410,53 @@ def create_tokens_and_sign_auth_requests():
 def push_service_data():
     """Send configuration to the load balancer, and close access to the
     public interface.
-
-    Note: This approach is deprecated in favor of the less complicated
-    lb-provider + kube-control approach.
     """
     kube_api = endpoint_from_flag("kube-api-endpoint.available")
 
-    external_endpoints = kubernetes_master.get_external_lb_endpoints()
-    if external_endpoints:
-        addresses = [e[0] for e in external_endpoints]
+    endpoints = kubernetes_master.get_endpoints_from_config()
+    if endpoints:
+        addresses = [e[0] for e in endpoints]
         kube_api.configure(kubernetes_master.STANDARD_API_PORT, addresses, addresses)
     else:
-        # no external addresses configured, so rely on the interface layer
+        # no manually configured LBs, so rely on the interface layer
         # to use the ingress address for each relation
         kube_api.configure(kubernetes_master.STANDARD_API_PORT)
 
 
 @when("leadership.is_leader")
-@when("endpoint.lb-provider.available")
-@when_not("kubernetes-master.sent-lb-request")
-def request_load_balancer():
-    """Request a LB from the related provider."""
-    lb_provider = endpoint_from_name("lb-provider")
-    req = lb_provider.get_request("api-server")
-    req.protocol = req.protocols.tcp
-    port = kubernetes_master.STANDARD_API_PORT
-    req.port_mapping = {port: port}
-    req.public = True
-    if not req.health_checks:
-        req.add_health_check(
-            protocol=req.protocols.http,
-            port=8080,
-            path="/livez",
-        )
-    lb_provider.send_request(req)
-    set_flag("kubernetes-master.sent-lb-request")
+@when_any(
+    "endpoint.loadbalancer-internal.available",
+    "endpoint.loadbalancer-external.available",
+)
+def request_load_balancers():
+    """Request LBs from the related provider(s)."""
+    for lb_type in ("internal", "external"):
+        lb_provider = endpoint_from_name("loadbalancer-" + lb_type)
+        req = lb_provider.get_request("api-server-" + lb_type)
+        req.protocol = req.protocols.tcp
+        api_port = kubernetes_master.STANDARD_API_PORT
+        req.port_mapping = {api_port: api_port}
+        req.public = lb_type == "external"
+        if not req.health_checks:
+            req.add_health_check(
+                protocol=req.protocols.http,
+                port=api_port,
+                path="/livez",
+            )
+        lb_provider.send_request(req)
 
 
 @when("kube-control.connected")
-def send_api_endpoints():
+def send_api_urls():
     kube_control = endpoint_from_name("kube-control")
     if not hasattr(kube_control, "set_api_endpoints"):
         # built with an old version of the kube-control interface
         # the old kube-api-endpoint relation must be used instead
         return
-    lb_provider = endpoint_from_name("lb-provider")
-    if lb_provider.is_available and not lb_provider.has_response:
-        # waiting for lb-provider
-        return
-    endpoints = kubernetes_master.get_lb_endpoints()
+    endpoints = kubernetes_master.get_internal_api_endpoints()
     if not endpoints:
-        for relation in kube_control.relations:
-            endpoints.append(kubernetes_master.get_api_endpoint(relation))
-    kube_control.set_api_endpoints(
-        ["https://{}:{}".format(address, port) for address, port in endpoints]
-    )
+        return
+    kube_control.set_api_endpoints(kubernetes_master.get_api_urls(endpoints))
 
 
 @when("certificates.available", "cni.available")
@@ -1508,8 +1501,8 @@ def send_data():
         + bind_ips
     )
 
-    lb_addrs = [e[0] for e in kubernetes_master.get_lb_endpoints()]
-    sans.extend(lb_addrs)
+    sans.extend(e[0] for e in kubernetes_master.get_internal_api_endpoints())
+    sans.extend(e[0] for e in kubernetes_master.get_external_api_endpoints())
 
     # maybe they have extra names they want as SANs
     extra_sans = hookenv.config("extra_sans")
@@ -2098,10 +2091,10 @@ def shutdown():
 def build_kubeconfig():
     """Gather the relevant data for Kubernetes configuration objects and create
     a config object with that information."""
-    local_address = get_ingress_address("kube-api-endpoint")
-    local_server = "https://{0}:{1}".format(local_address, 6443)
-    public_address, public_port = kubernetes_master.get_api_endpoint()
-    public_server = "https://{0}:{1}".format(public_address, public_port)
+    internal_endpoints = kubernetes_master.get_internal_api_endpoints()
+    internal_url = kubernetes_master.get_api_url(internal_endpoints)
+    external_endpoints = kubernetes_master.get_external_api_endpoints()
+    external_url = kubernetes_master.get_api_url(external_endpoints)
 
     # Do we have everything we need?
     if ca_crt_path.exists():
@@ -2148,7 +2141,7 @@ def build_kubeconfig():
         if ks:
             create_kubeconfig(
                 kubeconfig_path,
-                public_server,
+                external_url,
                 ca_crt_path,
                 user="admin",
                 token=client_pass,
@@ -2158,7 +2151,7 @@ def build_kubeconfig():
         else:
             create_kubeconfig(
                 kubeconfig_path,
-                public_server,
+                external_url,
                 ca_crt_path,
                 user="admin",
                 token=client_pass,
@@ -2172,7 +2165,7 @@ def build_kubeconfig():
         # make a kubeconfig for root (same location on k8s-masters and workers)
         create_kubeconfig(
             kubeclientconfig_path,
-            local_server,
+            internal_url,
             ca_crt_path,
             user="admin",
             token=client_pass,
@@ -2181,7 +2174,7 @@ def build_kubeconfig():
         # make a kubeconfig for cdk-addons
         create_kubeconfig(
             cdk_addons_kubectl_config_path,
-            local_server,
+            internal_url,
             ca_crt_path,
             user="admin",
             token=client_pass,
@@ -2192,7 +2185,7 @@ def build_kubeconfig():
         if proxy_token:
             create_kubeconfig(
                 kubeproxyconfig_path,
-                local_server,
+                internal_url,
                 ca_crt_path,
                 token=proxy_token,
                 user="kube-proxy",
@@ -2201,7 +2194,7 @@ def build_kubeconfig():
         if controller_manager_token:
             create_kubeconfig(
                 kubecontrollermanagerconfig_path,
-                local_server,
+                internal_url,
                 ca_crt_path,
                 token=controller_manager_token,
                 user="kube-controller-manager",
@@ -2210,7 +2203,7 @@ def build_kubeconfig():
         if scheduler_token:
             create_kubeconfig(
                 kubeschedulerconfig_path,
-                local_server,
+                internal_url,
                 ca_crt_path,
                 token=scheduler_token,
                 user="kube-scheduler",
@@ -2733,10 +2726,8 @@ def poke_network_unavailable():
     discussion about refactoring the affected code but nothing has happened
     in a while.
     """
-    local_address = get_ingress_address("kube-control")
-    local_server = "https://{0}:{1}".format(
-        local_address, kubernetes_master.STANDARD_API_PORT
-    )
+    internal_endpoints = kubernetes_master.get_internal_api_endpoints()
+    internal_url = kubernetes_master.get_api_url(internal_endpoints)
 
     client_token = get_token("admin")
     http_header = ("Authorization", "Bearer {}".format(client_token))
@@ -2756,7 +2747,7 @@ def poke_network_unavailable():
 
     for node in nodes:
         node_name = node["metadata"]["name"]
-        url = "{}/api/v1/nodes/{}/status".format(local_server, node_name)
+        url = "{}/api/v1/nodes/{}/status".format(internal_url, node_name)
         req = Request(url)
         req.add_header(*http_header)
         with urlopen(req) as response:
@@ -3222,7 +3213,9 @@ def configure_hacluster():
         send_data()
 
     # update workers
-    if is_state("kube-control.connected"):
+    if is_flag_set("kube-control.connected"):
+        send_api_urls()
+    if is_flag_set("kube-api-endpoint.available"):
         push_service_data()
 
     set_flag("hacluster-configured")
@@ -3239,9 +3232,9 @@ def remove_hacluster():
     if is_flag_set("certificates.available"):
         send_data()
     # update workers
-    if is_flag_set("kube-api-endpoint.available"):
-        push_service_data()
     if is_flag_set("kube-control.connected"):
+        send_api_urls()
+    if is_flag_set("kube-api-endpoint.available"):
         push_service_data()
 
     clear_flag("hacluster-configured")
diff --git a/tests/unit/test_kubernetes_master.py b/tests/unit/test_kubernetes_master.py
index 78e6dc63..793a051f 100644
--- a/tests/unit/test_kubernetes_master.py
+++ b/tests/unit/test_kubernetes_master.py
@@ -134,7 +134,7 @@ def test_status_set_on_missing_ca():
     )
 
 
-def test_stauts_set_on_incomplete_lb():
+def test_status_set_on_incomplete_lb():
     """Test that set_final_status() will set waiting if LB is pending."""
     set_flag("certificates.available")
     clear_flag("kubernetes-master.secure-storage.failed")
@@ -162,12 +162,25 @@ def test_stauts_set_on_incomplete_lb():
     kubernetes_master.set_final_status()
     hookenv.status_set.assert_called_with("active", mock.ANY)
 
-    # test new lb-provider relation
+    # test loadbalancer-internal relation
     clear_flag("kube-api-endpoint.available")
-    hookenv.goal_state.return_value = {"relations": {"lb-provider": None}}
+    hookenv.goal_state.return_value = {"relations": {"loadbalancer-internal": None}}
     endpoint_from_name.return_value.has_response = False
     kubernetes_master.set_final_status()
-    hookenv.status_set.assert_called_with("waiting", "Waiting for lb-provider")
+    hookenv.status_set.assert_called_with(
+        "waiting", "Waiting for loadbalancer-internal"
+    )
+    endpoint_from_name.return_value.has_response = True
+    kubernetes_master.set_final_status()
+    hookenv.status_set.assert_called_with("active", mock.ANY)
+
+    # test loadbalancer-external relation
+    hookenv.goal_state.return_value = {"relations": {"loadbalancer-external": None}}
+    endpoint_from_name.return_value.has_response = False
+    kubernetes_master.set_final_status()
+    hookenv.status_set.assert_called_with(
+        "waiting", "Waiting for loadbalancer-external"
+    )
     endpoint_from_name.return_value.has_response = True
     kubernetes_master.set_final_status()
     hookenv.status_set.assert_called_with("active", mock.ANY)

From af9877083a6d12f48d964adfe06bb8951453017f Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Thu, 22 Apr 2021 10:26:21 -0400
Subject: [PATCH 07/13] Fix handling of internal LB response and missing
 endpoints

---
 lib/charms/layer/kubernetes_master.py | 8 ++++++--
 reactive/kubernetes_master.py         | 8 +++++---
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/lib/charms/layer/kubernetes_master.py b/lib/charms/layer/kubernetes_master.py
index ec3bd06a..c906d1e5 100644
--- a/lib/charms/layer/kubernetes_master.py
+++ b/lib/charms/layer/kubernetes_master.py
@@ -74,10 +74,12 @@ def get_internal_api_endpoints(relation=None):
 
     # If the internal LB relation is attached, use that or nothing. If it's
     # not attached but the external LB relation is, use that or nothing.
-    for lb_endpoint in ("loadbalancer-internal", "loadbalancer-external"):
+    for lb_type in ("internal", "external"):
+        lb_endpoint = "loadbalancer-" + lb_type
+        request_name = "api-server-" + lb_type
         if lb_endpoint in goal_state["relations"]:
             lb_provider = endpoint_from_name(lb_endpoint)
-            lb_response = lb_provider.get_response("kube-api")
+            lb_response = lb_provider.get_response(request_name)
             if not lb_response or lb_response.error:
                 return []
             return [(lb_response.address, STANDARD_API_PORT)]
@@ -150,6 +152,8 @@ def get_api_url(endpoints):
     """
     Choose an API endpoint from the list and build a URL from it.
     """
+    if not endpoints:
+        return None
     urls = get_api_urls(endpoints)
     return urls[kubernetes_common.get_unit_number() % len(urls)]
 
diff --git a/reactive/kubernetes_master.py b/reactive/kubernetes_master.py
index ab189e7e..355c6c18 100644
--- a/reactive/kubernetes_master.py
+++ b/reactive/kubernetes_master.py
@@ -1432,6 +1432,8 @@ def request_load_balancers():
     """Request LBs from the related provider(s)."""
     for lb_type in ("internal", "external"):
         lb_provider = endpoint_from_name("loadbalancer-" + lb_type)
+        if not lb_provider.is_available:
+            continue
         req = lb_provider.get_request("api-server-" + lb_type)
         req.protocol = req.protocols.tcp
         api_port = kubernetes_master.STANDARD_API_PORT
@@ -2092,12 +2094,12 @@ def build_kubeconfig():
     """Gather the relevant data for Kubernetes configuration objects and create
     a config object with that information."""
     internal_endpoints = kubernetes_master.get_internal_api_endpoints()
-    internal_url = kubernetes_master.get_api_url(internal_endpoints)
     external_endpoints = kubernetes_master.get_external_api_endpoints()
-    external_url = kubernetes_master.get_api_url(external_endpoints)
 
     # Do we have everything we need?
-    if ca_crt_path.exists():
+    if ca_crt_path.exists() and internal_endpoints and external_endpoints:
+        internal_url = kubernetes_master.get_api_url(internal_endpoints)
+        external_url = kubernetes_master.get_api_url(external_endpoints)
         client_pass = get_token("admin")
         if not client_pass:
             # If we made it this far without a password, we're bootstrapping a new

From 6f16ac559da3cc6f6b3c10f0ea98d9156b37cc20 Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Thu, 29 Apr 2021 15:27:09 -0400
Subject: [PATCH 08/13] Move test to new LB relation pattern and use edge
 charms

---
 tests/data/bundle.yaml                        |  7 +++--
 .../test_kubernetes_master_integration.py     | 28 +++++++++++++------
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/tests/data/bundle.yaml b/tests/data/bundle.yaml
index 6ba9c1b2..56b2eb8e 100644
--- a/tests/data/bundle.yaml
+++ b/tests/data/bundle.yaml
@@ -10,13 +10,16 @@ machines:
 services:
   containerd:
     charm: cs:~containers/containerd
+    channel: edge
   easyrsa:
     charm: cs:~containers/easyrsa
+    channel: edge
     num_units: 1
     to:
     - '1'
   etcd:
     charm: cs:~containers/etcd
+    channel: edge
     num_units: 1
     options:
       channel: 3.4/stable
@@ -24,6 +27,7 @@ services:
     - '0'
   flannel:
     charm: cs:~containers/flannel
+    channel: edge
   kubernetes-master:
     charm: {{master_charm}}
     constraints: cores=2 mem=4G root-disk=16G
@@ -35,6 +39,7 @@ services:
     - '0'
   kubernetes-worker:
     charm: cs:~containers/kubernetes-worker
+    channel: edge
     constraints: cores=4 mem=4G root-disk=16G
     expose: true
     num_units: 1
@@ -43,8 +48,6 @@ services:
     to:
     - '1'
 relations:
-- - kubernetes-master:kube-api-endpoint
-  - kubernetes-worker:kube-api-endpoint
 - - kubernetes-master:kube-control
   - kubernetes-worker:kube-control
 - - kubernetes-master:certificates
diff --git a/tests/integration/test_kubernetes_master_integration.py b/tests/integration/test_kubernetes_master_integration.py
index a66bb981..aeeed3d1 100644
--- a/tests/integration/test_kubernetes_master_integration.py
+++ b/tests/integration/test_kubernetes_master_integration.py
@@ -9,6 +9,17 @@
 log = logging.getLogger(__name__)
 
 
+def _check_status_messages(ops_test):
+    """Validate that the status messages are correct."""
+    expected_messages = {
+        "kubernetes-master": "Kubernetes master running.",
+        "kubernetes-worker": "Kubernetes worker running.",
+    }
+    for app, message in expected_messages.items():
+        for unit in ops_test.model.applications[app].units:
+            assert unit.workload_status_message == message
+
+
 @pytest.mark.abort_on_fail
 async def test_build_and_deploy(ops_test):
     bundle = ops_test.render_bundle(
@@ -16,17 +27,16 @@ async def test_build_and_deploy(ops_test):
     )
     await ops_test.model.deploy(bundle)
     await ops_test.model.wait_for_idle(wait_for_active=True, timeout=60 * 60)
+    _check_status_messages(ops_test)
 
 
-async def test_status_messages(ops_test):
-    """Validate that the status messages are correct."""
-    expected_messages = {
-        "kubernetes-master": "Kubernetes master running.",
-        "kubernetes-worker": "Kubernetes worker running.",
-    }
-    for app, message in expected_messages.items():
-        for unit in ops_test.model.applications[app].units:
-            assert unit.workload_status_message == message
+async def test_kube_api_endpoint(ops_test):
+    """Validate that adding the kube-api-endpoint relation works"""
+    await ops_test.model.add_relation(
+        "kubernetes-master:kube-api-endpoint", "kubernetes-worker:kube-api-endpoint"
+    )
+    await ops_test.model.wait_for_idle(wait_for_active=True, timeout=10 * 60)
+    _check_status_messages(ops_test)
 
 
 async def juju_run(unit, cmd):

From 4652693f37beae8007cb513817ea787913ef1e94 Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Tue, 4 May 2021 17:28:02 -0400
Subject: [PATCH 09/13] Improve status reporting around the auth-webhook and
 skip trying to create secrets before apiserver is available

---
 reactive/kubernetes_master.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/reactive/kubernetes_master.py b/reactive/kubernetes_master.py
index 355c6c18..51bc68d8 100644
--- a/reactive/kubernetes_master.py
+++ b/reactive/kubernetes_master.py
@@ -863,10 +863,10 @@ def set_final_status():
         hookenv.status_set("waiting", "Waiting for API server to be configured")
         return
 
-    auth_setup = is_flag_set("authentication.setup")
-    webhook_tokens_setup = is_flag_set("kubernetes-master.auth-webhook-tokens.setup")
-    if auth_setup and not webhook_tokens_setup:
-        hookenv.status_set("waiting", "Failed to setup auth-webhook tokens; will retry")
+    is_leader = is_state("leadership.is_leader")
+    authentication_setup = is_state("authentication.setup")
+    if not is_leader and not authentication_setup:
+        hookenv.status_set("waiting", "Waiting on leader's crypto keys.")
         return
 
     if is_state("kubernetes-master.components.started"):
@@ -887,10 +887,10 @@ def set_final_status():
 
     # Note that after this point, kubernetes-master.components.started is
     # always True.
-    is_leader = is_state("leadership.is_leader")
-    authentication_setup = is_state("authentication.setup")
-    if not is_leader and not authentication_setup:
-        hookenv.status_set("waiting", "Waiting on leader's crypto keys.")
+
+    webhook_tokens_setup = is_flag_set("kubernetes-master.auth-webhook-tokens.setup")
+    if not webhook_tokens_setup:
+        hookenv.status_set("waiting", "Failed to setup auth-webhook tokens; will retry")
         return
 
     addons_configured = is_state("cdk-addons.configured")
@@ -1156,6 +1156,7 @@ def register_auth_webhook():
 
 @when(
     "kubernetes-master.apiserver.configured",
+    "kubernetes-master.components.started",
     "kubernetes-master.auth-webhook-service.started",
     "authentication.setup",
 )

From c74380580c48d0066444b39fd1411a85cc9fd45e Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Wed, 12 May 2021 11:09:17 -0400
Subject: [PATCH 10/13] Create a ~/.kube/config for the ubuntu user

---
 reactive/kubernetes_master.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/reactive/kubernetes_master.py b/reactive/kubernetes_master.py
index 51bc68d8..5afb8af3 100644
--- a/reactive/kubernetes_master.py
+++ b/reactive/kubernetes_master.py
@@ -2174,6 +2174,17 @@ def build_kubeconfig():
             token=client_pass,
         )
 
+        # Create kubernetes configuration in the default location for ubuntu.
+        create_kubeconfig(
+            "/home/ubuntu/.kube/config",
+            internal_url,
+            ca_crt_path,
+            user="admin",
+            token=client_pass,
+        )
+        # Make the config dir readable by the ubuntu user
+        check_call(["chown", "-R", "ubuntu:ubuntu", "/home/ubuntu/.kube"])
+
         # make a kubeconfig for cdk-addons
         create_kubeconfig(
             cdk_addons_kubectl_config_path,

From 8d8bf1f3f6eaac74db4dcc5d1eddafe7c5e2aee9 Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Thu, 13 May 2021 08:58:29 -0400
Subject: [PATCH 11/13] Fix NoneType error for old-style relation

---
 lib/charms/layer/kubernetes_master.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/charms/layer/kubernetes_master.py b/lib/charms/layer/kubernetes_master.py
index c906d1e5..81dea2ad 100644
--- a/lib/charms/layer/kubernetes_master.py
+++ b/lib/charms/layer/kubernetes_master.py
@@ -86,7 +86,7 @@ def get_internal_api_endpoints(relation=None):
 
     # Support the older loadbalancer relation (public-address interface).
     if "loadbalancer" in goal_state["relations"]:
-        loadbalancer = endpoint_from_flag("loadbalancer.available")
+        loadbalancer = endpoint_from_name("loadbalancer")
         lb_addresses = loadbalancer.get_addresses_ports()
         return [(host.get("public-address"), host.get("port")) for host in lb_addresses]
 

From 6710736c3efed51f83c03509ddd01b019ee2cef1 Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Thu, 13 May 2021 11:38:34 -0400
Subject: [PATCH 12/13] Fix unit test

---
 tests/unit/test_kubernetes_master.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/unit/test_kubernetes_master.py b/tests/unit/test_kubernetes_master.py
index 793a051f..876d234a 100644
--- a/tests/unit/test_kubernetes_master.py
+++ b/tests/unit/test_kubernetes_master.py
@@ -139,8 +139,11 @@ def test_status_set_on_incomplete_lb():
     set_flag("certificates.available")
     clear_flag("kubernetes-master.secure-storage.failed")
     set_flag("kube-control.connected")
+    set_flag("kubernetes-master.auth-webhook-service.started")
+    set_flag("kubernetes-master.apiserver.configured")
     set_flag("kubernetes-master.components.started")
     set_flag("cdk-addons.configured")
+    set_flag("kubernetes-master.auth-webhook-tokens.setup")
     set_flag("kubernetes-master.system-monitoring-rbac-role.applied")
     hookenv.config.return_value = "auto"
     host.service_running.return_value = True

From b41eee67cf2b5a0df53e2a0bbfafd1fb50d442ff Mon Sep 17 00:00:00 2001
From: Cory Johns <johnsca@gmail.com>
Date: Wed, 19 May 2021 10:42:07 -0400
Subject: [PATCH 13/13] Revert drive-by related to auth setup status reporting;
 splitting to separate bug

---
 reactive/kubernetes_master.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/reactive/kubernetes_master.py b/reactive/kubernetes_master.py
index 5afb8af3..895d0fa9 100644
--- a/reactive/kubernetes_master.py
+++ b/reactive/kubernetes_master.py
@@ -863,10 +863,10 @@ def set_final_status():
         hookenv.status_set("waiting", "Waiting for API server to be configured")
         return
 
-    is_leader = is_state("leadership.is_leader")
-    authentication_setup = is_state("authentication.setup")
-    if not is_leader and not authentication_setup:
-        hookenv.status_set("waiting", "Waiting on leader's crypto keys.")
+    auth_setup = is_flag_set("authentication.setup")
+    webhook_tokens_setup = is_flag_set("kubernetes-master.auth-webhook-tokens.setup")
+    if auth_setup and not webhook_tokens_setup:
+        hookenv.status_set("waiting", "Failed to setup auth-webhook tokens; will retry")
         return
 
     if is_state("kubernetes-master.components.started"):
@@ -887,10 +887,10 @@ def set_final_status():
 
     # Note that after this point, kubernetes-master.components.started is
     # always True.
-
-    webhook_tokens_setup = is_flag_set("kubernetes-master.auth-webhook-tokens.setup")
-    if not webhook_tokens_setup:
-        hookenv.status_set("waiting", "Failed to setup auth-webhook tokens; will retry")
+    is_leader = is_state("leadership.is_leader")
+    authentication_setup = is_state("authentication.setup")
+    if not is_leader and not authentication_setup:
+        hookenv.status_set("waiting", "Waiting on leader's crypto keys.")
         return
 
     addons_configured = is_state("cdk-addons.configured")
@@ -1156,7 +1156,6 @@ def register_auth_webhook():
 
 @when(
     "kubernetes-master.apiserver.configured",
-    "kubernetes-master.components.started",
     "kubernetes-master.auth-webhook-service.started",
     "authentication.setup",
 )