Skip to content

Commit

Permalink
[KubeIngressProxy] Add reuse_existing_services option
Browse files Browse the repository at this point in the history
  • Loading branch information
@dolfinus
dolfinus committed Nov 11, 2022
1 parent 0f22abf commit b0d2871
Show file tree
Hide file tree
Showing 3 changed files with 340 additions and 39 deletions.
79 changes: 64 additions & 15 deletions kubespawner/objects.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import json
import operator
import re
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Tuple
from urllib.parse import urlparse

from kubernetes_asyncio.client.models import (
Expand Down Expand Up @@ -62,6 +62,18 @@

from .utils import get_k8s_model, host_matching, update_k8s_model

# Matches Kubernetes DNS names template for services
# https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#namespaces-of-services:
# * service.namespace.svc.cluster.local
# * service.namespace.svc.cluster
# * service.namespace.svc
# * service.namespace
# * service
# User by `KubeIngressProxy.reuse_existing_services=True`
SERVICE_DNS_PATTERN = re.compile(
r"(?P<service>[^.]+)\.?(?P<namespace>[^.]+)?(?P<rest>\.svc(\.cluster(\.local)?)?)?"
)


def make_pod(
name,
Expand Down Expand Up @@ -735,12 +747,14 @@ def make_ingress(
routespec: str,
target: str,
data: dict,
namespace: str,
common_labels: Optional[dict] = None,
ingress_extra_labels: Optional[dict] = None,
ingress_extra_annotations: Optional[dict] = None,
ingress_class_name: Optional[str] = None,
ingress_specifications: Optional[List[Dict]] = None,
):
reuse_existing_services: bool = False,
) -> Tuple[Optional[V1Endpoints], Optional[V1Service], V1Ingress]:
"""
Returns an ingress, service, endpoint object that'll work for this service
"""
Expand Down Expand Up @@ -782,8 +796,14 @@ def make_ingress(
else:
target_is_ip = True

service_name = name

if target_is_ip:
# Make endpoint object
# c.KubeSpawner.services_enabled = False
# routespec='http://192.168.1.1:8888/'

# ingress -> service -> endpoint -> pod
# endpoint is used just to allow access from ingress to Pod IP (if PodNetworkPolicy is used)
endpoint = V1Endpoints(
kind='Endpoints',
metadata=meta,
Expand All @@ -807,17 +827,46 @@ def make_ingress(
else:
endpoint = None

# Make service object
service = V1Service(
kind='Service',
metadata=meta,
spec=V1ServiceSpec(
type='ExternalName',
external_name=target_parts.hostname,
cluster_ip='',
ports=[V1ServicePort(port=target_port, target_port=target_port)],
),
)
service_match = SERVICE_DNS_PATTERN.match(target_parts.hostname)
if (
reuse_existing_services
and service_match
and (
not service_match.group("namespace")
or service_match.group("namespace") == namespace
)
):
# c.KubeSpawner.services_enabled = True
# routespec='http://myservice.mynamespace.svc.cluster.local:8888/'
# routespec='http://myservice.mynamespace.svc.cluster:8888/'
# routespec='http://myservice.mynamespace.svc:8888/'
# routespec='http://myservice.mynamespace:8888/'
# routespec='http://hub:8888/'

# Ingress -> Service (same namespace, created by KubeSpawner)
service = None # just use this service in ingress, do not create it
service_name = service_match.group("service")
else:
# config: c.KubeSpawner.namespace='another-namespace'
# routespec: 'http://myservice.another-namespace...:8888/'
# or
# config: c.KubeSpawner.enable_user_namespaces=True
# routespec: 'http://myservice.user-namespace...:8888/'
# or
# c.JupyterHub.services = [{"name": "my-service", "url": "http://some.domain:9000"}]
# routespec: 'http://some.domain:9000/'

# Ingress -> ExternalName -> Service (different namespace or some external domain)
service = V1Service(
kind='Service',
metadata=meta,
spec=V1ServiceSpec(
type='ExternalName',
external_name=target_parts.hostname,
cluster_ip='',
ports=[V1ServicePort(port=target_port, target_port=target_port)],
),
)

# Make Ingress object

Expand Down Expand Up @@ -868,7 +917,7 @@ def make_ingress(
path_type="Prefix",
backend=V1IngressBackend(
service=V1IngressServiceBackend(
name=name,
name=service_name,
port=V1ServiceBackendPort(
number=target_port,
),
Expand Down
60 changes: 43 additions & 17 deletions kubespawner/proxy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from jupyterhub.proxy import Proxy
from jupyterhub.utils import exponential_backoff
from kubernetes_asyncio import client
from traitlets import Dict, List, Unicode
from traitlets import Bool, Dict, List, Unicode

from .clients import load_config, shared_client
from .objects import make_ingress
Expand Down Expand Up @@ -252,6 +252,25 @@ def _namespace_default(self):
""",
)

reuse_existing_services = Bool(
False,
config=True,
help="""
If `True`, proxy will try to reuse existing services created by `KubeSpawner.services_enabled=True`
or `KubeSpawner.internal_ssl=True`.
If `False` (default), KubeSpawner creates service with type `ExternalName`, pointing to the pod's service.
Sometimes `ExternalName` could lead to issues with accessing pod, like `500 Redirect loop detected`,
so setting this option to `True` could solve this issue.
By default, KubeSpawner does not create service for a pod at all (`service_enabled=False`, `internal_ssl=False`).
In such a case KubeSpawner creates service with type `ClusterIP`, pointing to the pod IP and port.
If KubeSpawner creates pod in a different namespace, this option is ignored,
because Ingress(namespace=hub) cannot point to Service(namespace=user).
""",
)

k8s_api_ssl_ca_cert = Unicode(
"",
config=True,
Expand Down Expand Up @@ -383,6 +402,7 @@ async def add_route(self, routespec, target, data):
# 'data' is JSON encoded and put in an annotation - we don't need to query for it

safe_name = self._safe_name_for_routespec(routespec).lower()
full_name = f'{self.namespace}/{safe_name}'

common_labels = self._expand_all(self.common_labels, routespec, data)
common_labels.update({'component': self.component_label})
Expand All @@ -405,11 +425,13 @@ async def add_route(self, routespec, target, data):
routespec=routespec,
target=target,
data=data,
namespace=self.namespace,
common_labels=common_labels,
ingress_extra_labels=ingress_extra_labels,
ingress_extra_annotations=ingress_extra_annotations,
ingress_class_name=self.ingress_class_name,
ingress_specifications=ingress_specifications,
reuse_existing_services=self.reuse_existing_services,
)

async def ensure_object(create_func, patch_func, body, kind):
Expand Down Expand Up @@ -439,8 +461,7 @@ async def ensure_object(create_func, patch_func, body, kind):
)

await exponential_backoff(
lambda: f'{self.namespace}/{safe_name}'
in self.endpoint_reflector.endpoints.keys(),
lambda: full_name in self.endpoint_reflector.endpoints,
'Could not find endpoints/%s after creating it' % safe_name,
)
else:
Expand All @@ -451,18 +472,24 @@ async def ensure_object(create_func, patch_func, body, kind):
)
await self._delete_if_exists('endpoint', safe_name, delete_endpoint)

await ensure_object(
self.core_api.create_namespaced_service,
self.core_api.patch_namespaced_service,
body=service,
kind='service',
)

await exponential_backoff(
lambda: f'{self.namespace}/{safe_name}'
in self.service_reflector.services.keys(),
'Could not find service/%s after creating it' % safe_name,
)
if service is not None:
await ensure_object(
self.core_api.create_namespaced_service,
self.core_api.patch_namespaced_service,
body=service,
kind='service',
)
await exponential_backoff(
lambda: full_name in self.service_reflector.services,
'Could not find services/%s after creating it' % safe_name,
)
else:
delete_service = self.core_api.delete_namespaced_service(
name=safe_name,
namespace=self.namespace,
body=client.V1DeleteOptions(grace_period_seconds=0),
)
await self._delete_if_exists('service', safe_name, delete_service)

await ensure_object(
self.networking_api.create_namespaced_ingress,
Expand All @@ -472,8 +499,7 @@ async def ensure_object(create_func, patch_func, body, kind):
)

await exponential_backoff(
lambda: f'{self.namespace}/{safe_name}'
in self.ingress_reflector.ingresses.keys(),
lambda: full_name in self.ingress_reflector.ingresses,
'Could not find ingress/%s after creating it' % safe_name,
)

Expand Down

0 comments on commit b0d2871

Please sign in to comment.