Improve cluster configuration when enableTls is true (#169)

## Motivation

Improve the basic functionality of `enabledTls` flag. Setting the flag to true means that the broker, proxy, and function worker will all enable TLS, but not use it unless configured. See the README update for more details.

### Changes

* Add documentation to README.
* Add `.Values.tls.<component>.enableHostnameVerification` flag that makes it possible to enable hostname verification for upstream connections. The default is false for now (old deployments that use `kind: Deployment` for broker would break otherwise). We will update to `true` in the next major version bump.
* Update `dev-values-tls.yaml` to deploy a broker stateful set (this is necessary for hostname verification to work)
* Improve bastion's client configuration to utilize TLS and to use the proxy, since that will ensure the function worker is integrated correctly
* Update hostnames in the self signed certificate to support broker and function worker correctly
* Update zookeeper initialization script to use the `brokerSts` component name, since statefulsets are the only way to ensure full-featured TLS connections.

README.md

@@ -562,6 +562,23 @@ enableTokenAuth: true
 
 ### TLS
 
+There are many components to consider when enabling TLS for a Pulsar Cluster. To enable TLS for all client facing
+endpoints, set `enableTls: true` in the values file and configure certificates. This setting will enable TLS endpoints
+for the Broker pods, Function Worker pods, and Proxy pods. However, this setting will not configure the proxy or the
+function worker to use TLS for connections with the broker. You can enable those by configuring
+`tls.proxy.enabled: true` and `tls.function.enabled: true`, respectively. Because the function worker only connects to
+the broker over TLS when authentication is configured, make sure to enable authentication if you'd like the function
+worker to connect to the broker over TLS.
+
+#### Hostname Verification
+
+In order for hostname verification to work, you must configure the helm chart to deploy the broker cluster as a
+StatefulSet. This kind of deployment gives each pod a stable network identifier, which is necessary for hostname
+verification and the Pulsar Protocol.
+
+To enable hostname verification with upstream servers, set `tls.<component>.enableHostnameVerification: true`. Note that
+these settings temporarily default to false for backwards compatibility, but will be updated to default to true in the
+next major version bump.
 
 ### Automatically generating certificates using cert-manager
 

examples/dev-values-tls.yaml

@@ -21,6 +21,8 @@ enableTokenAuth: false
 restartOnConfigMapChange:
   enabled: true
 extra:
+  broker: false
+  brokerSts: true
   function: true
   burnell: true
   burnellLogCollector: true
@@ -52,7 +54,7 @@ bookkeeper:
   configData:
     BOOKIE_MEM: "-Xms312m -Xmx312m -XX:MaxDirectMemorySize=200m -XX:+ExitOnOutOfMemoryError"
 
-broker:
+brokerSts:
   component: broker
   replicaCount: 1
   ledger:

helm-chart-sources/pulsar/templates/bastion/bastion-configmap.yaml

@@ -33,10 +33,30 @@ data:
   authParams: "file:///pulsar/token-superuser-stripped.jwt"
   authPlugin: "org.apache.pulsar.client.impl.auth.AuthenticationToken"
   {{- end }}
-  {{- if or .Values.secrets .Values.createCertificates.selfSigned.enabled}}
-  tlsTrustCertsFilePath: /pulsar/certs/ca.crt
-  {{- else }}
+  # If proxy is deployed, use that for web service URL to
+  # properly forward command to broker or function worker
+  {{- if .Values.enableTls }}
+  tlsEnableHostnameVerification: "true"
+    {{- if or .Values.secrets .Values.createCertificates.selfSigned.enabled}}
+  tlsTrustCertsFilePath: "/pulsar/certs/ca.crt"
+    {{- else }}
   tlsTrustCertsFilePath: "{{ .Values.tlsCaPath }}/{{ .Values.tlsCaCert }}"
+    {{- end }}
+    {{- if .Values.extra.proxy }}
+  brokerServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}:6651/"
+  webServiceUrl: "https://{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}:8443/"
+    {{- else }}
+  brokerServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}:6651/"
+  webServiceUrl: "https://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}:8443/"
+    {{- end }}
+  {{- else }}
+    {{- if .Values.extra.proxy }}
+  brokerServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}:6650/"
+  webServiceUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}:8080/"
+    {{- else }}
+  brokerServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:6650/"
+  webServiceUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:8080/"
+    {{- end }}
   {{- end }}
 {{- range $key, $val := $.Values.bastion.configData }}
   {{ $key }}: {{ $val | replace "\"" "" | trim | quote }}

helm-chart-sources/pulsar/templates/bastion/bastion-deployment.yaml

@@ -170,21 +170,4 @@ spec:
         envFrom:
         - configMapRef:
             name: "{{ template "pulsar.fullname" . }}-{{ .Values.bastion.component }}"
-        env:
-        # If proxy is deployed, use that for web service URL to
-        # properly forward command to broker or function worker
-        {{- if .Values.extra.proxy }}
-        {{- if .Values.enableTls }}
-        - name: webServiceUrl
-          value: https://{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}:8443/
-        {{- else }}
-        - name: webServiceUrl
-          value: http://{{ template "pulsar.fullname" . }}-{{ .Values.proxy.component }}:8080/
-        {{- end }}
-        {{- else }}
-        - name: webServiceUrl
-          value: http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:8080/
-        {{- end }}
-        - name: brokerServiceUrl
-          value: pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:6650/
 {{- end }}

helm-chart-sources/pulsar/templates/broker-sts/broker-sts-statefulset.yaml

@@ -283,16 +283,12 @@ spec:
         envFrom:
         - configMapRef:
             name: "{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}"
-        {{- if .Values.brokerSts.ledger }}
         env:
-        - name: advertisedAddress
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
         {{- if .Values.broker.kafkaOnPulsarEnabled }}
         - name: PULSAR_PREFIX_kafkaAdvertisedListeners
           value: "PLAINTEXT://$(advertisedAddress):9092"
         {{- end }}
+        {{- if .Values.brokerSts.ledger }}
         - name: managedLedgerDefaultAckQuorum
           value: "{{ .Values.brokerSts.ledger.defaultAckQuorum }}"
         - name: managedLedgerDefaultEnsembleSize

helm-chart-sources/pulsar/templates/cert-manager/self-signed-issuer.yaml

@@ -58,13 +58,21 @@ metadata:
   namespace: {{ .Release.Namespace }}
 spec:
   secretName: {{ .Values.tlsSecretName }}
+  # The wildcard names are needed to connect directly to the broker pods and will only work when the broker is deployed
+  # as a StatefulSet.
   dnsNames:
-  - "{{ template "pulsar.fullname" . }}-broker.{{ .Release.Namespace }}.svc.cluster.local"
-  - "{{ template "pulsar.fullname" . }}-broker.{{ .Release.Namespace }}"
-  - "{{ template "pulsar.fullname" . }}-broker"
+  - "*.{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ .Release.Namespace }}.svc.cluster.local"
+  - "*.{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ .Release.Namespace }}"
+  - "*.{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}"
+  - "{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ .Release.Namespace }}.svc.cluster.local"
+  - "{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ .Release.Namespace }}"
+  - "{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}"
   - "{{ template "pulsar.fullname" . }}-proxy.{{ .Release.Namespace }}.svc.cluster.local"
   - "{{ template "pulsar.fullname" . }}-proxy.{{ .Release.Namespace }}"
   - "{{ template "pulsar.fullname" . }}-proxy"
+  - "{{ template "pulsar.fullname" . }}-{{ .Values.function.component }}-ca.{{ .Release.Namespace }}.svc.cluster.local"
+  - "{{ template "pulsar.fullname" . }}-{{ .Values.function.component }}-ca.{{ .Release.Namespace }}"
+  - "{{ template "pulsar.fullname" . }}-{{ .Values.function.component }}-ca"
   {{- if .Values.createCertificates.selfSigned.includeDns }}
   - "{{ .Values.dnsName}}"
   {{- end }}

helm-chart-sources/pulsar/templates/function/function-configmap.yaml

@@ -62,6 +62,7 @@ data:
     workerHostname: "{{ template "pulsar.fullname" . }}-{{ .Values.function.component }}"
     workerPort: "6750"
     {{- if .Values.enableTls }}
+    tlsEnabled: "true"
     workerPortTls: "6751"
     tlsCertificateFilePath: "/pulsar/certs/tls.crt"
     {{- if or .Values.secrets .Values.createCertificates.selfSigned.enabled}}
@@ -74,10 +75,10 @@ data:
     tlsKeyFilePath: "/pulsar/tls-pk8.key"
     {{- end }}
     {{- if and .Values.enableTls .Values.tls.function.enabled }}
-    tlsEnabled: "true"
     useTls: "true"
-    pulsarServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:6651"
-    pulsarWebServiceUrl: "https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:8443"
+    tlsEnableHostnameVerification: "{{ .Values.tls.proxy.enableHostnameVerification }}"
+    pulsarServiceUrl: "pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}:6651"
+    pulsarWebServiceUrl: "https://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}:8443"
     {{- else }}
     pulsarServiceUrl: "pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:6650"
     pulsarWebServiceUrl: "http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}:8080"

helm-chart-sources/pulsar/templates/proxy/proxy-configmap.yaml

@@ -91,12 +91,17 @@ data:
   brokerServicePortTls: "6651"
   webServicePortTls: "8443"
   servicePortTls: "6651"
-  {{- if .Values.extra.function }}
-  {{- if .Values.tls.function.enabled }}
+  # Used to enable via .Values.tls.function.enabled, but that is deprecated now.
+  {{- if or .Values.tls.function.enabled .Values.tls.proxy.enabled }}
   tlsEnabledWithBroker: "true"
-  functionWorkerWebServiceURLTLS: "https://{{ template "pulsar.fullname" . }}-{{ .Values.function.component }}-ca:6751"
+  tlsHostnameVerificationEnabled: "{{ .Values.tls.proxy.enableHostnameVerification }}"
   {{- else }}
   tlsEnabledWithBroker: "false"
+  {{- end }}
+  {{- if .Values.extra.function }}
+  {{- if or .Values.tls.function.enabled .Values.tls.proxy.enabled }}
+  functionWorkerWebServiceURLTLS: "https://{{ template "pulsar.fullname" . }}-{{ .Values.function.component }}-ca:6751"
+  {{- else }}
   functionWorkerWebServiceURL: "http://{{ template "pulsar.fullname" . }}-{{ .Values.function.component }}-ca:6750"
   {{- end }}
   {{- end }}

helm-chart-sources/pulsar/templates/proxy/proxy-ws-configmap.yaml

@@ -61,7 +61,7 @@ data:
   PULSAR_PREFIX_openIDAllowedTokenIssuers: "{{ template "pulsar.get.http.or.https" . }}{{ template "pulsar.keycloak.fullname" .}}{{ template "pulsar.keycloak.issuer.port" .}}/auth/realms/{{ .Values.keycloak.realm }},{{ template "pulsar.get.http.or.https" . }}{{ template "pulsar.keycloak.fullname" .}}.{{ .Release.Namespace }}{{ template "pulsar.keycloak.issuer.port" .}}/auth/realms/{{ .Values.keycloak.realm }},{{ template "pulsar.get.http.or.https" . }}{{ template "pulsar.keycloak.fullname" .}}.{{ .Release.Namespace }}.svc.cluster.local{{ template "pulsar.keycloak.issuer.port" .}}/auth/realms/{{ .Values.keycloak.realm }}"
   {{- end }}
   {{- end }}
-{{- if and .Values.enableTls }}
+{{- if .Values.enableTls }}
   webServicePortTls: "{{ .Values.proxy.wsProxyPortTls }}"
   tlsEnabled: "true"
   tlsCertificateFilePath: /pulsar/certs/tls.crt

helm-chart-sources/pulsar/templates/zookeeper/zookeeper-metadata.yaml

@@ -76,8 +76,8 @@ spec:
               --zookeeper {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }} \
               --configuration-store {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }} \
               {{- if .Values.enableTls }}
-              --web-service-url-tls https://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ .Release.Namespace }}:8443/ \
-              --broker-service-url-tls pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ .Release.Namespace }}:6651/ \
+              --web-service-url-tls https://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ .Release.Namespace }}:8443/ \
+              --broker-service-url-tls pulsar+ssl://{{ template "pulsar.fullname" . }}-{{ .Values.brokerSts.component }}.{{ .Release.Namespace }}:6651/ \
               {{- end }}
               --web-service-url http://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ .Release.Namespace }}:8080/ \
               --broker-service-url pulsar://{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}.{{ .Release.Namespace }}:6650/;

helm-chart-sources/pulsar/values.yaml

@@ -115,12 +115,14 @@ default_storage:
 # When you enable TLS and are using a proxy, you need to expose the
 # TLS-enabled ports on the service. To allow TLS connections only, remove the plain-text ports.
 # See the proxy and broker sections for details.
-
+# This flag enables TLS for all client and admin facing components: broker, proxy, websocket proxy, standalone function
+# worker. You must deploy the broker as a StatefulSet for hostname verification to work. The function worker and the
+# proxy will network with the brokers over TLS when this flag is set to true.
 enableTls: false
 tlsSecretName: pulsar-tls
 
-# By default, TLS is enabled on the client- or admin-facing components (broker, proxy, websocket
-# proxy, standalone function worker).
+# When enableTls is true, TLS is enabled on the client- or admin-facing components (broker, proxy, websocket
+# proxy, standalone function worker) by default.
 # For added security, you can also enable TLS between the internal components (zookeeper, bookkeeper,
 # function worker [connect to broker])
 tls:
@@ -138,12 +140,28 @@ tls:
   # Enable TLS between broker and BookKeeper
   bookkeeper:
     enabled: false
+
+  ## TLS is enabled for the function worker, broker, and proxy when enableTls is true. The below <component>.enabled
+  ## flags are used to determine whether the component's client should use TLS when connecting to the broker or the
+  ## function worker. This is an inversion of the paradigm used for the zookeeper and bookkeeper configurations above,
+  ## which is used to enable TLS for all components interacting with bookkeeper or zookeeper.
+
   # Enable TLS between function worker and broker
+  # NOTE: the function worker's connection to the broker is only over TLS if brokerClientAuthenticationEnabled is true
+  # or if authenticationEnabled is true in the function's configuration.
   function:
     enabled: false
+    # NOTE: temporarily false to allow for easy transition. In next major version bump, this will default to true.
+    enableHostnameVerification: false
   # Enable TLS between WebSocket proxy and broker
   websocket:
     enabled: false
+  # Enable TLS between proxy and broker and between proxy and function worker
+  proxy:
+    enabled: false
+    # Applies to upstream broker and function worker TLS connections.
+    # NOTE: temporarily false to allow for easy transition. In next major version bump, this will default to true.
+    enableHostnameVerification: false
 
 # secrets:
   ## If you're providing your own certificates, please use this to add the certificates as secrets