From 3140200a5d4d3c120647ae28126b3bfb8424cdd7 Mon Sep 17 00:00:00 2001 From: Thomas Kooi Date: Tue, 3 Jul 2018 20:33:03 +0200 Subject: [PATCH 1/3] Add support for connecting to tiller using tls - Add support for connecting to tiller using TLS authentication. It is based on the way the helm client itself configures the TLS authentication. The default is tls disabled. - Updates the helm chart to support the TLS options. Expects the helm client certificates to be stored in a k8s secret in the same namespace as the helm-operator is deployed. The secretName can be passed along using the helm-deployment. When configuring TLS Verify, a CA certificate has to be passed and this will be created as a ConfigMap during the deployment. Closes #1198 --- .../templates/helm-operator-deployment.yaml | 32 +++++++++++++++ chart/flux/templates/helm-tls.yaml | 10 +++++ chart/flux/values.yaml | 8 ++++ cmd/helm-operator/main.go | 24 ++++++++++- integrations/helm/helm.go | 23 ++++++++++- site/helm/helm-integration.md | 41 +++++++++++-------- 6 files changed, 118 insertions(+), 20 deletions(-) create mode 100644 chart/flux/templates/helm-tls.yaml diff --git a/chart/flux/templates/helm-operator-deployment.yaml b/chart/flux/templates/helm-operator-deployment.yaml index f9a589a98..515003556 100644 --- a/chart/flux/templates/helm-operator-deployment.yaml +++ b/chart/flux/templates/helm-operator-deployment.yaml @@ -32,6 +32,18 @@ spec: secret: secretName: {{ template "flux.fullname" . }}-git-deploy defaultMode: 0400 + {{- if .Values.helmOperator.tls.enable }} + - name: helm-tls-certs + secret: + secretName: {{ .Values.helmOperator.tls.secretName }} + defaultMode: 0400 + {{- if .Values.helmOperator.tls.verify }} + - name: helm-tls-ca + configMap: + name: {{ template "flux.fullname" . }}-helm-tls-ca-config + defaultMode: 0600 + {{- end }} + {{- end }} containers: - name: flux-helm-operator image: "{{ .Values.helmOperator.repository }}:{{ .Values.helmOperator.tag }}" @@ -44,8 +56,28 @@ spec: - name: git-key mountPath: /etc/fluxd/ssh readOnly: true + {{- if .Values.helmOperator.tls.enable }} + - name: helm-tls-certs + mountPath: /etc/fluxd/helm + readOnly: true + {{- if .Values.helmOperator.tls.verify }} + - name: helm-tls-ca + mountPath: /etc/fluxd/helm-ca + readOnly: true + {{- end }} + {{- end }} args: - --git-url={{ .Values.git.url }} - --git-branch={{ .Values.git.branch }} - --git-charts-path={{ .Values.git.chartsPath }} + - --tiller-namespace={{ .Values.helmOperator.tillerNamespace }} + {{- if .Values.helmOperator.tls.enable }} + - --tiller-tls-enable={{ .Values.helmOperator.tls.enable }} + - --tiller-tls-key-path={{ .Values.helmOperator.tls.keyPath }} + - --tiller-tls-cert-path={{ .Values.helmOperator.tls.certPath }} + {{- if .Values.helmOperator.tls.verify }} + - --tiller-tls-verify={{ .Values.helmOperator.tls.verify }} + - --tiller-tls-ca-cert-path=/etc/fluxd/helm-ca/ca.crt + {{- end }} + {{- end }} {{- end -}} diff --git a/chart/flux/templates/helm-tls.yaml b/chart/flux/templates/helm-tls.yaml new file mode 100644 index 000000000..a1731c7a8 --- /dev/null +++ b/chart/flux/templates/helm-tls.yaml @@ -0,0 +1,10 @@ +{{- if .Values.helmOperator.tls.enable -}} +{{- if .Values.helmOperator.tls.verify -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "flux.fullname" . }}-helm-tls-ca-config +data: + ca.crt: {{ .Values.helmOperator.tls.caContent }} +{{- end -}} +{{- end -}} diff --git a/chart/flux/values.yaml b/chart/flux/values.yaml index ebda0124a..399d1e66f 100644 --- a/chart/flux/values.yaml +++ b/chart/flux/values.yaml @@ -19,6 +19,14 @@ helmOperator: repository: quay.io/weaveworks/helm-operator tag: 0.1.0-alpha pullPolicy: IfNotPresent + tillerNamespace: kube-system + tls: + secretName: 'helm-client-certs' + verify: false + enable: false + keyPath: '/etc/fluxd/helm/tls.key' + certPath: '/etc/fluxd/helm/tls.crt' + caContent: '' rbac: # Specifies whether RBAC resources should be created diff --git a/cmd/helm-operator/main.go b/cmd/helm-operator/main.go index ac9a5c5ca..aa2577577 100644 --- a/cmd/helm-operator/main.go +++ b/cmd/helm-operator/main.go @@ -45,6 +45,12 @@ var ( tillerPort *string tillerNamespace *string + tillerTLSVerify *bool + tillerTLSEnable *bool + tillerTLSKey *string + tillerTLSCert *string + tillerTLSCACert *string + chartsSyncInterval *time.Duration chartsSyncTimeout *time.Duration eventHandlerWorkers *uint @@ -91,6 +97,12 @@ func init() { tillerPort = fs.String("tiller-port", "", "Tiller port.") tillerNamespace = fs.String("tiller-namespace", "kube-system", "Tiller namespace. If not provided, the default is kube-system.") + tillerTLSVerify = fs.Bool("tiller-tls-verify", false, "Verify TLS certificate from Tiller. Will enable TLS communication when provided.") + tillerTLSEnable = fs.Bool("tiller-tls-enable", false, "Enable TLS communication with Tiller. If provided, requires TLSKey and TLSCert to be provided as well.") + tillerTLSKey = fs.String("tiller-tls-key-path", "/etc/fluxd/helm/tls.key", "Path to private key file used to communicate with the Tiller server.") + tillerTLSCert = fs.String("tiller-tls-cert-path", "/etc/fluxd/helm/tls.crt", "Path to certificate file used to communicate with the Tiller server.") + tillerTLSCACert = fs.String("tiller-tls-ca-cert-path", "", "Path to CA certificate file used to validate the Tiller server. Required if tiller-tls-verify is enabled.") + chartsSyncInterval = fs.Duration("charts-sync-interval", 3*time.Minute, "Interval at which to check for changed charts") chartsSyncTimeout = fs.Duration("charts-sync-timeout", 1*time.Minute, "Timeout when checking for changed charts") eventHandlerWorkers = fs.Uint("event-handler-workers", 2, "Number of workers processing events for Flux-Helm custom resources") @@ -162,7 +174,17 @@ func main() { } // HELM --------------------------------------------------------------------------------- - helmClient := fluxhelm.ClientSetup(log.With(logger, "component", "helm"), kubeClient, fluxhelm.TillerOptions{IP: *tillerIP, Port: *tillerPort, Namespace: *tillerNamespace}) + helmClient := fluxhelm.ClientSetup(log.With(logger, "component", "helm"), kubeClient, fluxhelm.TillerOptions{ + IP: *tillerIP, + Port: *tillerPort, + Namespace: *tillerNamespace, + + TLSVerify: *tillerTLSVerify, + TLSEnable: *tillerTLSEnable, + TLSKey: *tillerTLSKey, + TLSCert: *tillerTLSCert, + TLSCACert: *tillerTLSCACert, + }) // The status updater, to keep track the release status for each // FluxHelmRelease. It runs as a separate loop for now. diff --git a/integrations/helm/helm.go b/integrations/helm/helm.go index 4e02cfb09..3a4771f7b 100644 --- a/integrations/helm/helm.go +++ b/integrations/helm/helm.go @@ -10,12 +10,18 @@ import ( "k8s.io/client-go/kubernetes" k8shelm "k8s.io/helm/pkg/helm" rls "k8s.io/helm/pkg/proto/hapi/services" + "k8s.io/helm/pkg/tlsutil" ) type TillerOptions struct { IP string Port string Namespace string + TLSVerify bool + TLSEnable bool + TLSKey string + TLSCert string + TLSCACert string } // Helm struct provides access to helm client @@ -32,7 +38,22 @@ func newClient(kubeClient *kubernetes.Clientset, opts TillerOptions) (*k8shelm.C return &k8shelm.Client{}, err } - return k8shelm.NewClient(k8shelm.Host(host)), nil + options := []k8shelm.Option{k8shelm.Host(host)} + if opts.TLSVerify || opts.TLSEnable { + tlscfg, err := tlsutil.ClientConfig(tlsutil.Options{ + KeyFile: opts.TLSKey, + CertFile: opts.TLSCert, + InsecureSkipVerify: !opts.TLSVerify, + CaCertFile: opts.TLSCACert, + }) + + if err != nil { + return &k8shelm.Client{}, err + } + options = append(options, k8shelm.WithTLS(tlscfg)) + } + + return k8shelm.NewClient(options...), nil } func ClientSetup(logger log.Logger, kubeClient *kubernetes.Clientset, tillerOpts TillerOptions) *k8shelm.Client { diff --git a/site/helm/helm-integration.md b/site/helm/helm-integration.md index f30c9526b..cb97161d8 100644 --- a/site/helm/helm-integration.md +++ b/site/helm/helm-integration.md @@ -65,24 +65,29 @@ helm-operator requires setup and offers customization though a multitude of flag |flag | default | purpose | |------------------------|-------------------------------|---------| -|--kubernetes-kubectl | | Optional, explicit path to kubectl tool.| -|--kubeconfig | | Path to a kubeconfig. Only required if out-of-cluster.| -|--master | | The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.| -| | | **Tiller options**| -|--tillerIP | | Tiller IP address. Only required if out-of-cluster.| -|--tillerPort | | Tiller port.| -|--tillerNamespace | | Tiller namespace. If not provided, the default is kube-system.| -| | | **Git repo & key etc.**| -|--git-url | | URL of git repo with Helm Charts; e.g., `ssh://git@github.com/weaveworks/flux-example`| -|--git-branch | `master` | Branch of git repo to use for Kubernetes manifests| -|--git-charts-path | `charts` | Path within git repo to locate Kubernetes Charts (relative path)| -| | | **repo chart changes** (none of these need overriding, usually) | -|--git-poll-interval | `5 minutes` | period at which to poll git repo for new commits| -|--chartsSyncInterval | 3*time.Minute | Interval at which to check for changed charts.| -|--chartsSyncTimeout | 1*time.Minute | Timeout when checking for changed charts.| -| | | **k8s-secret backed ssh keyring configuration**| +|--kubernetes-kubectl | | Optional, explicit path to kubectl tool.| +|--kubeconfig | | Path to a kubeconfig. Only required if out-of-cluster.| +|--master | | The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.| +| | | **Tiller options**| +|--tillerIP | | Tiller IP address. Only required if out-of-cluster.| +|--tillerPort | | Tiller port.| +|--tillerNamespace | | Tiller namespace. If not provided, the default is kube-system.| | +|--tiller-tls-enable |`false` | Enable TLS communication with Tiller. If provided, requires TLSKey and TLSCert to be provided as well. | +|--tiller-tls-verify |`false` | Verify TLS certificate from Tiller. Will enable TLS communication when provided. | +|--tiller-tls-tls-key-path |`/etc/fluxd/helm/tls.key` | Path to private key file used to communicate with the Tiller server. | +|--tiller-tls-tls-cert-path |`/etc/fluxd/helm/tls.crt` | Path to certificate file used to communicate with the Tiller server. | +|--tiller-tls-tls-ca-cert-path | | Path to CA certificate file used to validate the Tiller server. Required if tiller-tls-verify is enabled. | +| | | **Git repo & key etc.**| +|--git-url | | URL of git repo with Helm Charts; e.g., `ssh://git@github.com/weaveworks/flux-example`| +|--git-branch | `master` | Branch of git repo to use for Kubernetes manifests| +|--git-charts-path | `charts` | Path within git repo to locate Kubernetes Charts (relative path)| +| | | **repo chart changes** (none of these need overriding, usually) | +|--git-poll-interval | `5 minutes` | period at which to poll git repo for new commits| +|--chartsSyncInterval | 3*time.Minute | Interval at which to check for changed charts.| +|--chartsSyncTimeout | 1*time.Minute | Timeout when checking for changed charts.| +| | | **k8s-secret backed ssh keyring configuration**| |--k8s-secret-volume-mount-path | `/etc/fluxd/ssh` | Mount location of the k8s secret storing the private SSH key| -|--k8s-secret-data-key | `identity` | Data key holding the private SSH key within the k8s secret| -|--queueWorkerCount | 2 | Number of workers to process queue with Chart release jobs.| +|--k8s-secret-data-key | `identity` | Data key holding the private SSH key within the k8s secret| +|--queueWorkerCount | 2 | Number of workers to process queue with Chart release jobs.| [Requirements](./helm-integration-requirements.md) From 02fb7b1ccb31bb26156276fd04afc3da689be16d Mon Sep 17 00:00:00 2001 From: Thomas Kooi Date: Wed, 4 Jul 2018 17:32:25 +0200 Subject: [PATCH 2/3] Change helm chart to only template tls key and cert file names instead of the full path - Includes a fix for the CA Certificate ConfigMap. The ca.crt file content was not properly templated. - Instead of the full path, a user only has the option to configure the key and certificate file name, as the path is hardcoded within the template. - Add new helmOperator properties to the chart's README file --- chart/flux/README.md | 7 +++++++ chart/flux/templates/helm-operator-deployment.yaml | 4 ++-- chart/flux/templates/helm-tls.yaml | 3 ++- chart/flux/values.yaml | 4 ++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/chart/flux/README.md b/chart/flux/README.md index 379404df0..8a8611cb9 100755 --- a/chart/flux/README.md +++ b/chart/flux/README.md @@ -104,6 +104,13 @@ The following tables lists the configurable parameters of the Weave Flux chart a | `helmOperator.repository` | Helm operator image repository | `quay.io/weaveworks/helm-operator` | `helmOperator.tag` | Helm operator image tag | `0.1.0-alpha` | `helmOperator.pullPolicy` | Helm operator image pull policy | `IfNotPresent` +| `helmOperator.tillerNamespace` | Namespace in which the Tiller server can be found | `kube-system` +| `helmOperator.tls.enable` | Enable TLS for communicating with Tiller | `false` +| `helmOperator.tls.verify` | Verify the Tiller certificate, also enables TLS when set to true | `false` +| `helmOperator.tls.secretName` | Name of the secret containing the TLS client certificates for communicating with Tiller | `helm-client-certs` +| `helmOperator.tls.keyFile` | Name of the key file within the k8s secret | `tls.key` +| `helmOperator.tls.certFile` | Name of the certificate file within the k8s secret | `tls.crt` +| `helmOperator.tls.caContent` | Certificate Authority content used to validate the Tiller server certificate | None | `token` | Weave Cloud service token | None Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example: diff --git a/chart/flux/templates/helm-operator-deployment.yaml b/chart/flux/templates/helm-operator-deployment.yaml index 515003556..a214566b9 100644 --- a/chart/flux/templates/helm-operator-deployment.yaml +++ b/chart/flux/templates/helm-operator-deployment.yaml @@ -73,8 +73,8 @@ spec: - --tiller-namespace={{ .Values.helmOperator.tillerNamespace }} {{- if .Values.helmOperator.tls.enable }} - --tiller-tls-enable={{ .Values.helmOperator.tls.enable }} - - --tiller-tls-key-path={{ .Values.helmOperator.tls.keyPath }} - - --tiller-tls-cert-path={{ .Values.helmOperator.tls.certPath }} + - --tiller-tls-key-path=/etc/fluxd/helm/{{ .Values.helmOperator.tls.keyFile }} + - --tiller-tls-cert-path=/etc/fluxd/helm/{{ .Values.helmOperator.tls.certFile }} {{- if .Values.helmOperator.tls.verify }} - --tiller-tls-verify={{ .Values.helmOperator.tls.verify }} - --tiller-tls-ca-cert-path=/etc/fluxd/helm-ca/ca.crt diff --git a/chart/flux/templates/helm-tls.yaml b/chart/flux/templates/helm-tls.yaml index a1731c7a8..1f926bd97 100644 --- a/chart/flux/templates/helm-tls.yaml +++ b/chart/flux/templates/helm-tls.yaml @@ -5,6 +5,7 @@ kind: ConfigMap metadata: name: {{ template "flux.fullname" . }}-helm-tls-ca-config data: - ca.crt: {{ .Values.helmOperator.tls.caContent }} + ca.crt: | + {{ .Values.helmOperator.tls.caContent | indent 4 }} {{- end -}} {{- end -}} diff --git a/chart/flux/values.yaml b/chart/flux/values.yaml index 399d1e66f..4308697d7 100644 --- a/chart/flux/values.yaml +++ b/chart/flux/values.yaml @@ -24,8 +24,8 @@ helmOperator: secretName: 'helm-client-certs' verify: false enable: false - keyPath: '/etc/fluxd/helm/tls.key' - certPath: '/etc/fluxd/helm/tls.crt' + keyFile: 'tls.key' + certFile: 'tls.crt' caContent: '' rbac: From 8fdf91708509b4aa82dbdc77832eb039bdf02c3d Mon Sep 17 00:00:00 2001 From: Thomas Kooi Date: Wed, 4 Jul 2018 18:58:49 +0200 Subject: [PATCH 3/3] Add documentation on how to configure the helm-operator and tiller with TLS Expans the flux chart README file with a section on how to configure the Helm Operator and install Tiller with TLS authentication enabled. --- chart/flux/README.md | 184 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) diff --git a/chart/flux/README.md b/chart/flux/README.md index 8a8611cb9..24e5274fd 100755 --- a/chart/flux/README.md +++ b/chart/flux/README.md @@ -133,5 +133,189 @@ helm upgrade --reuse-values flux \ weaveworks/flux ``` +## Installing Weave Flux helm-operator and Helm with TLS enabled +### Installing Helm / Tiller +Generate certificates for Tiller and Flux. This will provide a CA, server certs for tiller and client certs for helm / weave flux. + +The following script can be used for that (requires [cfssl](https://github.com/cloudflare/cfssl)): + +```bash +export TILLER_HOSTNAME=tiller-server +export TILLER_SERVER=server +export USER_NAME=flux-helm-operator + +mkdir tls +cd ./tls + +# Prep the configuration +echo '{"CN":"CA","key":{"algo":"rsa","size":4096}}' | cfssl gencert -initca - | cfssljson -bare ca - +echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","server auth","client auth"]}}}' > ca-config.json + +# Create the tiller certificate +echo '{"CN":"'$TILLER_SERVER'","hosts":[""],"key":{"algo":"rsa","size":4096}}' | cfssl gencert \ + -config=ca-config.json -ca=ca.pem \ + -ca-key=ca-key.pem \ + -hostname="$TILLER_HOSTNAME" - | cfssljson -bare $TILLER_SERVER + +# Create a client certificate +echo '{"CN":"'$USER_NAME'","hosts":[""],"key":{"algo":"rsa","size":4096}}' | cfssl gencert \ + -config=ca-config.json -ca=ca.pem -ca-key=ca-key.pem \ + -hostname="$TILLER_HOSTNAME" - | cfssljson -bare $USER_NAME +``` + +Alternatively, you can follow the [Helm documentation for configuring TLS](https://docs.helm.sh/using_helm/#using-ssl-between-helm-and-tiller). + + +Next deploy Helm with TLS and RBAC enabled; + +Create a file called `helm-rbac.yaml`. This contains all the RBAC configuration for Tiller: +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tiller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: tiller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: tiller + namespace: kube-system + +--- +# Helm client serviceaccount +apiVersion: v1 +kind: ServiceAccount +metadata: + name: helm + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: tiller-user + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - pods/portforward + verbs: + - create +- apiGroups: + - "" + resources: + - pods + verbs: + - list +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: tiller-user-binding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: tiller-user +subjects: +- kind: ServiceAccount + name: helm + namespace: kube-system +``` + +Deploy Tiller: + +```bash +kubectl apply -f helm-rbac.yaml + +# Deploy helm with mutual TLS enabled +helm init --upgrade --service-account tiller \ + --override 'spec.template.spec.containers[0].command'='{/tiller,--storage=secret}' \ + --tiller-tls \ + --tiller-tls-cert ./tls/server.pem \ + --tiller-tls-key ./tls/server-key.pem \ + --tiller-tls-verify \ + --tls-ca-cert ./tls/ca.pem +``` + +To check if Tiller installed succesfully with TLS enabled, try `helm ls`. This should give an error: + +```bash +# Should give an error +$ helm ls +Error: transport is closing +``` + +When providing the certificates, it should work correctly: + +```bash +helm --tls \ + --tls-ca-cert ./tls/ca.pem \ + --tls-cert ./tls/flux-helm-operator.pem \ + --tls-key ././tls/flux-helm-operator-key.pem \ + ls +``` + +## deploy weave flux helm-operator + +First create a new Kubernetes TLS secret for the client certs; + +```bash +kubectl create secret tls helm-client --cert=tls/flux-helm-operator.pem --key=./tls/flux-helm-operator-key.pem +``` + +> note; this has to be in the same namespace as the helm-operator is deployed in. + +Deploy flux with Helm; + +```bash +helm repo add weaveworks https://weaveworks.github.io/flux + +helm upgrade --install \ + --set helmOperator.create=true \ + --set git.url=$YOUR_GIT_REPO \ + --set helmOperator.tls.enable=true \ + --set helmOperator.tls.verify=true \ + --set helmOperator.tls.secretName=helm-client \ + --set helmOperator.tls.caContent="$(cat ./tls/ca.pem)" \ + flux \ + weaveworks/flux +``` + +### Check if it worked + +Use `kubectl logs` on the helm-operator and observe the helm client being created. + +### Debugging + +#### Error creating helm client: failed to append certificates from file: /etc/fluxd/helm-ca/ca.crt + +Your CA certificate content is not set correctly, check if your configMap contains the correct values. Example: + +```bash +$ kubectl get configmaps flux-helm-tls-ca-config -o yaml +apiVersion: v1 +data: + ca.crt: | + -----BEGIN CERTIFICATE----- + .... + -----END CERTIFICATE----- +kind: ConfigMap +metadata: + creationTimestamp: 2018-07-04T15:27:25Z + name: flux-helm-tls-ca-config + namespace: helm-system + resourceVersion: "1267257" + selfLink: /api/v1/namespaces/helm-system/configmaps/flux-helm-tls-ca-config + uid: c106f866-7f9e-11e8-904a-025000000001 +```