Permalink
Switch branches/tags
Find file Copy path
313 lines (312 sloc) 10.5 KB
# This configuration file sets up a secure StatefulSet running CockroachDB with
# tweaks to make it more performant than our default configuration files. All
# changes from the default secure configuration have been marked with a comment
# starting with "NOTE" or "TODO".
#
# To use it, customize all the parts of the file labeled "TODO" before running:
# kubectl create -f cockroachdb-statefulset-secure.yaml
#
# You will then have to approve certificate-signing requests and initialize the
# cluster as described in the parent directory's README.md file.
#
# If you don't see any pods being created, it's possible that your cluster was
# not able to meet the resource requests asked for, whether it was the amount
# of CPU, memory, or disk or the disk type. To find information about why pods
# haven't been created, you can run:
# kubectl get events
#
# For more information on improving CockroachDB performance in Kubernetes, see
# our docs:
# https://www.cockroachlabs.com/docs/stable/kubernetes-performance.html
apiVersion: v1
kind: ServiceAccount
metadata:
name: cockroachdb
labels:
app: cockroachdb
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: cockroachdb
labels:
app: cockroachdb
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- create
- get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: cockroachdb
labels:
app: cockroachdb
rules:
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests
verbs:
- create
- get
- watch
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: cockroachdb
labels:
app: cockroachdb
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: cockroachdb
subjects:
- kind: ServiceAccount
name: cockroachdb
namespace: default
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: cockroachdb
labels:
app: cockroachdb
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cockroachdb
subjects:
- kind: ServiceAccount
name: cockroachdb
namespace: default
---
apiVersion: v1
kind: Service
metadata:
# This service is meant to be used by clients of the database. It exposes a ClusterIP that will
# automatically load balance connections to the different database pods.
name: cockroachdb-public
labels:
app: cockroachdb
spec:
ports:
# The main port, served by gRPC, serves Postgres-flavor SQL, internode
# traffic and the cli.
- port: 26257
targetPort: 26257
name: grpc
# The secondary port serves the UI as well as health and debug endpoints.
- port: 8080
targetPort: 8080
name: http
selector:
app: cockroachdb
---
apiVersion: v1
kind: Service
metadata:
# This service only exists to create DNS entries for each pod in the stateful
# set such that they can resolve each other's IP addresses. It does not
# create a load-balanced ClusterIP and should not be used directly by clients
# in most circumstances.
name: cockroachdb
labels:
app: cockroachdb
annotations:
# Use this annotation in addition to the actual publishNotReadyAddresses
# field below because the annotation will stop being respected soon but the
# field is broken in some versions of Kubernetes:
# https://github.com/kubernetes/kubernetes/issues/58662
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
# Enable automatic monitoring of all instances when Prometheus is running in the cluster.
prometheus.io/scrape: "true"
prometheus.io/path: "_status/vars"
prometheus.io/port: "8080"
spec:
ports:
- port: 26257
targetPort: 26257
name: grpc
- port: 8080
targetPort: 8080
name: http
# We want all pods in the StatefulSet to have their addresses published for
# the sake of the other CockroachDB pods even before they're ready, since they
# have to be able to talk to each other in order to become ready.
publishNotReadyAddresses: true
clusterIP: None
selector:
app: cockroachdb
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: cockroachdb-budget
labels:
app: cockroachdb
spec:
selector:
matchLabels:
app: cockroachdb
maxUnavailable: 1
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: cockroachdb
spec:
serviceName: "cockroachdb"
replicas: 3
template:
metadata:
labels:
app: cockroachdb
spec:
serviceAccountName: cockroachdb
# NOTE: Running with `hostNetwork: true` means that CockroachDB will use
# the host machines' IP address and hostname, and that nothing else on
# the machines will be able to use the same ports. This means that only 1
# CockroachDB pod will ever be schedulable on the same machine, because
# otherwise their ports would conflict.
#
# If your client pods generate a lot of network traffic to and from the
# CockroachDB cluster, you may see a benefit to doing the same thing in
# their configurations.
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
# Init containers are run only once in the lifetime of a pod, before
# it's started up for the first time. It has to exit successfully
# before the pod's main containers are allowed to start.
initContainers:
# The init-certs container sends a certificate signing request to the
# kubernetes cluster.
# You can see pending requests using: kubectl get csr
# CSRs can be approved using: kubectl certificate approve <csr name>
#
# All addresses used to contact a node must be specified in the --addresses arg.
#
# In addition to the node certificate and key, the init-certs entrypoint will symlink
# the cluster CA to the certs directory.
- name: init-certs
image: cockroachdb/cockroach-k8s-request-cert:0.4
imagePullPolicy: IfNotPresent
command:
- "/bin/ash"
- "-ecx"
- "/request-cert -namespace=${POD_NAMESPACE} -certs-dir=/cockroach-certs -type=node -addresses=localhost,127.0.0.1,$(hostname -f),$(hostname -f|cut -f 1-2 -d '.'),cockroachdb-public,cockroachdb-public.$(hostname -f|cut -f 3- -d '.') -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: certs
mountPath: /cockroach-certs
# NOTE: If you are running clients that generate heavy load, you may find
# it useful to copy this anti-affinity policy into the client pods'
# configurations as well to avoid running them on the same machines as
# CockroachDB and interfering with each other's performance.
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- cockroachdb
topologyKey: kubernetes.io/hostname
containers:
- name: cockroachdb
# NOTE: Always use the most recent version of CockroachDB for the best
# performance and reliability.
image: cockroachdb/cockroach:v2.1.2
imagePullPolicy: IfNotPresent
# TODO: Change these to appropriate values for the hardware that you're running. You can see
# the amount of allocatable resources on each of your Kubernetes nodes by running:
# kubectl describe nodes
resources:
requests:
cpu: "16"
memory: "8Gi"
limits:
# NOTE: Unless you have enabled the non-default Static CPU Management Policy
# and are using an integer number of CPUs, we don't recommend setting a CPU limit.
# See:
# https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy
# https://github.com/kubernetes/kubernetes/issues/51135
#cpu: "16"
memory: "8Gi"
ports:
- containerPort: 26257
name: grpc
- containerPort: 8080
name: http
livenessProbe:
httpGet:
path: "/health"
port: http
scheme: HTTPS
initialDelaySeconds: 30
periodSeconds: 5
readinessProbe:
httpGet:
path: "/health?ready=1"
port: http
scheme: HTTPS
initialDelaySeconds: 10
periodSeconds: 5
failureThreshold: 2
volumeMounts:
- name: datadir
mountPath: /cockroach/cockroach-data
- name: certs
mountPath: /cockroach/cockroach-certs
env:
- name: COCKROACH_CHANNEL
value: kubernetes-secure
command:
- "/bin/bash"
- "-ecx"
# The use of qualified `hostname -f` is crucial:
# Other nodes aren't able to look up the unqualified hostname.
- "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-host 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%"
# No pre-stop hook is required, a SIGTERM plus some time is all that's
# needed for graceful shutdown of a node.
terminationGracePeriodSeconds: 60
volumes:
- name: datadir
persistentVolumeClaim:
claimName: datadir
- name: certs
emptyDir: {}
podManagementPolicy: Parallel
updateStrategy:
type: RollingUpdate
volumeClaimTemplates:
- metadata:
name: datadir
spec:
accessModes:
- "ReadWriteOnce"
# TODO: This specifically asks for a storage class with the name "ssd". A
# storage class of this name doesn't exist by default. See our docs for
# more information on how to create an optimized storage class for use here:
# https://www.cockroachlabs.com/docs/stable/orchestrate-cockroachdb-with-kubernetes-performance.html#disk-type
storageClassName: ssd
resources:
requests:
# TODO: This asks for a fairly large disk by default because on
# certain popular clouds there is a direct correlation between disk
# size and the IOPS provisioned to the disk. Change this as necessary
# to suit your needs, but be aware that smaller disks will typically
# mean worse performance.
storage: 1024Gi