Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
216 lines (215 sloc) 8.1 KB
# This configuration file sets up an insecure StatefulSet running CockroachDB with
# tweaks to make it more performant than our default configuration files. All
# changes from the default insecure configuration have been marked with a comment
# starting with "NOTE" or "TODO".
#
# Beware that this configuration is quite insecure. By default, it will make
# CockroachDB accesible on port 26257 on your Kubernetes nodes' network
# interfaces, meaning that if your nodes are reachable from the Internet, then
# this CockroachDB cluster will be too. To disable this behavior, remove the
# `hostNetwork` configuration field below.
#
# To use this file, customize all the parts labeled "TODO" before running:
# kubectl create -f cockroachdb-statefulset-insecure.yaml
#
# You will then have to initialize the cluster as described in the parent
# directory's README.md file.
#
# If you don't see any pods being created, it's possible that your cluster was
# not able to meet the resource requests asked for, whether it was the amount
# of CPU, memory, or disk or the disk type. To find information about why pods
# haven't been created, you can run:
# kubectl get events
#
# For more information on improving CockroachDB performance in Kubernetes, see
# our docs:
# https://www.cockroachlabs.com/docs/stable/kubernetes-performance.html
apiVersion: v1
kind: Service
metadata:
# This service is meant to be used by clients of the database. It exposes a ClusterIP that will
# automatically load balance connections to the different database pods.
name: cockroachdb-public
labels:
app: cockroachdb
spec:
ports:
# The main port, served by gRPC, serves Postgres-flavor SQL, internode
# traffic and the cli.
- port: 26257
targetPort: 26257
name: grpc
# The secondary port serves the UI as well as health and debug endpoints.
- port: 8080
targetPort: 8080
name: http
selector:
app: cockroachdb
---
apiVersion: v1
kind: Service
metadata:
# This service only exists to create DNS entries for each pod in the stateful
# set such that they can resolve each other's IP addresses. It does not
# create a load-balanced ClusterIP and should not be used directly by clients
# in most circumstances.
name: cockroachdb
labels:
app: cockroachdb
annotations:
# Use this annotation in addition to the actual publishNotReadyAddresses
# field below because the annotation will stop being respected soon but the
# field is broken in some versions of Kubernetes:
# https://github.com/kubernetes/kubernetes/issues/58662
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
# Enable automatic monitoring of all instances when Prometheus is running in the cluster.
prometheus.io/scrape: "true"
prometheus.io/path: "_status/vars"
prometheus.io/port: "8080"
spec:
ports:
- port: 26257
targetPort: 26257
name: grpc
- port: 8080
targetPort: 8080
name: http
# We want all pods in the StatefulSet to have their addresses published for
# the sake of the other CockroachDB pods even before they're ready, since they
# have to be able to talk to each other in order to become ready.
publishNotReadyAddresses: true
clusterIP: None
selector:
app: cockroachdb
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: cockroachdb-budget
labels:
app: cockroachdb
spec:
selector:
matchLabels:
app: cockroachdb
maxUnavailable: 1
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: cockroachdb
spec:
serviceName: "cockroachdb"
replicas: 3
template:
metadata:
labels:
app: cockroachdb
spec:
# NOTE: Running with `hostNetwork: true` means that CockroachDB will use
# the host machines' IP address and hostname, and that nothing else on
# the machines will be able to use the same ports. This means that only 1
# CockroachDB pod will ever be schedulable on the same machine, because
# otherwise their ports would conflict.
#
# If your client pods generate a lot of network traffic to and from the
# CockroachDB cluster, you may see a benefit to doing the same thing in
# their configurations.
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
# NOTE: If you are running clients that generate heavy load, you may find
# it useful to copy this anti-affinity policy into the client pods'
# configurations as well to avoid running them on the same machines as
# CockroachDB and interfering with each other's performance.
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- cockroachdb
topologyKey: kubernetes.io/hostname
containers:
- name: cockroachdb
# NOTE: Always use the most recent version of CockroachDB for the best
# performance and reliability.
image: cockroachdb/cockroach:v19.1.3
imagePullPolicy: IfNotPresent
# TODO: Change these to appropriate values for the hardware that you're running. You can see
# the amount of allocatable resources on each of your Kubernetes nodes by running:
# kubectl describe nodes
resources:
requests:
cpu: "16"
memory: "8Gi"
limits:
# NOTE: Unless you have enabled the non-default Static CPU Management Policy
# and are using an integer number of CPUs, we don't recommend setting a CPU limit.
# See:
# https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy
# https://github.com/kubernetes/kubernetes/issues/51135
#cpu: "16"
memory: "8Gi"
ports:
- containerPort: 26257
name: grpc
- containerPort: 8080
name: http
livenessProbe:
httpGet:
path: "/health"
port: http
initialDelaySeconds: 30
periodSeconds: 5
readinessProbe:
httpGet:
path: "/health?ready=1"
port: http
initialDelaySeconds: 10
periodSeconds: 5
failureThreshold: 2
volumeMounts:
- name: datadir
mountPath: /cockroach/cockroach-data
env:
- name: COCKROACH_CHANNEL
value: kubernetes-insecure
command:
- "/bin/bash"
- "-ecx"
# The use of qualified `hostname -f` is crucial:
# Other nodes aren't able to look up the unqualified hostname.
- "exec /cockroach/cockroach start --logtostderr --insecure --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%"
# No pre-stop hook is required, a SIGTERM plus some time is all that's
# needed for graceful shutdown of a node.
terminationGracePeriodSeconds: 60
volumes:
- name: datadir
persistentVolumeClaim:
claimName: datadir
podManagementPolicy: Parallel
updateStrategy:
type: RollingUpdate
volumeClaimTemplates:
- metadata:
name: datadir
spec:
accessModes:
- "ReadWriteOnce"
# TODO: This specifically asks for a storage class with the name "ssd". A
# storage class of this name doesn't exist by default. See our docs for
# more information on how to create an optimized storage class for use here:
# https://www.cockroachlabs.com/docs/stable/orchestrate-cockroachdb-with-kubernetes-performance.html#disk-type
storageClassName: ssd
resources:
requests:
# TODO: This asks for a fairly large disk by default because on
# certain popular clouds there is a direct correlation between disk
# size and the IOPS provisioned to the disk. Change this as necessary
# to suit your needs, but be aware that smaller disks will typically
# mean worse performance.
storage: 1024Gi
You can’t perform that action at this time.